123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475(* A verifiable toy task for language-model RL: two-digit addition.
The prompt "27+48=" asks for the completion "75" followed by an
end-of-sequence token. Answers have two or three digits, so completions
have different lengths. *)letplus=10letequals=11leteos=12letpad=13letvocab=14typeproblem={a:int;b:int;prompt:intarray;answer:intarray(* digits, then eos *)}letdigitsn=String.to_seq(string_of_intn)|>Seq.map(func->Char.codec-48)|>Array.of_seqletproblemab={a;b;prompt=Array.concat[digitsa;[|plus|];digitsb;[|equals|]];answer=Array.append(digits(a+b))[|eos|]}(* every pair of operands in [10, 99], shuffled with [seed] and split in two *)letsplit?(test_fraction=0.2)~seed()=letall=Array.init(90*90)(funi->problem(10+(i/90))(10+(imod90)))inletorder=Rollout.permutation~rng:(Random.State.make[|seed|])(Array.lengthall)inletn_test=int_of_float(test_fraction*.float_of_int(Array.lengthall))inletshuffled=Array.map(Array.getall)orderinArray.subshuffledn_test(Array.lengthall-n_test),Array.subshuffled0n_test(* longest completion the verifier can accept *)letmax_answer=4(* 1 for exactly the right digits followed by eos, 0 otherwise *)letoutcome_rewardpcompletion=ifcompletion=p.answerthen1.else0.(* one reward per generated token: 1 / |answer| for each token in the right
place, so a perfect completion still earns 1 in total *)letprocess_rewardspcompletion=letn=Array.lengthp.answerinArray.mapi(funit->ifi<n&&p.answer.(i)=tthen1./.float_of_intnelse0.)completion(* What generation and fine-tuning need to know about a task: the length
limit, the verifier, and the completion SFT imitates. *)typetask={max_len:int;reward:problem->intarray->float;target:Random.State.t->problem->intarray}letaddition={max_len=max_answer;reward=outcome_reward;target=(fun_p->p.answer)}(* For length-bias experiments (examples/llm/b7a_dr_grpo.ml). Up to [max_filler] filler tokens (pad) may follow
the digits before eos and the verifier ignores them, so completion length
varies without affecting the reward: any drift in length comes from the
training objective. SFT imitates a uniformly random amount of filler.
Contexts then grow beyond Lm's window and lose the start of the prompt,
but only after the digits have been written. *)letmax_filler=4letwithout_fillercompletion=Array.of_list(List.filter((<>)pad)(Array.to_listcompletion))letaddition_with_filler={max_len=max_answer+max_filler;reward=(funpcompletion->outcome_rewardp(without_fillercompletion));target=(funrngp->letdigits=Array.subp.answer0(Array.lengthp.answer-1)inArray.concat[digits;Array.make(Random.State.intrng(max_filler+1))pad;[|eos|]])}letto_stringtokens=Array.map(funt->ift<10thenstring_of_inttelseift=plusthen"+"elseift=equalsthen"="elseift=eosthen"$"else"_")tokens|>Array.to_list|>String.concat""