1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
module Make (Graph : Owl_neural_graph_sig.Sig) = struct
module L = Lm.Make (Graph)
type reward =
| Outcome
| Process
let make ?(reward = Outcome) ?(truncate_at_limit = false) ~rng (problems : Lm_task.problem array) : int L.E.t =
let observe (p, tokens) = L.encode (Array.append p.Lm_task.prompt (Array.of_list (List.rev tokens))) in
let advance (p, tokens) token =
let tokens = token :: tokens in
let completion = Array.of_list (List.rev tokens) in
let finished = token = Lm_task.eos || Array.length completion = Lm_task.max_answer in
let r =
match reward with
| Outcome -> if finished then Lm_task.outcome_reward p completion else 0.
| Process -> (Lm_task.process_rewards p completion).(Array.length completion - 1)
in
(p, tokens), r, if truncate_at_limit then token = Lm_task.eos else finished
in
L.E.simulator
~observation_space:(Space.box ~low:(Array.make L.input_size 0.) ~high:(Array.make L.input_size 1.) ~shape:[| L.input_size |])
~action_space:(Space.Discrete Lm_task.vocab) ~max_steps:(if truncate_at_limit then Lm_task.max_answer else Lm_task.max_answer + 1)
~reset_state:(fun () -> problems.(Random.State.int rng (Array.length problems)), [])
~observe ~advance
end
module S = Make (Owl.Neural.S.Graph)
module D = Make (Owl.Neural.D.Graph)