1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
module Make (Graph : Owl_neural_graph_sig.Sig) = struct
include Neural.Make (Graph)
open Algodiff
let vocab = Lm_task.vocab
let window = 6 + Lm_task.max_answer - 1
let input_size = window * vocab
let encode context =
let x = Array.make input_size 0. in
let len = Array.length context in
for k = 0 to window - 1 do
let j = len - window + k in
let t = if j < 0 then Lm_task.pad else context.(j) in
x.((k * vocab) + t) <- 1.
done;
x
let features contexts =
let flat = Array.concat (Array.to_list (Array.map encode contexts)) |> Array.map A.float_to_elt in
Arr (A.of_array flat [| Array.length contexts; input_size |])
let create ?(hidden = [ 128; 128 ]) ?(outputs = vocab) ~seed () =
mlp ~seed ~shape:[| input_size |] ~hidden ~outputs
let token_log_probs logits actions =
Distribution.Categorical.log_prob (Distribution.Categorical.of_output logits) (one_hot vocab actions)
type sample =
{ prompt : int array
; tokens : int array
; log_probs : float array
}
let generate ?(greedy = false) ?(max_len = Lm_task.max_answer) ~rng net prompts =
let n = Array.length prompts in
let tokens = Array.make n [] and logps = Array.make n [] in
let finished = Array.make n false in
for _ = 1 to max_len do
let active = List.filter (fun i -> not finished.(i)) (List.init n Fun.id) |> Array.of_list in
if Array.length active > 0
then (
let contexts = Array.map (fun i -> Array.append prompts.(i) (Array.of_list (List.rev tokens.(i)))) active in
let logp = Graph.run (features contexts) net |> Distribution.Categorical.of_output |> unpack_arr in
Array.iteri
(fun row i ->
let lp = Array.init vocab (fun v -> A.get logp [| row; v |] |> A.elt_to_float) in
let t = if greedy then Dp.argmax lp else Mdp.sample_from rng (List.init vocab (fun v -> v, exp lp.(v))) in
tokens.(i) <- t :: tokens.(i);
logps.(i) <- lp.(t) :: logps.(i);
if t = Lm_task.eos then finished.(i) <- true)
active)
done;
Array.init n (fun i ->
{ prompt = prompts.(i); tokens = Array.of_list (List.rev tokens.(i)); log_probs = Array.of_list (List.rev logps.(i)) })
let rows (completions : (int array * int array) array) =
let contexts = ref [] and actions = ref [] and sequence = ref [] in
Array.iteri
(fun s (prompt, tokens) ->
Array.iteri
(fun t token ->
contexts := Array.append prompt (Array.sub tokens 0 t) :: !contexts;
actions := token :: !actions;
sequence := s :: !sequence)
tokens)
completions;
let rev l = Array.of_list (List.rev l) in
rev !contexts, rev !actions, rev !sequence
let accuracy ?(sample = false) ?(task = Lm_task.addition) net (problems : Lm_task.problem array) =
let samples =
generate ~greedy:(not sample) ~max_len:task.max_len ~rng:(Random.State.make [| 0 |]) net
(Array.map (fun (p : Lm_task.problem) -> p.prompt) problems)
in
let correct = Array.map2 (fun p s -> task.reward p s.tokens) problems samples in
Array.fold_left ( +. ) 0. correct /. float_of_int (Array.length problems)
let sft ?(batch_size = 128) ?(learning_rate = 1e-3) ?(report = fun _ _ -> ()) ?(task = Lm_task.addition) ~steps ~rng
net (problems : Lm_task.problem array)
=
let stepper = Stepper.init (adam learning_rate) (Graph.mkpri net) in
for step = 1 to steps do
let batch = Array.init batch_size (fun _ -> problems.(Random.State.int rng (Array.length problems))) in
let contexts, actions, _ = rows (Array.map (fun (p : Lm_task.problem) -> p.prompt, task.target rng p) batch) in
let logits, _ = Graph.forward net (features contexts) in
let loss = Maths.(neg (mean (token_log_probs logits actions))) in
let value = unpack_flt loss in
update net stepper loss;
report step value
done
end
module S = Make (Owl.Neural.S.Graph)
module D = Make (Owl.Neural.D.Graph)