123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136(* Gymnasium environments as Ndenv values, run in a Python child process.
[seed] seeds the first reset, as in env.reset(seed=seed); later resets
continue from the environment's own generator, as Gymnasium does.
Observations must be a Box; actions are Discrete ([discrete]) or a Box
([continuous]). Terminated and truncated come from Gymnasium unchanged. *)openOwl_rllibletscript={|
try:
import gymnasium as gym
import numpy as np
env = gym.make(sys.argv[1], **({"render_mode": "rgb_array"} if sys.argv[3] == "1" else {}))
seed = int(sys.argv[2])
def space(s):
if isinstance(s, gym.spaces.Discrete) and s.start == 0:
return f'discrete {s.n}'
if isinstance(s, gym.spaces.Box):
return ' '.join(['box', str(len(s.shape)), *map(str, s.shape), hexes(s.low.ravel()), hexes(s.high.ravel())])
raise ValueError(f'unsupported space {s}')
reply(space(env.observation_space), space(env.action_space))
except Exception as e:
fail(e)
sys.exit(1)
def observation(obs):
return hexes(np.asarray(obs, dtype=np.float64).ravel())
def handle(command, args):
global seed
if command == 'reset':
obs, _ = env.reset(seed=seed)
seed = None
reply(observation(obs))
elif command == 'render':
frame = np.asarray(env.render(), dtype=np.uint8)
if frame.ndim != 3 or frame.shape[2] != 3:
raise ValueError('Gym: expected an RGB frame')
reply(frame.shape[1], frame.shape[0], frame.tobytes().hex())
elif command == 'step':
if isinstance(env.action_space, gym.spaces.Discrete):
action = int(args[0])
else:
action = np.array([float.fromhex(a) for a in args], dtype=env.action_space.dtype)
action = action.reshape(env.action_space.shape)
obs, reward, terminated, truncated, _ = env.step(action)
reply(float(reward).hex(), int(terminated), int(truncated), observation(obs))
serve(handle)
|}letrecsplitnxs=ifn=0then[],xselsematchxswith|x::rest->letfirst,rest=split(n-1)restinx::first,rest|[]->failwith"Gym: short reply"letfloatswords=Array.of_list(List.mapfloat_of_stringwords)letspace=function|"discrete"::n::rest->Space.Discrete(int_of_stringn),rest|"box"::rank::rest->letshape,rest=split(int_of_stringrank)restinletshape=Array.of_list(List.mapint_of_stringshape)inletn=Array.fold_left(*)1shapeinletlow,rest=splitnrestinlethigh,rest=splitnrestinSpace.box~low:(floatslow)~high:(floatshigh)~shape,rest|_->failwith"Gym: malformed space"moduleMake(A:Owl_types_ndarray_algodiff.Sig)=structletarrayshapewords=A.of_array(Array.mapA.float_to_elt(floatswords))shapeletmake~render~seedid~encode=letprocess,info=Python.start~name:("Gym "^id)~script[|id;string_of_intseed;ifrenderthen"1"else"0"|]inletobservation_space,rest=spaceinfoinletaction_space,_=spacerestinletshape=matchobservation_spacewith|Space.Boxb->b.shape|Discrete_->invalid_arg"Gym: observations must be a Box"inletended=reftrueinletreset()=letobs=arrayshape(Python.requestprocess"reset")inended:=false;obsinletstepaction=if!endedtheninvalid_arg"Ndenv.step: reset before stepping a new episode";matchPython.requestprocess("step "^encodeaction)with|reward::terminated::truncated::obs->letterminated=terminated="1"andtruncated=truncated="1"inended:=terminated||truncated;{Ndenv.obs=arrayshapeobs;reward=float_of_stringreward;terminated;truncated}|_->failwith("Gym "^id^": malformed step reply")inletframe()=matchPython.requestprocess"render"with|[width;height;hex]->letwidth=int_of_stringwidthandheight=int_of_stringheightinletrgb=Bytes.init(String.lengthhex/2)(funi->Char.chr(int_of_string("0x"^String.subhex(2*i)2)))inifBytes.lengthrgb<>width*height*3thenfailwith"Gym: short RGB frame";width,height,rgb|_->failwith"Gym: malformed frame reply"in{Ndenv.observation_space;action_space;reset;step},frameletdiscrete~seedid:(A.arr,int)Ndenv.t=letenv,_=make~render:false~seedid~encode:string_of_intin(matchenv.action_spacewith|Discrete_->()|Box_->invalid_arg("Gym.discrete: "^id^" has continuous actions"));envletdiscrete_with_renderer~seedid=letenv,frame=make~render:true~seedid~encode:string_of_intin(matchenv.Ndenv.action_spacewith|Discrete_->()|Box_->invalid_arg("Gym.discrete_with_renderer: "^id^" has continuous actions"));env,frameletcontinuous~seedid:(A.arr,A.arr)Ndenv.t=letencodeaction=letflat=A.reshapeaction[|A.numelaction|]inString.concat" "(List.init(A.numelaction)(funi->Python.hex(A.elt_to_float(A.getflat[|i|]))))inletenv,_=make~render:false~seedid~encodein(matchenv.action_spacewith|Box_->()|Discrete_->invalid_arg("Gym.continuous: "^id^" has discrete actions"));envendmoduleS=Make(Owl.Neural.S.Algodiff.A)moduleD=Make(Owl.Neural.D.Algodiff.A)