Machine Learning in Modeling and Control

NI-AML: Advanced Machine Learning, 30. 3. 2023

Zdeněk Buk, Miroslav Čepek, Rodrigo Da Silva Alves, Vojtěch Rybář and Petr Šimánek

Introduction

Tasks

Experiment with various models

Experiment with transfer (NN model → Gym)

Implement the sensor and trajectory noise

measure the tranfer quality based on the noise intensity  [HOMEWORK]

measure the controller training speed (convergency) based on the noise intensity [HOMEWORK]

Controller extrapolation measurements (Training and Evaluation with different maximum number of steps) [HOMEWORK]

Simulator Wrapper

ClearAll[randomState]
randomState[]:=RandomReal[{-0.05,0.05},4]

Clear[endQ]
endQ[state_List]:=Or[
		(* Pole Angle *) Abs[state[[3]]]>0.2,
		(* Cart Position *)Abs[state[[1]]]>2.4
	] (* Ignoring the episode length truncation *)
ClearAll[episodeEndCheck]
episodeEndCheck[environment_Association]:=Append[environment,<|"Ended"->#,"ObservedState"->If[#,randomState[],environment["ObservedState"]]|>&@endQ[environment["ObservedState"]]]
SetAttributes[episodeEndCheck,HoldFirst]

ClearAll[simulatedEnvironment]
simulatedEnvironment[model_NetChain|model_NetGraph]:=<|"Model"->model,"Data"-><|"ObservedState"->randomState[]|>|>

ClearAll[gymEnvironment]
gymEnvironment[env_DeviceObject]:=<|"Model"->env,"Data"->DeviceExecute[env,"Reset"]|>

ClearAll[reset,step]
SetAttributes[{reset,step},HoldFirst]

(* Gym Wrapper*)
reset[environment_]:=AppendTo[environment,<|"Data"->DeviceExecute[Evaluate[environment]["Model"],"Reset"]|>]/;MatchQ[Evaluate[environment]["Model"],_DeviceObject]

step[environment_,action_]:=AppendTo[environment,<|"Data"->DeviceExecute[Evaluate[environment]["Model"],"Step",action]|>]/;MatchQ[Evaluate[environment]["Model"],_DeviceObject]

(* NN-model Wrapper*)
reset[environment_]:=AppendTo[environment,<|"Data"-><|"ObservedState"->randomState[]|>|>]/;MatchQ[Evaluate[environment]["Model"],_NetChain|_NetGraph]

step[environment_,action_]:=AppendTo[environment,<|
"Data"->episodeEndCheck@<|
"ObservedState"->environment["Model"][<|"State"->Evaluate[environment]["Data","ObservedState"],"Action"->action|>],
"Reward"->1
|>
|>]/;MatchQ[Evaluate[environment]["Model"],_NetChain|_NetGraph]

ClearAll[render]
render[data_List,action_:None]:=Graphics[
		{
			Black,Thickness[Medium],
			Line[{{-4.8,0},{4.8,0}}],
			Green,Opacity[0.75],EdgeForm[Black],
			Rectangle[{#[[1]]-0.4,-0.25},{#[[1]]+0.4,0.25}],
			Opacity[1],Blue,Thickness[0.015],
			Line[{{#[[1]],0},{#[[1]]+2Cos[π/2-#[[3]]],2Sin[π/2-#[[3]]]}}],
			PointSize[0.025],Red,Point[{#[[1]],0}],
			Text[action/.{Left->"←",Right->"→",None->""},{#[[1]],-0.4}]
		},
		PlotRange->{{-5,5},{-0.5,2.5}}
	]&@data
render[environment_Association,action_:None]:=render[environment["Data","ObservedState"],action]

Model

Training Data

env=DeviceOpen["SimulatedCartPole"];
AbsoluteTiming[
data=Table[
Function[{action},
{
DeviceRead[env]["ObservedState"],
action,
DeviceExecute[env,"Step",action]
}][DeviceExecute[env,"RandomAction"]],
{10000}
];
]
DeviceClose[env];

The measured data contains the discontinuities at the end of episodes (“Ended” -> True), the new state is generated to start the new episode. These states/transitions need to be removed.

Length[data]

data[[1;;5]]

observations=Select[data,#[[3,"Ended"]]==False&];
Length[observations]

observations[[1;;2]]

trainingData=<|"State"->#[[1]],"Action"->#[[2]],"Output"->#[[3,"ObservedState"]]|>&/@observations;
Length[trainingData]
RandomSample[trainingData,3]

Model Definition & Training

model=NetGraph[<|
"net"->NetChain[{CatenateLayer[],LinearLayer[16],Tanh,4}]
|>,
{
NetPort["State"]->"net",
NetPort["Action"]->"net"
},
"State"->{4},
"Action"->NetEncoder[{"Class",{Left,Right},"IndicatorVector"}]
]

res=NetTrain[model,trainingData,All,ValidationSet->Scaled[0.1],MaxTrainingRounds->1000,TargetDevice->"CPU",BatchSize->512]

net=res["TrainedNet"]

Environments

envModel=simulatedEnvironment[net]

envGym=gymEnvironment[DeviceOpen["SimulatedCartPole"]]

Controller

Parameters:

Number of neurons

Maximum length of episode (simulation steps)

Population size

Number of iterations (EA)

Differential Evolution parameters - F, CR

Neuroevolution

ClearAll[evaluateCtrl];
evaluateCtrl[ctrl_,ctrlState_,inputs_]:=Module[{y},
	y=Tanh/@(ctrl.Join[ctrlState,{1},inputs]);
	{If[y[[1]]<0,Left,Right],y} (* {action, newState} *)
];

Parameters & Initialization

numberOfNeurons=4;
numberOfInputs=4; (* Observation Space *)

initCtrlState=ConstantArray[0,numberOfNeurons];

randomController[]:=RandomReal[{-0.05,0.05},{numberOfNeurons,numberOfNeurons+numberOfInputs+1}];

Simulation Step

ClearAll[simulationStep]
SetAttributes[simulationStep,HoldFirst]
simulationStep[env_,ctrl_,ctrlState_]:=Module[
{action,newCtrlState,observation,newObservation},
observation=env[["Data","ObservedState"]];
{action,newCtrlState}=evaluateCtrl[ctrl,ctrlState,observation];
newObservation=step[env,action];
Sow[{observation,action}];
{newCtrlState,(newObservation["Data","Ended"]/.{Missing[___]->False})}
]

(* For the simplicity we use a global variable "environment" *)
environment=envModel;

ClearAll[fitness];
fitness[ctrl_?(MatrixQ[#,NumericQ]&),maxSteps_:300]:=Module[{run},
reset[environment];
run=Rest@NestWhileList[simulationStep[environment,ctrl,#[[1]]]&,{initCtrlState,False},!#[[2]]&,1,maxSteps];
Length[run]
]

Differential Evolution

𝕨=Table[ToExpression@StringTemplate["w$`1`$`2`"][m,n],{m,numberOfNeurons},{n,numberOfNeurons+numberOfInputs+1}];

Length[Flatten[𝕨]]

(*environment=envGym;*)
environment=envModel;
fitStep=0;
evoLog={};
(res=NMaximize[
fitness[𝕨,100],
Flatten[𝕨],
Method->{
"DifferentialEvolution",
"ScalingFactor"->0.6,
"CrossProbability"->0.5,
"PostProcess"->False,
"SearchPoints"->20
(*"RandomSeed"->1*)
},
MaxIterations->50,
StepMonitor:>(fitStep++)
(*EvaluationMonitor:>(AppendTo[evoLog,{fitness[𝕨],𝕨}])*)
])//AbsoluteTiming
fitStep

Evaluation

resultCtrl=𝕨/.res[[2]];
MatrixPlot[resultCtrl,PlotTheme->"Detailed"]

environment=envModel;
log=Reap[fitness[resultCtrl,800]][[2,1]];
Length[log]

ListAnimate[render[#[[1]],#[[2]]]&/@log]

Transfer

environment=envGym;
log=Reap[fitness[resultCtrl,800]][[2,1]];
Length[log]

ListAnimate[render[#[[1]],#[[2]]]&/@log]

environment=envGym;
gymFit=Table[fitness[resultCtrl,1000],{10}];
environment=envModel;
modelFit=Table[fitness[resultCtrl,1000],{10}];

BoxWhiskerChart[{gymFit,modelFit},ChartLabels->{"Gym","Model"},FrameLabel->"Fitness"]

N[Mean[gymFit]]

N[Mean[modelFit]]