diff --git a/MLAgents/config/BusTrack2.yaml b/MLAgents/config/BusTrack2.yaml new file mode 100644 index 0000000..49b7e68 --- /dev/null +++ b/MLAgents/config/BusTrack2.yaml @@ -0,0 +1,72 @@ +behaviors: + BusTrack2: + trainer_type: ppo + + hyperparameters: + # Hyperparameters common to PPO and SAC + batch_size: 50 + buffer_size: 10240 + learning_rate: 3.0e-4 + learning_rate_schedule: linear + + # PPO-specific hyperparameters + # Replaces the "PPO-specific hyperparameters" section above + beta: 5.0e-2 + epsilon: 0.2 + lambd: 0.95 + num_epoch: 3 + + # Configuration of the neural network (common to PPO/SAC) + network_settings: + vis_encoder_type: simple + normalize: false + hidden_units: 128 + num_layers: 2 + + # Trainer configurations common to all trainers + max_steps: 5.0e6 + time_horizon: 64 + summary_freq: 10000 + keep_checkpoints: 5 + checkpoint_interval : 500000 + threaded: false + init_path: null + + # # behavior cloning + # behavioral_cloning: + # demo_path: 'c:\Users\noahk\Documents\Unity projects\Racesm\Assets\Demonstrations\BensonV3M.demo' + # strength: 0.5 + # # steps: 150000 + # # batch_size: 512 + # # num_epoch: 3 + # # samples_per_update: 0 + + reward_signals: + # environment reward (default) + extrinsic: + strength: 1.0 + gamma: 0.99 + + # self_play: + # window: 3 + # play_against_latest_model_ratio: 0.5 + # save_steps: 100000 + # swap_steps: 200000 + # team_change: 1000000 + + # # curiosity module + # curiosity: + # strength: 0.02 + # gamma: 0.99 + # encoding_size: 256 + # learning_rate: 3.0e-4 + + # # GAIL + # gail: + # strength: 0.5 + # # gamma: 0.99 + # # encoding_size: 128 + # demo_path: 'c:\Users\noahk\Documents\Unity projects\Racesm\Assets\Demonstrations\BensonV3M.demo' + # # learning_rate: 3.0e-4 + # # use_actions: false + # # use_vail: false \ No newline at end of file diff --git a/MLAgents/results/BusTrack2/BusTrack2.onnx b/MLAgents/results/BusTrack2/BusTrack2.onnx new file mode 100644 index 0000000..7199d8c Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2.onnx differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.onnx b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.onnx new file mode 100644 index 0000000..0f875ed Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.onnx differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.pt b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.pt new file mode 100644 index 0000000..739d19b Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-12000.pt differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.onnx b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.onnx new file mode 100644 index 0000000..7199d8c Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.onnx differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.pt b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.pt new file mode 100644 index 0000000..0f9eede Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/BusTrack2-52512.pt differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/checkpoint.pt b/MLAgents/results/BusTrack2/BusTrack2/checkpoint.pt new file mode 100644 index 0000000..69b88ac Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/checkpoint.pt differ diff --git a/MLAgents/results/BusTrack2/BusTrack2/events.out.tfevents.1713956588.Noahs-Episke-PC.26720.0 b/MLAgents/results/BusTrack2/BusTrack2/events.out.tfevents.1713956588.Noahs-Episke-PC.26720.0 new file mode 100644 index 0000000..97624e5 Binary files /dev/null and b/MLAgents/results/BusTrack2/BusTrack2/events.out.tfevents.1713956588.Noahs-Episke-PC.26720.0 differ diff --git a/MLAgents/results/BusTrack2/configuration.yaml b/MLAgents/results/BusTrack2/configuration.yaml new file mode 100644 index 0000000..39b50fc --- /dev/null +++ b/MLAgents/results/BusTrack2/configuration.yaml @@ -0,0 +1,76 @@ +default_settings: null +behaviors: + BusTrack2: + trainer_type: ppo + hyperparameters: + batch_size: 50 + buffer_size: 10240 + learning_rate: 0.0003 + beta: 0.05 + epsilon: 0.2 + lambd: 0.95 + num_epoch: 3 + shared_critic: false + learning_rate_schedule: linear + beta_schedule: linear + epsilon_schedule: linear + network_settings: + normalize: false + hidden_units: 128 + num_layers: 2 + vis_encode_type: simple + memory: null + goal_conditioning_type: hyper + deterministic: false + reward_signals: + extrinsic: + gamma: 0.99 + strength: 1.0 + network_settings: + normalize: false + hidden_units: 128 + num_layers: 2 + vis_encode_type: simple + memory: null + goal_conditioning_type: hyper + deterministic: false + init_path: null + keep_checkpoints: 5 + checkpoint_interval: 500000 + max_steps: 5000000 + time_horizon: 64 + summary_freq: 10000 + threaded: false + self_play: null + behavioral_cloning: null +env_settings: + env_path: null + env_args: null + base_port: 5005 + num_envs: 1 + num_areas: 1 + seed: -1 + max_lifetime_restarts: 10 + restarts_rate_limit_n: 1 + restarts_rate_limit_period_s: 60 +engine_settings: + width: 84 + height: 84 + quality_level: 5 + time_scale: 20 + target_frame_rate: -1 + capture_frame_rate: 60 + no_graphics: false +environment_parameters: null +checkpoint_settings: + run_id: BusTrack2 + initialize_from: null + load_model: false + resume: false + force: true + train_model: false + inference: false + results_dir: results +torch_settings: + device: null +debug: false diff --git a/MLAgents/results/BusTrack2/run_logs/timers.json b/MLAgents/results/BusTrack2/run_logs/timers.json new file mode 100644 index 0000000..2403140 --- /dev/null +++ b/MLAgents/results/BusTrack2/run_logs/timers.json @@ -0,0 +1,313 @@ +{ + "name": "root", + "gauges": { + "BusTrack2.Policy.Entropy.mean": { + "value": 2.1863579750061035, + "min": 2.1749253273010254, + "max": 2.1947851181030273, + "count": 5 + }, + "BusTrack2.Policy.Entropy.sum": { + "value": 21968.525390625, + "min": 21644.857421875, + "max": 22053.201171875, + "count": 5 + }, + "BusTrack2.Step.mean": { + "value": 49984.0, + "min": 9984.0, + "max": 49984.0, + "count": 5 + }, + "BusTrack2.Step.sum": { + "value": 49984.0, + "min": 9984.0, + "max": 49984.0, + "count": 5 + }, + "BusTrack2.Policy.ExtrinsicValueEstimate.mean": { + "value": 0.0749274343252182, + "min": 0.0749274343252182, + "max": 0.2529940903186798, + "count": 5 + }, + "BusTrack2.Policy.ExtrinsicValueEstimate.sum": { + "value": 11.98838996887207, + "min": 11.98838996887207, + "max": 40.22605895996094, + "count": 5 + }, + "BusTrack2.Environment.EpisodeLength.mean": { + "value": 499.0, + "min": 499.0, + "max": 499.0, + "count": 5 + }, + "BusTrack2.Environment.EpisodeLength.sum": { + "value": 7984.0, + "min": 7984.0, + "max": 11976.0, + "count": 5 + }, + "BusTrack2.Environment.CumulativeReward.mean": { + "value": -0.30098484913126894, + "min": -0.7619615873187761, + "max": -0.19303640429825464, + "count": 5 + }, + "BusTrack2.Environment.CumulativeReward.sum": { + "value": -5.116742435231572, + "min": -17.52511650833185, + "max": -3.088582468772074, + "count": 5 + }, + "BusTrack2.Policy.ExtrinsicReward.mean": { + "value": -0.30098484913126894, + "min": -0.7619615873187761, + "max": -0.19303640429825464, + "count": 5 + }, + "BusTrack2.Policy.ExtrinsicReward.sum": { + "value": -5.116742435231572, + "min": -17.52511650833185, + "max": -3.088582468772074, + "count": 5 + }, + "BusTrack2.IsTraining.mean": { + "value": 1.0, + "min": 1.0, + "max": 1.0, + "count": 5 + }, + "BusTrack2.IsTraining.sum": { + "value": 1.0, + "min": 1.0, + "max": 1.0, + "count": 5 + }, + "BusTrack2.Losses.PolicyLoss.mean": { + "value": 0.11586630168607447, + "min": 0.11050240805861049, + "max": 0.11743067037271877, + "count": 4 + }, + "BusTrack2.Losses.PolicyLoss.sum": { + "value": 0.11586630168607447, + "min": 0.11050240805861049, + "max": 0.11743067037271877, + "count": 4 + }, + "BusTrack2.Losses.ValueLoss.mean": { + "value": 0.0026205912706508275, + "min": 0.0026205912706508275, + "max": 0.01049612028478976, + "count": 4 + }, + "BusTrack2.Losses.ValueLoss.sum": { + "value": 0.0026205912706508275, + "min": 0.0026205912706508275, + "max": 0.01049612028478976, + "count": 4 + }, + "BusTrack2.Policy.LearningRate.mean": { + "value": 0.00029747712084095994, + "min": 0.00029747712084095994, + "max": 0.0002993664002111999, + "count": 4 + }, + "BusTrack2.Policy.LearningRate.sum": { + "value": 0.00029747712084095994, + "min": 0.00029747712084095994, + "max": 0.0002993664002111999, + "count": 4 + }, + "BusTrack2.Policy.Epsilon.mean": { + "value": 0.19915904000000004, + "min": 0.19915904000000004, + "max": 0.19978880000000004, + "count": 4 + }, + "BusTrack2.Policy.Epsilon.sum": { + "value": 0.19915904000000004, + "min": 0.19915904000000004, + "max": 0.19978880000000004, + "count": 4 + }, + "BusTrack2.Policy.Beta.mean": { + "value": 0.04957960409599999, + "min": 0.04957960409599999, + "max": 0.049894421120000004, + "count": 4 + }, + "BusTrack2.Policy.Beta.sum": { + "value": 0.04957960409599999, + "min": 0.04957960409599999, + "max": 0.049894421120000004, + "count": 4 + } + }, + "metadata": { + "timer_format_version": "0.1.0", + "start_time_seconds": "1713956580", + "python_version": "3.9.13 (tags/v3.9.13:6de2ca5, May 17 2022, 16:36:42) [MSC v.1929 64 bit (AMD64)]", + "command_line_arguments": "C:\\Users\\noahk\\Documents\\Unity projects\\Racesm\\.venv\\Scripts\\mlagents-learn config/BusTrack2.yaml --run-id=BusTrack2 --force", + "mlagents_version": "0.30.0", + "mlagents_envs_version": "0.30.0", + "communication_protocol_version": "1.5.0", + "pytorch_version": "2.2.2+cu118", + "numpy_version": "1.21.2", + "end_time_seconds": "1713956677" + }, + "total": 96.8911192, + "count": 1, + "self": 0.004357900000016457, + "children": { + "run_training.setup": { + "total": 0.08156290000000022, + "count": 1, + "self": 0.08156290000000022 + }, + "TrainerController.start_learning": { + "total": 96.8051984, + "count": 1, + "self": 0.10506819999997674, + "children": { + "TrainerController._reset_env": { + "total": 9.1141883, + "count": 1, + "self": 9.1141883 + }, + "TrainerController.advance": { + "total": 87.46949260000002, + "count": 6585, + "self": 0.09444109999992634, + "children": { + "env_step": { + "total": 49.186684400000495, + "count": 6585, + "self": 21.425331300000444, + "children": { + "SubprocessEnvManager._take_step": { + "total": 27.69659450000002, + "count": 6585, + "self": 0.28653870000042403, + "children": { + "TorchPolicy.evaluate": { + "total": 27.410055799999597, + "count": 6585, + "self": 27.410055799999597 + } + } + }, + "workers": { + "total": 0.06475860000003131, + "count": 6584, + "self": 0.0, + "children": { + "worker_root": { + "total": 88.33605969999992, + "count": 6584, + "is_parallel": true, + "self": 72.29476539999987, + "children": { + "steps_from_proto": { + "total": 0.0005465000000004494, + "count": 1, + "is_parallel": true, + "self": 0.00022170000000087953, + "children": { + "_process_rank_one_or_two_observation": { + "total": 0.0003247999999995699, + "count": 6, + "is_parallel": true, + "self": 0.0003247999999995699 + } + } + }, + "UnityEnvironment.step": { + "total": 16.04074780000004, + "count": 6584, + "is_parallel": true, + "self": 0.6872718000007936, + "children": { + "UnityEnvironment._generate_step_input": { + "total": 0.7578845999999206, + "count": 6584, + "is_parallel": true, + "self": 0.7578845999999206 + }, + "communicator.exchange": { + "total": 12.44123759999931, + "count": 6584, + "is_parallel": true, + "self": 12.44123759999931 + }, + "steps_from_proto": { + "total": 2.154353800000017, + "count": 6584, + "is_parallel": true, + "self": 0.8423023000004495, + "children": { + "_process_rank_one_or_two_observation": { + "total": 1.3120514999995674, + "count": 39504, + "is_parallel": true, + "self": 1.3120514999995674 + } + } + } + } + } + } + } + } + } + } + }, + "trainer_advance": { + "total": 38.1883670999996, + "count": 6584, + "self": 0.13078589999910406, + "children": { + "process_trajectory": { + "total": 4.465832300000498, + "count": 6584, + "self": 4.465832300000498 + }, + "_update_policy": { + "total": 33.5917489, + "count": 5, + "self": 6.835913400000042, + "children": { + "TorchPPOOptimizer.update": { + "total": 26.755835499999957, + "count": 3147, + "self": 26.755835499999957 + } + } + } + } + } + } + }, + "trainer_threads": { + "total": 1.1999999998124622e-06, + "count": 1, + "self": 1.1999999998124622e-06 + }, + "TrainerController._save_models": { + "total": 0.1164480999999995, + "count": 1, + "self": 0.0068753000000043585, + "children": { + "RLTrainer._checkpoint": { + "total": 0.10957279999999514, + "count": 1, + "self": 0.10957279999999514 + } + } + } + } + } + } +} \ No newline at end of file diff --git a/MLAgents/results/BusTrack2/run_logs/training_status.json b/MLAgents/results/BusTrack2/run_logs/training_status.json new file mode 100644 index 0000000..74d3405 --- /dev/null +++ b/MLAgents/results/BusTrack2/run_logs/training_status.json @@ -0,0 +1,29 @@ +{ + "BusTrack2": { + "checkpoints": [ + { + "steps": 52512, + "file_path": "results\\BusTrack2\\BusTrack2\\BusTrack2-52512.onnx", + "reward": null, + "creation_time": 1713956677.5616412, + "auxillary_file_paths": [ + "results\\BusTrack2\\BusTrack2\\BusTrack2-52512.pt" + ] + } + ], + "final_checkpoint": { + "steps": 52512, + "file_path": "results\\BusTrack2\\BusTrack2.onnx", + "reward": null, + "creation_time": 1713956677.5616412, + "auxillary_file_paths": [ + "results\\BusTrack2\\BusTrack2\\BusTrack2-52512.pt" + ] + } + }, + "metadata": { + "stats_format_version": "0.3.0", + "mlagents_version": "0.30.0", + "torch_version": "2.2.2+cu118" + } +} \ No newline at end of file diff --git a/MLAgents/train.bat b/MLAgents/train.bat index 385a98d..65f4e59 100644 --- a/MLAgents/train.bat +++ b/MLAgents/train.bat @@ -1,2 +1,2 @@ -@REM mlagents-learn config/BensonV7.yaml --run-id=BensonV7 --resume -mlagents-learn config/BensonV7.yaml --run-id=BensonV7 --resume --env="C:\Users\noahk\Desktop\Benson slepe\Racesm_L.exe" --no-graphics --num-envs=1 \ No newline at end of file +mlagents-learn config/BusTrack2.yaml --run-id=BusTrack2 --force +@REM mlagents-learn config/BensonV7.yaml --run-id=BensonV7 --resume --env="C:\Users\noahk\Desktop\Benson slepe\Racesm_L.exe" --no-graphics --num-envs=1 \ No newline at end of file