self learning test

Assets/ML-Agents/Timers/AI training_timers.json

@@ -1 +1 @@
-{"count":1,"self":809.2977152,"total":4632.3942763,"children":{"InitializeActuators":{"count":10,"self":0.0025077999999999997,"total":0.0025077999999999997,"children":null},"InitializeSensors":{"count":10,"self":0.0025223,"total":0.0025223,"children":null},"AgentSendState":{"count":1255617,"self":31.879206399999998,"total":70.695593899999992,"children":{"CollectObservations":{"count":2511240,"self":11.534297599999999,"total":11.5342975,"children":null},"WriteActionMask":{"count":2511240,"self":1.0299128,"total":1.0299128,"children":null},"RequestDecision":{"count":2511240,"self":4.8350608,"total":26.2521764,"children":{"AgentInfo.ToProto":{"count":2511240,"self":3.9296072,"total":21.417115499999998,"children":{"GenerateSensorData":{"count":2511240,"self":14.390806399999999,"total":17.4875084,"children":{"RayPerceptionSensor.Perceive":{"count":5022480,"self":3.0967018,"total":3.0967018,"children":null}}}}}}}}},"DecideAction":{"count":1255617,"self":3650.8291071999997,"total":3650.8292428,"children":null},"AgentAct":{"count":1255617,"self":101.3596032,"total":101.56567179999999,"children":{"AgentInfo.ToProto":{"count":13246,"self":0.042745899999999996,"total":0.20606549999999998,"children":{"GenerateSensorData":{"count":13246,"self":0.1278361,"total":0.16331959999999998,"children":{"RayPerceptionSensor.Perceive":{"count":26492,"self":0.0354835,"total":0.0354835,"children":null}}}}}}}},"gauges":{"BensonV6.1.CumulativeReward":{"count":13246,"max":26.7048168,"min":-0.06671302,"runningAverage":17.17633,"value":18.1915569,"weightedAverage":18.4008961}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1713626948","unity_version":"2022.3.11f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2022.3.11f1\\Editor\\Unity.exe -projectpath C:\\Users\\noahk\\Documents\\Unity projects\\Racesm -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-noahk -hubSessionId e08cc479-296a-466a-a392-68af5330fc09 -accessToken S1StEIprioU2rzDDo5vTCfsRe2zgz-zUchvR2e0iCUQ005f","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"AI training","end_time_seconds":"1713631580"}}
+{"count":1,"self":57.8489728,"total":273.2728647,"children":{"InitializeActuators":{"count":40,"self":0.0015049,"total":0.0015049,"children":null},"InitializeSensors":{"count":40,"self":0.0025047999999999997,"total":0.0025047999999999997,"children":null},"AgentSendState":{"count":39417,"self":3.278764,"total":7.4229638,"children":{"CollectObservations":{"count":315360,"self":1.0711507,"total":1.0711507,"children":null},"WriteActionMask":{"count":315360,"self":0.11854429999999999,"total":0.11854429999999999,"children":null},"RequestDecision":{"count":315360,"self":0.5394548,"total":2.9545049,"children":{"AgentInfo.ToProto":{"count":315360,"self":0.3807658,"total":2.4150500999999998,"children":{"GenerateSensorData":{"count":315360,"self":1.6535278,"total":2.0342843,"children":{"RayPerceptionSensor.Perceive":{"count":630720,"self":0.3807565,"total":0.3807565,"children":null}}}}}}}}},"DecideAction":{"count":39417,"self":201.36573439999998,"total":201.3657307,"children":null},"AgentAct":{"count":39417,"self":6.5899427999999993,"total":6.6301796999999993,"children":{"AgentInfo.ToProto":{"count":3862,"self":0.0095677,"total":0.040236999999999995,"children":{"GenerateSensorData":{"count":3862,"self":0.0241485,"total":0.0306693,"children":{"RayPerceptionSensor.Perceive":{"count":7724,"self":0.0065207999999999993,"total":0.0065207999999999993,"children":null}}}}}}}},"gauges":{"BensonV7.CumulativeReward":{"count":3862,"max":10.1914558,"min":-0.5617464,"runningAverage":-0.0396344438,"value":0.0600896031,"weightedAverage":0.125912115}},"metadata":{"timer_format_version":"0.1.0","start_time_seconds":"1713809080","unity_version":"2022.3.11f1","command_line_arguments":"C:\\Program Files\\Unity\\Hub\\Editor\\2022.3.11f1\\Editor\\Unity.exe -projectpath C:\\Users\\noahk\\Documents\\Unity projects\\Racesm -useHub -hubIPC -cloudEnvironment production -licensingIpc LicenseClient-noahk -hubSessionId 837b02b4-3460-4d4b-98bd-f0a6ffa15664 -accessToken DWB9fkNvzhznkjWNmlbZaGbkZ-bcd_svxKhJE2hkqZI005f","communication_protocol_version":"1.5.0","com.unity.ml-agents_version":"2.0.1","scene_name":"AI training","end_time_seconds":"1713809353"}}

Assets/Prefabs/Training Areas/TrainingV4.prefab.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: 742a3192b43bde644ad6d9a3cc58e51d
+PrefabImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scripts/AgentControllerV7.cs

@@ -0,0 +1,326 @@
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+using Unity.MLAgents; 
+using Unity.MLAgents.Sensors; 
+using Unity.MLAgents.Actuators;
+using UnityEngine.UIElements;
+using System.Linq;
+using Unity.Mathematics;
+using Unity.VisualScripting;
+using System.Reflection;
+using System;
+
+public class AgentControllerV7 : Agent
+{
+    public float motorTorque = 300;
+    public float brakeTorque = 500;
+    public float maxSpeed = 400;
+    public float steeringRange = 9;
+    public float steeringRangeAtMaxSpeed = 7;
+    public float autoBrake = 100;
+    WheelControl[] wheels;
+    Rigidbody rigidBody;
+    public List<GameObject> checkpoints;
+    Vector3 startPosition;
+    Quaternion startRotation;
+    int currentStep = 0;
+    float totalReward = 0;
+    float totalMentalPain = 0;
+    int stepsSinceCheckpoint = 0;
+    int checkpointsReached = 0;
+    public int maxStepsPerCheckpoint = 300;
+    public int distanceBetweenCheckpoints = 5;
+    public bool ignoreMentalPain = true;
+
+    // Start is called before the first frame update
+    void Start()
+    {
+        rigidBody = GetComponent<Rigidbody>();
+        // Find all child GameObjects that have the WheelControl script attached
+        wheels = GetComponentsInChildren<WheelControl>();
+
+        startPosition = transform.localPosition;
+        startRotation = transform.localRotation;
+        
+    }
+
+    public override void OnEpisodeBegin()
+    {
+        stepsSinceCheckpoint = 0;
+        checkpointsReached = 0;
+        totalReward = 0;
+        totalMentalPain = 0;
+
+        // reset wheels
+        foreach (var wheel in wheels)
+        {
+            wheel.WheelCollider.brakeTorque = 0;
+            wheel.WheelCollider.motorTorque = 0;
+            wheel.WheelCollider.steerAngle = 0;
+        }
+
+
+        // reset car
+        transform.localPosition = startPosition;
+        transform.localRotation = startRotation;
+        rigidBody.velocity = Vector3.zero;
+        rigidBody.angularVelocity = Vector3.zero;
+
+        // reset checkpoints
+        foreach (GameObject checkpoint in checkpoints)
+        {
+            checkpoint.GetComponent<Checkpoint>().isCollected = false;
+        }
+    }
+
+    public override void CollectObservations(VectorSensor sensor)
+    {
+        Transform currentCheckpoint = checkpoints[0].transform;
+        foreach (GameObject checkpoint in checkpoints)
+        {
+            bool isCollected = checkpoint.GetComponent<Checkpoint>().isCollected;
+
+            if (!isCollected)
+            {
+                currentCheckpoint = checkpoint.transform;
+                break;
+            }
+        }
+        
+        // distance to next checkpoint
+        sensor.AddObservation(distanceToCheckpoint(currentCheckpoint));
+
+        // relative angle to checkpoint
+        sensor.AddObservation(angleToCheckpoint(currentCheckpoint));
+
+        // relative vector pointing to checkpoint
+        Vector3 position = transform.localPosition;
+        Vector3 checkpointPosition = currentCheckpoint.localPosition;
+
+        Vector3 toCheckpoint = new Vector3(
+            checkpointPosition.x - position.x,
+            0,
+            checkpointPosition.z - position.z
+        );
+
+        float carAngle = transform.localEulerAngles.y;
+
+        toCheckpoint = Quaternion.Euler(0, -carAngle, 0) * toCheckpoint.normalized;
+
+        sensor.AddObservation(toCheckpoint.x);
+        sensor.AddObservation(toCheckpoint.z);
+
+    
+        // relative Velocity
+        Vector3 velocity = new Vector3(
+            rigidBody.velocity.x,
+            0,
+            rigidBody.velocity.z
+        );
+
+        Vector3 relativeVelocity = Quaternion.Euler(0, -carAngle, 0) * velocity;
+
+        sensor.AddObservation(relativeVelocity.x);
+        sensor.AddObservation(relativeVelocity.z);
+    }
+
+    public override void OnActionReceived(ActionBuffers actions)
+    {
+        // Actions size = 2 [vertical speed, horizontal speed] = [-1..1, -1..1] // discrete = [{0, 1, 2}, {0, 1, 2}] = [{-1, 0, 1}...]
+        float vInput = 0;
+        float hInput = 0;
+
+        if (actions.DiscreteActions[0] == 0)
+            vInput = -1f;
+        if (actions.DiscreteActions[0] == 1)
+            vInput = 1f;
+
+        if (actions.DiscreteActions[1] == 0)
+            hInput = -1f;
+        if (actions.DiscreteActions[1] == 1)
+            hInput = 1f;
+
+        // give benson mental pain for existing (punishment for maximizing first checkpoint by standing still)
+        // AddReward(-0.002f);
+        AddReward(-0.0018f); // less pain because of V4
+        totalMentalPain -= 0.0018f;
+
+        if (ignoreMentalPain)
+            totalReward -= 0.0018f;
+
+        float forwardSpeed = Vector3.Dot(transform.forward, rigidBody.velocity);
+        // Calculate how close the car is to top speed
+        // as a number from zero to one
+        float speedFactor = Mathf.InverseLerp(0, maxSpeed / 4, forwardSpeed);
+
+        // Use that to calculate how much torque is available 
+        // (zero torque at top speed)
+        float currentMotorTorque = Mathf.Lerp(motorTorque, 0, speedFactor);
+
+        // …and to calculate how much to steer 
+        // (the car steers more gently at top speed)
+
+        float currentSteerRange = Mathf.Lerp(steeringRange, steeringRangeAtMaxSpeed, speedFactor);
+
+        // Check whether the user input is in the same direction 
+        // as the car's velocity
+        bool isAccelerating = Mathf.Sign(vInput) == Mathf.Sign(forwardSpeed);
+
+        bool isStopping = vInput == 0; // range
+
+        bool isBraking = (vInput < 0 && forwardSpeed > 0) || (vInput > 0 && forwardSpeed < 0);
+
+        if (vInput > 0 && forwardSpeed < 0)
+        {
+            isAccelerating = false;
+        }
+
+        foreach (var wheel in wheels)
+        {
+            // Apply steering to Wheel colliders that have "Steerable" enabled
+            if (wheel.steerable)
+            {
+                wheel.WheelCollider.steerAngle = hInput * currentSteerRange;
+            }
+
+            if (isBraking)
+            {
+                wheel.WheelCollider.brakeTorque = Mathf.Abs(vInput) * brakeTorque;
+                //wheel.WheelCollider.motorTorque = 0;
+            }
+            
+            if (isAccelerating)
+            {
+                // Apply torque to Wheel colliders that have "Motorized" enabled
+                if (wheel.motorized)
+                {
+                    wheel.WheelCollider.motorTorque = vInput * currentMotorTorque;
+                }
+                wheel.WheelCollider.brakeTorque = 0;
+            }
+
+            if (isStopping)
+            {
+                // If the user is trying to go in the opposite direction
+                // apply brakes to all wheels
+                wheel.WheelCollider.brakeTorque = Mathf.Abs(vInput) * brakeTorque + autoBrake;
+
+                if (forwardSpeed < 0)
+                {
+                    wheel.WheelCollider.brakeTorque = (Mathf.Abs(vInput) * brakeTorque + autoBrake) * 5;
+                }
+
+            }
+        }
+
+        // rewards
+        Transform currentCheckpoint = checkpoints[0].transform;
+        foreach (GameObject checkpoint in checkpoints)
+        {
+            bool isCollected = checkpoint.GetComponent<Checkpoint>().isCollected;
+
+            if (!isCollected)
+            {
+                currentCheckpoint = checkpoint.transform;
+                break;
+            }
+        }
+
+        float checkpintDistance = distanceToCheckpoint(currentCheckpoint);
+        float reward = (1 - Mathf.InverseLerp(0, distanceBetweenCheckpoints, checkpintDistance)) / 500;
+
+        totalReward += reward;
+        AddReward(reward);
+
+        float checkpointAngle = angleToCheckpoint(currentCheckpoint);
+
+        if (checkpointAngle > 0)
+            reward = (1 - Mathf.InverseLerp(0, 60, checkpointAngle)) / 2000;
+        else
+            reward = Mathf.InverseLerp(-60, 0, checkpointAngle) / 2000;
+
+        AddReward(reward);
+
+
+
+        
+
+        if (checkpintDistance < 0.1f)
+        {
+            currentCheckpoint.GetComponent<Checkpoint>().isCollected = true;
+            stepsSinceCheckpoint = 0;
+            checkpointsReached += 1;
+
+            // If last checkpoint
+            if (currentCheckpoint == checkpoints[checkpoints.Count - 1].transform)
+            {
+                AddReward(10f);
+                EndEpisode();           
+            }
+
+            //TODO fix variable names
+
+            AddReward(1f);
+            AddReward(-totalReward);
+
+            totalReward = 0;
+        }
+
+        currentStep += 1;
+        stepsSinceCheckpoint += 1;
+
+        if (stepsSinceCheckpoint >= maxStepsPerCheckpoint)
+        {
+            stepsSinceCheckpoint = 0;
+            EndEpisode();
+        }
+
+        // print(GetCumulativeReward());
+    }
+
+    public override void Heuristic(in ActionBuffers actionsOut)
+    {
+        var discreteActionsOut = actionsOut.DiscreteActions;
+
+        discreteActionsOut[0] = 2;
+        discreteActionsOut[1] = 2;
+
+        if (Input.GetAxis("Vertical") < -0.5)
+            discreteActionsOut[0] = 0;
+        if (Input.GetAxis("Vertical") > 0.5)
+            discreteActionsOut[0] = 1;
+
+        if (Input.GetAxis("Horizontal") < -0.5)
+            discreteActionsOut[1] = 0;
+        if (Input.GetAxis("Horizontal") > 0.5)
+            discreteActionsOut[1] = 1;
+    }
+
+    // finds distance from agent to closest point on the checkpoint line
+    float distanceToCheckpoint(Transform checkpoint)
+    {
+        var closestPoint = checkpoint.GetComponent<Collider>().ClosestPointOnBounds(transform.position);
+        var distanceToCheckpoint = Vector3.Distance(transform.position, closestPoint);
+        return distanceToCheckpoint;
+    }
+
+    float angleToCheckpoint(Transform checkpoint)
+    {
+        Vector3 checkpointDirection = checkpoint.localPosition - transform.localPosition;
+
+        float angle = Vector3.SignedAngle(transform.forward, checkpointDirection, Vector3.up);
+        return angle;
+    }
+
+    private void OnCollisionEnter(Collision other) {
+        // if (other.gameObject.tag == "NPC")
+        // {
+        //     AddReward(0.1f);
+        // }
+        if (other.gameObject.tag == "Player")
+        {
+            AddReward(-0.1f);
+        }
+    }
+}

Assets/Scripts/AgentControllerV7.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 2eeab89a9b8a5f04291ce55ceb722904
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: