From 47ae5e66af7d59803e9fc1831b6e1aed2070deb7 Mon Sep 17 00:00:00 2001 From: Kiremai Date: Fri, 5 Jul 2024 17:40:17 +0200 Subject: [PATCH] Moved world redirection visualization data to JsonLogging --- .../Corridor/Scenes/RedirectionCorridor.unity | 2 + .../Scripts/Logging/JsonLogging.cs | 243 ++++++++++++++++-- .../Utils/Editor/Logging/JsonLoggingEditor.cs | 44 ++++ .../Editor/Logging/JsonLoggingEditor.cs.meta | 11 + .../Scripts/Visualisation/Socket.cs | 37 ++- python/RedirectionPlotter.py | 2 +- 6 files changed, 311 insertions(+), 28 deletions(-) create mode 100644 Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs create mode 100644 Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs.meta diff --git a/Visuo-haptic Toolkit/Assets/Demo Scenes/Corridor/Scenes/RedirectionCorridor.unity b/Visuo-haptic Toolkit/Assets/Demo Scenes/Corridor/Scenes/RedirectionCorridor.unity index d39e93c5..17b82998 100644 --- a/Visuo-haptic Toolkit/Assets/Demo Scenes/Corridor/Scenes/RedirectionCorridor.unity +++ b/Visuo-haptic Toolkit/Assets/Demo Scenes/Corridor/Scenes/RedirectionCorridor.unity @@ -2357,6 +2357,8 @@ MonoBehaviour: m_EditorClassIdentifier: logDirectoryPath: LoggedData\ optionalFilenamePrefix: + filename: + pythonPath: --- !u!114 &844723486 MonoBehaviour: m_ObjectHideFlags: 0 diff --git a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Logging/JsonLogging.cs b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Logging/JsonLogging.cs index 9c2e3da7..74b714fc 100644 --- a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Logging/JsonLogging.cs +++ b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Logging/JsonLogging.cs @@ -8,6 +8,12 @@ using VHToolkit.Redirection.BodyRedirection; using VHToolkit.Redirection.WorldRedirection; using Newtonsoft.Json; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using System.Diagnostics; +using System.Linq; +using UnityEditor; namespace VHToolkit.Logging { @@ -36,9 +42,31 @@ public record PhysicalLimbData { public PhysicalLimbData(Limb l) => limb = l; } + /// + /// Record type WorldRedirectionData is a wrapper around WRData. + /// It allows for serializing the redirection values : rotation over time, rotational and curvature + /// given by the WorldRedirection script + /// + public record WorldRedirectionData + { + public readonly float RotationOverTime; + public readonly float Rotational; + public readonly float Curvature; + public readonly float Time; + + public WorldRedirectionData(float overTime, float rotational, float curvature, float time) + { + this.RotationOverTime = overTime; + this.Rotational = rotational; + this.Curvature = curvature; + this.Time = time; + } + } + public record JsonRedirectionData { public readonly DateTime TimeStamp = DateTime.Now; private readonly Interaction script; + private readonly DateTime startTime; /// /// Property Strategy is a string corresponding to the name of the current World Redirection @@ -62,6 +90,12 @@ public record JsonRedirectionData { public bool Redirecting => script.redirect; + public WorldRedirectionData RedirectionData => + new WorldRedirectionData((script.redirect && script.scene.enableHybridOverTime) ? Razzaque2001OverTimeRotation.GetRedirection(script.scene) : 0f, + (script.redirect && script.scene.enableHybridRotational) ? Razzaque2001Rotational.GetRedirection(script.scene) : 0f, + (script.redirect && script.scene.enableHybridCurvature) ? Razzaque2001Curvature.GetRedirection(script.scene) : 0f, + (float)(TimeStamp - startTime).TotalSeconds); + public List Limbs => script.scene.limbs.ConvertAll(l => new PhysicalLimbData(l)); public TransformData PhysicalHead => script.scene.physicalHead ? new(script.scene.physicalHead) : null; public List Targets => script.scene.targets.ConvertAll(t => new TransformData(t)); @@ -69,36 +103,74 @@ public record JsonRedirectionData { public TransformData VirtualTarget => script.scene.virtualTarget ? new(script.scene.virtualTarget) : null; public string StrategyDirection => script.scene.forwardTarget != null ? Convert.ToString(script.scene.forwardTarget.ToString()) : null; - public JsonRedirectionData(Interaction script) => this.script = script; + public JsonRedirectionData(Interaction script, DateTime startTime) + { + this.script = script; + this.startTime = startTime; + } } /// /// Logs structured data in the JSON Lines format. /// - public class JsonLogging : Logger { + public class JsonLogging : Logger + { + private Scene scene; + private DateTime startTime; + + [SerializeField] private string filename; + + [SerializeField] private string pythonPath; private void Start() { CreateNewFile(logDirectoryPath, optionalFilenamePrefix ?? ""); script = GetComponent(); - } + scene = script.scene; - private void Update() { - records.Enqueue(new JsonRedirectionData(script)); - WriteRecords(records); - } - /// - /// Create a new log file. - /// - /// The path to the directory where the file should be placed. - /// An optional prefix string to appear in the filename before its timestamp. - public void CreateNewFile(string logDirectoryPath, string optionalFilenamePrefix = "") { + if (script is WorldRedirection) + CreateTcpClient(filename, pythonPath); + + startTime = DateTime.Now; + } + + private void Update() + { + records.Enqueue(new JsonRedirectionData(script, startTime)); + WriteRecords(records); + } + + private void OnDestroy() + { + while (observers.Count > 0) + { + var o = observers.FirstOrDefault(); + o.OnCompleted(); + } + } + + /// + /// Create a new log file. + /// + /// The path to the directory where the file should be placed. + /// An optional prefix string to appear in the filename before its timestamp. + public void CreateNewFile(string logDirectoryPath, string optionalFilenamePrefix = "") { Directory.CreateDirectory(logDirectoryPath); var fileName = $"{logDirectoryPath}{optionalFilenamePrefix}{DateTime.Now:yyyy-MM-dd_HH-mm-ss}.jsonl"; var observer = new JsonFileObserver(fileName); observer.Subscribe(this); - observers.Add(observer); } + /// + /// Create a new TCP client + /// + /// The path to the python script that starts the server socket locally + /// The path to the python executable file + public void CreateTcpClient(string filename, string pythonPath) + { + var wrapper = new TcpSocketObserver(filename, pythonPath); + wrapper.Subscribe(this); + } + /// /// The class JsonFileObserver implements the Observer pattern for JsonRedirectionData instances, /// serializing the information which it receives and writing it to a file. @@ -108,5 +180,146 @@ public JsonFileObserver(string filename) : base(filename) { } override public void OnNext(JsonRedirectionData value) => writer.WriteLine(JsonConvert.SerializeObject(value)); } - } + + /// + /// The class TcpSocketObserver implements the Observer pattern for JsonRedirectionData instances, + /// serializing the World Redirection information which it receives and sending it over a TCP connection to a local socket. + /// + private sealed class TcpSocketObserver : IObserver + { + private IDisposable unsubscriber; + + private TcpClient client; + private CancellationTokenSource periodicConnectTokenSrc, periodicSendMsgTokenSrc; + + private string filename; + private string pythonPath; + + private WRData currentRedirectionData; + + public TcpSocketObserver(string filename, string pythonPath) + { + // TODO move the python process in an entire different wrapper than the Socket observer + this.filename = filename; + this.pythonPath = pythonPath; + periodicConnectTokenSrc = new(); + periodicSendMsgTokenSrc = new(); + + currentRedirectionData = new(); + + if (filename is null || !filename.EndsWith(".py") || !File.Exists(filename)) + { + UnityEngine.Debug.LogWarning($"Invalid Python script {filename}."); + } + else if (pythonPath is null || !pythonPath.EndsWith(".exe") || !File.Exists(pythonPath)) + { + UnityEngine.Debug.LogWarning($"Invalid Python executable path {filename}."); + } + else + { + UnityEngine.Debug.Log($"Launch visualizer with Python {pythonPath}."); + // TODO not great for non-windows + System.Diagnostics.Process p = new() + { + StartInfo = new System.Diagnostics.ProcessStartInfo(pythonPath, filename) + { + ErrorDialog = false, + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + } + }; + + // allow for redirecting python process standard output to Unity's console + p.ErrorDataReceived += (sendingProcess, errorLine) => UnityEngine.Debug.LogError(errorLine.Data); + p.OutputDataReceived += (sendingProcess, dataLine) => UnityEngine.Debug.Log(dataLine.Data); + + p.Start(); + p.BeginErrorReadLine(); + p.BeginOutputReadLine(); + } + + PeriodicAsync(GetClient, TimeSpan.FromSeconds(5), periodicConnectTokenSrc.Token); + PeriodicAsync(StartSendingMessages, TimeSpan.FromSeconds(1), periodicSendMsgTokenSrc.Token); + } + public void Subscribe(IObservable observable) => unsubscriber = observable?.Subscribe(this); + + public void OnCompleted() + { + periodicConnectTokenSrc.Cancel(); + periodicSendMsgTokenSrc.Cancel(); + client.Close(); + unsubscriber.Dispose(); + } + + public void OnError(Exception error) + { + throw new NotImplementedException(); + } + + void IObserver.OnNext(JsonRedirectionData value) + { + var redirectionData = value.RedirectionData; + currentRedirectionData.AddTo(redirectionData.RotationOverTime, redirectionData.Rotational, + redirectionData.Curvature, redirectionData.Time); + } + + private void GetClient() + { + client ??= new(); + if (!client.Connected) + { + try + { + client.ConnectAsync("localhost", 13000); + } + catch (SocketException) { } + } + } + private void StartSendingMessages() + { + if (client != null && client.Connected) + { + string json = JsonConvert.SerializeObject(currentRedirectionData); + Thread thread = new(() => SendMessage(client, json)); + thread.Start(); + currentRedirectionData.Reset(); + } + } + + private void SendMessage(TcpClient client, string json) + { + + // Translate the passed message into ASCII and store it as a Byte array. + byte[] messageBytes = System.Text.Encoding.ASCII.GetBytes(json + '\n'); + + try + { + // Get a client stream for reading and writing. + NetworkStream stream = client.GetStream(); + // Send the message to the connected TcpServer. + stream.Write(messageBytes, 0, messageBytes.Length); + stream.Flush(); + } + catch (IOException) { UnityEngine.Debug.LogWarning("Socket closed."); } + } + + private static async void PeriodicAsync(Action action, TimeSpan interval, + CancellationToken cancellationToken = default) + { + while (true) + { + await Task.Run(action); + + try + { + await Task.Delay(interval, cancellationToken); + } + catch (TaskCanceledException) { break; } + } + } + } + + } } \ No newline at end of file diff --git a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs new file mode 100644 index 00000000..4e9737b8 --- /dev/null +++ b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs @@ -0,0 +1,44 @@ +using UnityEngine; +using UnityEditor; + +namespace VHToolkit.Logging +{ + [CustomEditor(typeof(JsonLogging)), CanEditMultipleObjects] + public class JsonLoggingEditor : Editor + { + SerializedProperty logDirectoryPath, optionalFilenamePrefix, + filename, pythonPath; + private void OnEnable() + { + logDirectoryPath = serializedObject.FindProperty("logDirectoryPath"); + optionalFilenamePrefix = serializedObject.FindProperty("optionalFilenamePrefix"); + filename = serializedObject.FindProperty("filename"); + pythonPath = serializedObject.FindProperty("pythonPath"); + } + private void MakePropertyField(SerializedProperty property, string text, string tooltip = null) + { + EditorGUILayout.PropertyField(property, new GUIContent(text, tooltip)); + } + + public override void OnInspectorGUI() + { + GUI.enabled = false; + EditorGUILayout.ObjectField("Script:", MonoScript.FromMonoBehaviour((JsonLogging) target), typeof(JsonLogging), false); + GUI.enabled = true; + + serializedObject.Update(); + + EditorGUILayout.Space(5); + EditorGUILayout.LabelField("Json logging Parameters", EditorStyles.largeLabel); + + MakePropertyField(logDirectoryPath, "Log directory Path", "The directory path inside which json file(s) will be created"); + MakePropertyField(optionalFilenamePrefix, "Optional filename prefix", "Optional prefix for the json file"); + + EditorGUILayout.Space(5); + EditorGUILayout.LabelField("TCP Socket Parameters", EditorStyles.largeLabel); + + MakePropertyField(filename, "Python visualizer script", "File name for the Python visualization script."); + MakePropertyField(pythonPath, "Python Path", "File name for the Python executable path."); + } + } +} \ No newline at end of file diff --git a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs.meta b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs.meta new file mode 100644 index 00000000..beb0b3b2 --- /dev/null +++ b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Utils/Editor/Logging/JsonLoggingEditor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2f9e5029405c15245ab528a1ef981958 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Visualisation/Socket.cs b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Visualisation/Socket.cs index 212599a6..f7377e0f 100644 --- a/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Visualisation/Socket.cs +++ b/Visuo-haptic Toolkit/Assets/Visuo-Haptic Toolkit/Scripts/Visualisation/Socket.cs @@ -13,11 +13,24 @@ namespace VHToolkit.Logging { [Serializable] - public struct WorldRedirectionData { + public struct WRData + { [SerializeField] public float overTime, rotational, curvature; [SerializeField] public float overTimeSum, rotationalSum, curvatureSum; - [SerializeField] float time; - [SerializeField] float[] maxSums; + [SerializeField] public float time; + [SerializeField] public float[] maxSums; + + public WRData(float overTime, float rotational, float curvature, float time) + { + this.overTime = overTime; + this.rotational = rotational; + this.curvature = curvature; + this.overTimeSum = 0.0f; + this.rotationalSum = 0.0f; + this.curvatureSum = 0.0f; + this.time = time; + this.maxSums = null; + } public void AddTo(float overTime, float rotational, float curvature, float time) { if (maxSums?.Length != 3) { @@ -43,12 +56,12 @@ public void Reset() { this.curvature = 0f; } } - public class VisualizerWrapper : IObserver { + public class VisualizerWrapper : IObserver { private string filename; private string pythonPath; private IDisposable unsubscriber; - public void Subscribe(IObservable observable) => unsubscriber = observable?.Subscribe(this); + public void Subscribe(IObservable observable) => unsubscriber = observable?.Subscribe(this); public VisualizerWrapper(string filename, string pythonPath) { this.filename = filename; @@ -81,17 +94,17 @@ public void OnError(Exception error) { throw new NotImplementedException(); } - void IObserver.OnNext(WorldRedirectionData value) { + void IObserver.OnNext(WRData value) { throw new NotImplementedException(); } } - public class Socket : MonoBehaviour, IObservable { + public class Socket : MonoBehaviour, IObservable { private Scene scene; private WorldRedirection script; private DateTime startTime; - private readonly HashSet> observers = new(); + private readonly HashSet> observers = new(); private TcpClient client; @@ -103,7 +116,7 @@ public class Socket : MonoBehaviour, IObservable { private Razzaque2001Hybrid loggingTechnique; - private WorldRedirectionData redirectionData; + private WRData redirectionData; private void Start() { script = GetComponent(); @@ -121,7 +134,7 @@ public void LaunchVisualizer() { wrapper.Subscribe(this); } - private void GetClient() { + private void GetClient() { client ??= new(); if (!client.Connected) { try { @@ -162,9 +175,9 @@ private void Update() { (float)(DateTime.Now - startTime).TotalSeconds); } - public IDisposable Subscribe(IObserver observer) { + public IDisposable Subscribe(IObserver observer) { observers.Add(observer); - return new HashSetUnsubscriber(observers, observer); + return new HashSetUnsubscriber(observers, observer); } } } diff --git a/python/RedirectionPlotter.py b/python/RedirectionPlotter.py index 0ea99d5d..ff6fd845 100644 --- a/python/RedirectionPlotter.py +++ b/python/RedirectionPlotter.py @@ -35,7 +35,7 @@ serversocket.bind(("localhost", 13000)) serversocket.listen(5) with serversocket.accept()[0] as clientsocket: - with clientsocket.makefile() as file: + with clientsocket.makefile('r') as file: for line in file: print(line) if not plt.get_fignums():