Implement GPU acceleration for VR overlays

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- GPU acceleration for SteamVR overlays (Community contribution by [BenjaminZehowlt](https://github.com/BenjaminZehowlt))
+
 ## [1.14.3]
 
 ### Changed

src-overlay-sidecar/Managers/OvrManager.cs

@@ -6,6 +6,11 @@
 using SharpDX.DXGI;
 using Valve.VR;
 using Device = SharpDX.Direct3D11.Device;
+using Device1 = SharpDX.Direct3D11.Device1;
+using Device2 = SharpDX.Direct3D11.Device2;
+using Device3 = SharpDX.Direct3D11.Device3;
+using Device4 = SharpDX.Direct3D11.Device4;
+using Device5 = SharpDX.Direct3D11.Device5;
 
 namespace overlay_sidecar;
 
@@ -24,6 +29,9 @@ public class OvrManager {
   private MicMuteIndicatorOverlay? _micMuteIndicatorOverlay;
   private bool _active;
   private Device? _device;
+  private Device1? _device1;
+  private DeviceMultithread? _deviceMultithread;
+  private Query? _query;
   private Dictionary<string, List<OvrInputDevice>> inputActions = new();
   public event EventHandler<Dictionary<string, List<OvrInputDevice>>> OnInputActionsChanged;
 
@@ -35,6 +43,8 @@ public class OvrManager {
   public OverlayPointer? OverlayPointer => _overlayPointer;
 
   public Device D3D11Device => _device!;
+  public Device1? D3D11Device1 => _device1;
+  public Query D3D11Query => _query!;
 
   private OvrManager()
   {
@@ -65,6 +75,18 @@ private async Task InitializeDevice()
             DeviceCreationFlags.BgraSupport)
           : new Device(DriverType.Hardware,
             DeviceCreationFlags.BgraSupport);
+        UpgradeDevice();
+
+        _device1 = _device.QueryInterface<Device1>();
+
+        _deviceMultithread = _device.QueryInterfaceOrNull<DeviceMultithread>();
+        _deviceMultithread?.SetMultithreadProtected(true);
+
+        _query = new Query(_device, new QueryDescription
+        {
+          Type = QueryType.Event,
+          Flags = QueryFlags.None
+        });
       }
       catch (SharpDXException err)
       {
@@ -82,6 +104,45 @@ private async Task InitializeDevice()
     }
   }
 
+  // Upgrades the device to the newest supported interface.
+  private void UpgradeDevice()
+  {
+    Device5 device5 = _device!.QueryInterfaceOrNull<Device5>();
+    if (device5 != null)
+    {
+      _device.Dispose();
+      _device = device5;
+      return;
+    }
+    Device4 device4 = _device.QueryInterfaceOrNull<Device4>();
+    if (device4 != null)
+    {
+      _device.Dispose();
+      _device = device4;
+      return;
+    }
+    Device3 device3 = _device.QueryInterfaceOrNull<Device3>();
+    if (device3 != null)
+    {
+      _device.Dispose();
+      _device = device3;
+      return;
+    }
+    Device2 device2 = _device.QueryInterfaceOrNull<Device2>();
+    if (device2 != null)
+    {
+      _device.Dispose();
+      _device = device2;
+      return;
+    }
+    Device1 device1 = _device.QueryInterfaceOrNull<Device1>();
+    if (device1 != null)
+    {
+      _device.Dispose();
+      _device = device1;
+    }
+  }
+
   private void OverlayRenderLoop()
   {
     var timer = new RefreshRateTimer();

src-overlay-sidecar/Overlays/BaseWebOverlay.cs

@@ -71,11 +71,11 @@ private async void Init(int resolution)
             ArraySize = 1,
             Format = Format.B8G8R8A8_UNorm,
             SampleDescription = new SampleDescription(1, 0),
-            Usage = ResourceUsage.Dynamic,
-            BindFlags = BindFlags.ShaderResource,
-            CpuAccessFlags = CpuAccessFlags.Write
+            BindFlags = BindFlags.ShaderResource
           }
         );
+
+        Browser!.UpdateTexture(_texture);
       }
       catch (SharpDXException err)
       {
@@ -210,8 +210,7 @@ public void UpdateFrame()
     // Stop here if we are not ready, already disposed, or if the browser hasn't painted anything new for the past second or so.
     if (_texture == null || Disposed || Browser == null ||
         DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - Browser.LastPaint >= 1000) return;
-    // Render the browser to the texture
-    Browser.RenderToTexture(_texture);
+
     var texture = new Texture_t
     {
       handle = _texture.NativePointer

src-overlay-sidecar/Overlays/Helpers/OffScreenBrowser.cs

@@ -1,5 +1,7 @@
 // Source: https://github.com/vrcx-team/VRCX/blob/master/OffScreenBrowser.cs
 
+using SharpDX.Mathematics.Interop;
+
 namespace overlay_sidecar;
 
 using CefSharp;
@@ -12,25 +14,32 @@ namespace overlay_sidecar;
 using System.Threading;
 
 public class OffScreenBrowser : ChromiumWebBrowser, IRenderHandler {
-  private readonly ReaderWriterLockSlim _paintBufferLock;
-  private GCHandle _paintBuffer;
-  private int _width;
-  private int _height;
+  private Texture2D? _texture;
   private long _lastPaint;
   public long LastPaint => _lastPaint;
 
   public OffScreenBrowser(string address, int width, int height)
     : base(
       address,
-      new BrowserSettings()
-      {
-        WindowlessFrameRate = 60,
-        WebGl = CefState.Enabled,
-        DefaultEncoding = "UTF-8"
-      }
+      automaticallyCreateBrowser: false
     )
   {
-    _paintBufferLock = new ReaderWriterLockSlim();
+    var windowInfo = new WindowInfo();
+    windowInfo.SetAsWindowless(IntPtr.Zero);
+    windowInfo.WindowlessRenderingEnabled = true;
+    windowInfo.SharedTextureEnabled = true;
+    windowInfo.Width = width;
+    windowInfo.Height = height;
+
+    var browserSettings = new BrowserSettings()
+    {
+      WindowlessFrameRate = 60,
+      WebGl = CefState.Enabled,
+      DefaultEncoding = "UTF-8"
+    };
+
+    CreateBrowser(windowInfo, browserSettings);
+
     Size = new System.Drawing.Size(width, height);
     RenderHandler = this;
   }
@@ -40,72 +49,19 @@ public OffScreenBrowser(string address, int width, int height)
     RenderHandler = null;
     if (IsDisposed) return;
     base.Dispose();
-
-    _paintBufferLock.EnterWriteLock();
-    try
-    {
-      if (_paintBuffer.IsAllocated) _paintBuffer.Free();
-    }
-    finally
-    {
-      _paintBufferLock.ExitWriteLock();
-    }
-
-    _paintBufferLock.Dispose();
   }
 
-  public void RenderToTexture(Texture2D texture)
+  public void UpdateTexture(Texture2D texture)
   {
-    _paintBufferLock.EnterReadLock();
-    try
-    {
-      if (_width > 0 &&
-          _height > 0)
-      {
-        var context = texture.Device.ImmediateContext;
-        var dataBox = context.MapSubresource(
-          texture,
-          0,
-          MapMode.WriteDiscard,
-          MapFlags.None
-        );
-        if (dataBox.IsEmpty == false)
-        {
-          var sourcePtr = _paintBuffer.AddrOfPinnedObject();
-          var destinationPtr = dataBox.DataPointer;
-          var pitch = _width * 4;
-          var rowPitch = dataBox.RowPitch;
-          if (pitch == rowPitch)
-            WinApi.CopyMemory(
-              destinationPtr,
-              sourcePtr,
-              (uint)(_width * _height * 4)
-            );
-          else
-            for (var y = _height; y > 0; --y)
-            {
-              WinApi.CopyMemory(
-                destinationPtr,
-                sourcePtr,
-                (uint)pitch
-              );
-              sourcePtr += pitch;
-              destinationPtr += rowPitch;
-            }
-        }
-        context.UnmapSubresource(texture, 0);
-      }
-    }
-    finally
-
-    {
-      _paintBufferLock.ExitReadLock();
-    }
+    _texture = texture;
   }
 
   ScreenInfo? IRenderHandler.GetScreenInfo()
   {
-    return null;
+    return new ScreenInfo
+    {
+      DeviceScaleFactor = 1.0F
+    };
   }
 
   bool IRenderHandler.GetScreenPoint(int viewX, int viewY, out int screenX, out int screenY)
@@ -120,8 +76,30 @@ Rect IRenderHandler.GetViewRect()
     return new Rect(0, 0, Size.Width, Size.Height);
   }
 
-  void IRenderHandler.OnAcceleratedPaint(PaintElementType type, Rect dirtyRect, IntPtr sharedHandle)
+  void IRenderHandler.OnAcceleratedPaint(PaintElementType type, Rect dirtyRect, AcceleratedPaintInfo paintInfo)
   {
+    if (type != PaintElementType.View) return;
+    if (OvrManager.Instance.D3D11Device1 == null) return;
+    if (_texture == null) return;
+
+    using var cefTexture =
+      OvrManager.Instance.D3D11Device1.OpenSharedResource1<Texture2D>(paintInfo.SharedTextureHandle);
+    var context = OvrManager.Instance.D3D11Device.ImmediateContext;
+    context.CopyResource(cefTexture, _texture);
+
+    Query query = OvrManager.Instance.D3D11Query;
+    context.End(query);
+    context.Flush();
+
+    RawBool q = context.GetData<RawBool>(query, AsynchronousFlags.DoNotFlush);
+
+    while (!q)
+    {
+      Thread.Yield();
+      q = context.GetData<RawBool>(query, AsynchronousFlags.DoNotFlush);
+    }
+
+    _lastPaint = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
   }
 
   void IRenderHandler.OnCursorChange(IntPtr cursor, CursorType type, CursorInfo customCursorInfo)
@@ -134,35 +112,6 @@ void IRenderHandler.OnImeCompositionRangeChanged(CefSharp.Structs.Range selected
 
   void IRenderHandler.OnPaint(PaintElementType type, Rect dirtyRect, IntPtr buffer, int width, int height)
   {
-    if (type != PaintElementType.View) return;
-    _paintBufferLock.EnterWriteLock();
-    try
-    {
-      if (_width != width ||
-          _height != height)
-      {
-        _width = width;
-        _height = height;
-        if (_paintBuffer.IsAllocated) _paintBuffer.Free();
-
-        _paintBuffer = GCHandle.Alloc(
-          new byte[_width * _height * 4],
-          GCHandleType.Pinned
-        );
-      }
-
-      WinApi.CopyMemory(
-        _paintBuffer.AddrOfPinnedObject(),
-        buffer,
-        (uint)(width * height * 4)
-      );
-
-      _lastPaint = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-    }
-    finally
-    {
-      _paintBufferLock.ExitWriteLock();
-    }
   }
 
   void IRenderHandler.OnPopupShow(bool show)

src-overlay-sidecar/oyasumivr-overlay-sidecar.csproj

@@ -15,7 +15,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="CefSharp.OffScreen.NETCore" Version="114.2.100"/>
+    <PackageReference Include="CefSharp.OffScreen.NETCore" Version="129.0.110" />
     <PackageReference Include="Grpc.AspNetCore.Web" Version="2.54.0"/>
     <PackageReference Include="Serilog" Version="3.0.1"/>
     <PackageReference Include="Serilog.AspNetCore" Version="7.0.0"/>

src-ui/app/views/dashboard-view/views/about-view/about-view.component.html

@@ -59,6 +59,12 @@
             <span translate>about.contributionType.feature</span></span
           >
         </div>
+        <div class="contributor-list-entry">
+          <span
+          ><a href="https://github.com/BenjaminZehowlt" target="_blank">BenjaminZehowlt</a> |
+            <span translate>about.contributionType.feature</span></span
+          >
+        </div>
         <div class="contributor-list-entry">
           <span>spaecd | <span translate>about.contributionType.soundFx</span></span>
         </div>