orx-kinect refactoring + new general orx-depth-camera

.gitignore

@@ -7,3 +7,5 @@ build/
 .idea/
 gradle.properties
 /ShaderError.txt
+/hs_err_pid*.log
+/gui-parameters/

README.md

@@ -37,6 +37,7 @@ A growing library of assorted data structures, algorithms and utilities.
 | [`orx-temporal-blur`](orx-temporal-blur/) | Post-processing temporal-blur video effect. CPU intense, therefore not intended  for use with the `ScreenRecorder` extension or other real-time uses. |
 | [`orx-time-operators`](orx-time-operators/) | A collection of time-sensitive functions aimed at controlling raw data over-time,  such as Envelope and LFO. |
 | [`orx-timer`](orx-timer/) | Simple timer functionality providing `repeat`, to run code with a given interval and `timeOut`, to run code once after a given delay. |
+| [`orx-depth-camera`](orx-depth-camera/) | Common API for various depth cameras like Kinect 1 and 2. |
 
 ## JVM only
 

build.gradle

@@ -33,7 +33,8 @@ def multiplatformModules = [
     "orx-shader-phrases",
     "orx-shapes",
     "orx-quadtree",
-    "orx-hash-grid"
+    "orx-hash-grid",
+    "orx-depth-camera"
 
 ]
 

orx-color/src/commonMain/kotlin/Color.kt

@@ -0,0 +1,5 @@
+// keeping this file here will stop IntelliJ from showing warning in nested relative packages
+/**
+ * orx-color
+ */
+package org.openrndr.extra.color

orx-depth-camera/build.gradle.kts

@@ -0,0 +1,31 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+kotlin {
+    jvm {
+        compilations.all {
+            kotlinOptions.jvmTarget = libs.versions.jvmTarget.get()
+            kotlinOptions.apiVersion = libs.versions.kotlinApi.get()
+        }
+        testRuns["test"].executionTask.configure {
+            useJUnitPlatform()
+        }
+    }
+    js(IR) {
+        browser()
+        nodejs()
+    }
+
+    sourceSets {
+        @Suppress("UNUSED_VARIABLE")
+        val commonMain by getting {
+            dependencies {
+                implementation(libs.openrndr.application)
+                implementation(libs.openrndr.math)
+                implementation(libs.kotlin.coroutines)
+            }
+        }
+    }
+
+}

orx-depth-camera/src/commonMain/kotlin/DepthCamera.kt

@@ -0,0 +1,42 @@
+package org.openrndr.extra.depth.camera
+
+import kotlinx.coroutines.flow.Flow
+import org.openrndr.draw.ColorBuffer
+import org.openrndr.math.IntVector2
+
+/**
+ * Defines how pixel values encoded in depth [ColorBuffer] will be interpreted.
+ */
+enum class DepthMeasurement {
+
+    /**
+     * Raw values, but normalized to the range 0-1.
+     * Useful for debugging, because full range of captured values can be rendered
+     * as a texture. Therefore it's a default setting.
+     */
+    RAW_NORMALIZED,
+
+    /**
+     * Raw values, exactly as they are provided by the device.
+     * Note: it might imply that [ColorBuffer] of the depth camera frame
+     * is provided in integer-based format (for example in case of Kinect devices).
+     */
+    RAW,
+
+    /**
+     * Expressed in meters.
+     * It is using floating point numbers.
+     * Note: values above `1.0` will not be visible if displayed as a texture.
+     */
+    METERS,
+
+}
+
+interface DepthCamera {
+    val resolution: IntVector2
+    var depthMeasurement: DepthMeasurement
+    var flipH: Boolean
+    var flipV: Boolean
+    val currentFrame: ColorBuffer
+    val frameFlow: Flow<ColorBuffer>
+}

orx-fx/README.md

@@ -11,7 +11,7 @@ All filters provided by orx-fx assume pre-multiplied alpha inputs, which is OPEN
 
 ## Effects index
 
-Here's a (potentially incomplete) list of the effects provded by orx-fx.
+Here's a (potentially incomplete) list of the effects provided by orx-fx.
 
 ### Anti-alias
 
@@ -107,8 +107,26 @@ All distortion effects are opacity preserving
 
  ### Transform
   - `FlipVertically` - flips the source input vertically.
-
-
+
+### Colormap
+
+Colormap filters operate only on the RED color channel. For example
+depth maps from
+[orx-depth-camera](https://github.com/openrndr/orx/tree/master/orx-depth-camera).
+
+They allow selection of `min` / `max` value range and applying exponential
+shaping `curve` within this range:
+
+ - `GrayscaleColormap` - maps to gray tones
+ - `SpectralZucconiColormap` - maps to natural light dispersion spectrum as described
+   by Alan Zucconi in the
+   [Improving the Rainbow](https://www.alanzucconi.com/2017/07/15/improving-the-rainbow/)
+   article.
+ - `TurboColormap` - maps to Turbo Colormap according to
+   [Turbo, An Improved Rainbow Colormap for Visualization](https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html)
+   by Google.
+
+
 <!-- __demos__ >
 # Demos
 [DemoFluidDistort01Kt](src/demo/kotlin/DemoFluidDistort01Kt.kt

orx-fx/src/commonMain/kotlin/colormap/ColormapFilter.kt

@@ -0,0 +1,24 @@
+package org.openrndr.extra.fx.colormap
+
+import org.openrndr.draw.Filter
+import org.openrndr.extra.fx.mppFilterShader
+import org.openrndr.extra.parameters.DoubleParameter
+
+abstract class ColormapFilter(code: String, name: String) : Filter(mppFilterShader(code, name)) {
+
+    @DoubleParameter(label = "min value", low = 0.0, high = 1.0, order = 0)
+    var minValue: Double by parameters
+
+    @DoubleParameter(label = "max value", low = 0.0, high = 1.0, order = 1)
+    var maxValue: Double by parameters
+
+    @DoubleParameter(label = "curve", low = 0.001, high = 10.0, order = 2)
+    var curve: Double by parameters
+
+    init {
+        minValue = 0.0
+        maxValue = 1.0
+        curve = 1.0
+    }
+
+}

orx-fx/src/commonMain/kotlin/colormap/GrayscaleColormap.kt

@@ -0,0 +1,10 @@
+package org.openrndr.extra.fx.colormap
+
+import org.openrndr.extra.fx.fx_grayscale_colormap
+import org.openrndr.extra.parameters.Description
+
+/**
+ * Maps values of the RED color channel to grayscale.
+ */
+@Description("grayscale colormap")
+class GrayscaleColormap : ColormapFilter(fx_grayscale_colormap, "grayscale-colormap")

orx-fx/src/commonMain/kotlin/colormap/SpectralZucconiColormap.kt

@@ -0,0 +1,13 @@
+package org.openrndr.extra.fx.colormap
+
+import org.openrndr.extra.fx.fx_spectral_zucconi_colormap
+import org.openrndr.extra.parameters.Description
+
+/**
+ * Maps values of the RED color channel to natural light dispersion spectrum as described
+ * by Alan Zucconi in the
+ * [Improving the Rainbow](https://www.alanzucconi.com/2017/07/15/improving-the-rainbow/)
+ * article.
+ */
+@Description("spectral colormap")
+class SpectralZucconiColormap : ColormapFilter(fx_spectral_zucconi_colormap, "spectral-zucconi-colormap")

orx-fx/src/commonMain/kotlin/colormap/TurboColormap.kt

@@ -0,0 +1,12 @@
+package org.openrndr.extra.fx.colormap
+
+import org.openrndr.extra.fx.fx_turbo_colormap
+import org.openrndr.extra.parameters.Description
+
+/**
+ * Maps values of the RED color channel to Turbo Colormap according to
+ * [Turbo, An Improved Rainbow Colormap for Visualization](https://ai.googleblog.com/2019/08/turbo-improved-rainbow-colormap-for.html)
+ * by Google.
+ */
+@Description("turbo colormap")
+open class TurboColormap : ColormapFilter(fx_turbo_colormap, "turbo-colormap")

orx-fx/src/shaders/glsl/colormap/grayscale-colormap.frag

@@ -0,0 +1,31 @@
+#ifdef OR_IN_OUT
+in vec2 v_texCoord0;
+#else
+varying vec2 v_texCoord0;
+#endif
+
+uniform sampler2D   tex0;
+uniform float       minValue;
+uniform float       maxValue;
+uniform float       curve;
+
+#ifndef OR_GL_FRAGCOLOR
+out vec4 o_color;
+#endif
+
+void main() {
+    #ifndef OR_GL_TEXTURE2D
+    float red = texture(tex0, v_texCoord0).r;
+    #else
+    float red = texture2D(tex0, v_texCoord0).r;
+    #endif
+    float value = (red - minValue) / (maxValue - minValue);
+    vec3 color = vec3(pow(value, curve));
+    color *= step(value, 1.) * step(0., value);
+    vec4 result = vec4(color, 1.);
+    #ifdef OR_GL_FRAGCOLOR
+    gl_FragColor = result;
+    #else
+    o_color = result;
+    #endif
+}

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-zucconi6.frag → orx-fx/src/shaders/glsl/colormap/spectral-zucconi-colormap.frag

@@ -1,8 +1,3 @@
-#version 330
-
-uniform sampler2D tex0;             // kinect raw
-out     vec3 color;
-
 // Spectral Colour Schemes
 // By Alan Zucconi
 // Website: www.alanzucconi.com
@@ -19,7 +14,20 @@ out     vec3 color;
 // Read "Improving the Rainbow" for more information
 // http://www.alanzucconi.com/?p=6703
 
+#ifdef OR_IN_OUT
+in vec2 v_texCoord0;
+#else
+varying vec2 v_texCoord0;
+#endif
+
+uniform sampler2D tex0;             // kinect raw
+uniform float     minValue;
+uniform float     maxValue;
+uniform float     curve;
 
+#ifndef OR_GL_FRAGCOLOR
+out vec4 o_color;
+#endif
 
 float saturate (float x)
 {
@@ -58,11 +66,23 @@ vec3 spectral_zucconi6 (float x)
     const vec3 y2 = vec3(0.84897130, 0.88445281, 0.73949448);
 
     return
-    bump3y(c1 * (x - x1), y1) +
-    bump3y(c2 * (x - x2), y2) ;
+        bump3y(c1 * (x - x1), y1) +
+        bump3y(c2 * (x - x2), y2);
 }
 
 void main() {
-    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
-    color = (depth >= .999) ? vec3(0) : spectral_zucconi6(depth);
+    #ifndef OR_GL_TEXTURE2D
+    float red = texture(tex0, v_texCoord0).r;
+    #else
+    float red = texture2D(tex0, v_texCoord0).r;
+    #endif
+    float value = (red - minValue) / (maxValue - minValue);
+    vec3 color = spectral_zucconi6(pow(value, curve));
+    color *= step(value, 1.) * step(0., value);
+    vec4 result = vec4(color, 1.);
+    #ifdef OR_GL_FRAGCOLOR
+    gl_FragColor = result;
+    #else
+    o_color = result;
+    #endif
 }

orx-jvm/orx-kinect-common/src/main/resources/org/openrndr/extra/kinect/depth-to-colors-turbo.frag → orx-fx/src/shaders/glsl/colormap/turbo-colormap.frag

@@ -1,12 +1,4 @@
-#version 330
-
-uniform sampler2D tex0;
-out     vec3 color;
-
-float saturate(in float x) {
-    return max(0, min(1, x));
-}
-
+// TurboColormap
 // Copyright 2019 Google LLC.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -17,6 +9,25 @@ float saturate(in float x) {
 //   Colormap Design: Anton Mikhailov (mikhailov@google.com)
 //   GLSL Approximation: Ruofei Du (ruofei@google.com)
 
+#ifdef OR_IN_OUT
+in vec2 v_texCoord0;
+#else
+varying vec2 v_texCoord0;
+#endif
+
+uniform sampler2D tex0;
+uniform float     minValue;
+uniform float     maxValue;
+uniform float     curve;
+
+#ifndef OR_GL_FRAGCOLOR
+out vec4 o_color;
+#endif
+
+float saturate(in float x) {
+    return max(0, min(1, x));
+}
+
 vec3 TurboColormap(in float x) {
     const vec4 kRedVec4 = vec4(0.13572138, 4.61539260, -42.66032258, 132.13108234);
     const vec4 kGreenVec4 = vec4(0.09140261, 2.19418839, 4.84296658, -14.18503333);
@@ -29,13 +40,25 @@ vec3 TurboColormap(in float x) {
     vec4 v4 = vec4( 1.0, x, x * x, x * x * x);
     vec2 v2 = v4.zw * v4.z;
     return vec3(
-    dot(v4, kRedVec4)   + dot(v2, kRedVec2),
-    dot(v4, kGreenVec4) + dot(v2, kGreenVec2),
-    dot(v4, kBlueVec4)  + dot(v2, kBlueVec2)
+        dot(v4, kRedVec4)   + dot(v2, kRedVec2),
+        dot(v4, kGreenVec4) + dot(v2, kGreenVec2),
+        dot(v4, kBlueVec4)  + dot(v2, kBlueVec2)
     );
 }
 
 void main() {
-    float depth = texelFetch(tex0, ivec2(int(gl_FragCoord.x), int(gl_FragCoord.y)), 0).r;
-    color = (depth >= .999) ? vec3(0) : TurboColormap(depth);
+    #ifndef OR_GL_TEXTURE2D
+    float red = texture(tex0, v_texCoord0).r;
+    #else
+    float red = texture2D(tex0, v_texCoord0).r;
+    #endif
+    float value = (red - minValue) / (maxValue - minValue);
+    vec3 color = TurboColormap(pow(value, curve));
+    color *= step(value, 1.) * step(0., value);
+    vec4 result = vec4(color, 1.);
+    #ifdef OR_GL_FRAGCOLOR
+    gl_FragColor = result;
+    #else
+    o_color = result;
+    #endif
 }

orx-jvm/orx-kinect-common/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    api project(":orx-depth-camera")
+}