✨ blur and body segmentation tweaks, adding kernel presets

package.json

@@ -1,6 +1,6 @@
 {
   "name": "graphfx",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Graph image processing pipeline",
   "main": "src/index.ts",
   "scripts": {

src/nodes/AI/BodySegmentation.ts

@@ -15,28 +15,17 @@ import {
 } from '@tensorflow-models/body-segmentation'
 import {canvasPool2D} from "../../canvas/CanvasPool";
 import {waitForMedia} from "../../utils";
+import { mediaSize } from '../canvas';
 
 const inputs = {
     image: {
         type: 'Image'
     } as ImageVar,
-    width: {
-        type: 'Number',
-        default: 100,
-        step: 1,
-        min: 1
-    } as NumberVar,
-    height: {
-        type: 'Number',
-        default: 100,
-        step: 1,
-        min: 1,
-    } as NumberVar,
     // blur mask edges by number of pixels
     maskBlurAmount: {
         type: 'Number',
-        default: 0,
+        default: 1,
-        step: 1,
+        step: 0.1,
     } as NumberVar,
     // Default to 0.5. The minimum probability to color a pixel as foreground rather than background. The alpha channel integer values will be taken as the probabilities (for more information refer to Segmentation type's documentation).
     foregroundThreshold: {
@@ -70,29 +59,6 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
         super('BodySegmentation', inputs, outputs);
     }
 
-    get width() {
-        return this.__in.width.value;
-    }
-
-    get height() {
-        return this.__in.height.value;
-    }
-
-    get image() {
-        if (!this.__in.image.value) {
-            return null;
-        } else {
-            const {width, height} = this.__in.image.value;
-            return {
-                image: this.__in.image.value,
-                top: 0,
-                left: 0,
-                width,
-                height,
-            };
-        }
-    }
-
     async initSegmenter() {
         if (this.segmenter) {
             return;
@@ -108,13 +74,14 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
     async segmentPeople() {
         await this.initSegmenter();
         await waitForMedia(this.in.image.value);
+        const {width, height} = mediaSize(this.__in.image.value);
         if (!this.in.image.value) {
             return;
         }
 
         const originalImageCanvas = canvasPool2D.createCanvas();
-        originalImageCanvas.width = this.in.width.value;
+        originalImageCanvas.width = width;
-        originalImageCanvas.height = this.in.height.value
+        originalImageCanvas.height = height;
         originalImageCanvas.acquire();
         const originalCanvasCtx = originalImageCanvas.getContext('2d');
 
@@ -137,16 +104,15 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
         const canvas = canvasPool2D.createCanvas();
         canvas.acquire();
         const ctx = canvas.getContext('2d');
-        canvas.width = this.in.width.value;
+        canvas.width = width;
-        canvas.height = this.in.height.value
+        canvas.height = height;
 
         const maskBlurAmount = this.in.maskBlurAmount.value;
         await drawMask(canvas, canvas as CanvasImageSource, backgroundDarkeningMask, 1, maskBlurAmount);
-
         this.out.image.value = canvas;
         this.out.originalImage.value = originalImageCanvas;
-        this.out.width.value = this.in.width.value;
+        this.out.width.value = width;
-        this.out.height.value = this.in.height.value;
+        this.out.height.value = height;
     }
 
     async _update(){

src/nodes/Canvas2d/Flip.ts

@@ -1,7 +1,7 @@
 import Canvas2d from './Canvas2d';
 import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
 import {
-    BooleanVar
+    BooleanVar, ImageVar
   } from '../io/AbstractIOSet';
 
 const inputs = {

src/nodes/WebGL/Blur.ts

@@ -10,11 +10,14 @@ const inputs = {
     } as ImageVar,
     radius: {
         type: 'Number',
-        default: 0,
+        default: 8,
+        min: 0,
     } as NumberVar,
     sigma: {
         type: 'Number',
-        default: 0,
+        default: 1,
+        step: 0.1,
+        min: 0,
     } as NumberVar,
     passes: {
         type: 'Number',
@@ -105,7 +108,6 @@ export default class Blur extends WebGL<typeof inputs> {
             }
 
             gl_FragColor = colorSum / totalWeight;
-            gl_FragColor.a = 1.0;
         }
         `
     }

src/nodes/WebGL/KernelPreset.ts

@@ -0,0 +1,216 @@
+import WebGL from './WebGl';
+import {
+    ImageVar,
+    NumberVar,
+    StringVar
+} from '../io/AbstractIOSet';
+
+const kernels = {
+    'sobel': {
+        size: 3,
+        kernel: [
+            -1.0, -2.0, -1.0,
+            0.0,  0.0,  0.0,
+            1.0,  2.0,  1.0
+        ]
+    },
+    'laplacian': {
+        size: 3,
+        kernel: [
+            0.0,  1.0, 0.0,
+            1.0, -4.0, 1.0,
+            0.0,  1.0, 0.0
+        ]
+    },
+    'laplacianOfGaussian': {
+        size: 5,
+        kernel: [
+            0.0,  0.0, -1.0,  0.0,  0.0,
+            0.0, -1.0, -2.0, -1.0,  0.0,
+           -1.0, -2.0, 16.0, -2.0, -1.0,
+            0.0, -1.0, -2.0, -1.0,  0.0,
+            0.0,  0.0, -1.0,  0.0,  0.0
+        ]
+    },
+    'emboss': {
+        size: 3,
+        kernel: [
+            -2.0, -1.0,  0.0,
+            -1.0,  1.0,  1.0,
+            0.0,  1.0,  2.0
+        ]
+    },
+    'sharpen': {
+        size: 3,
+        kernel: [
+            0.0, -1.0,  0.0,
+            -1.0,  5.0, -1.0,
+             0.0, -1.0,  0.0
+        ]
+    },
+    'boxBlur': {
+        size: 3,
+        kernel: [
+            1.0/9.0, 1.0/9.0, 1.0/9.0,
+            1.0/9.0, 1.0/9.0, 1.0/9.0,
+            1.0/9.0, 1.0/9.0, 1.0/9.0
+        ]
+    },
+    'outline': {
+        size: 3,
+        kernel: [
+            -2.0, -1.0,  0.0,
+            -1.0,  1.0,  1.0,
+            0.0,  1.0,  2.0
+        ]
+    },
+    'erosion': {
+        size: 3,
+        kernel: [
+            0.0, 1.0, 0.0,
+            1.0, 1.0, 1.0,
+            0.0, 1.0, 0.0
+        ]
+    },
+    'dilation': {
+        size: 3,
+        kernel: [
+            1.0, 1.0, 1.0,
+            1.0, 0.0, 1.0,
+            1.0, 1.0, 1.0
+        ]
+    },
+    'unsharpMask': {
+        size: 5,
+        kernel: [
+             -1.0/256.0,   -4.0/256.0,   -6.0/256.0,   -4.0/256.0,  -1.0/256.0,
+             -4.0/256.0,    1.0/256.0,   13.0/256.0,    1.0/256.0,  -4.0/256.0,
+             -6.0/256.0,   13.0/256.0,  269.0/256.0,   13.0/256.0,  -6.0/256.0,
+             -4.0/256.0,    1.0/256.0,   13.0/256.0,    1.0/256.0,  -4.0/256.0,
+             -1.0/256.0,   -4.0/256.0,   -6.0/256.0,   -4.0/256.0,  -1.0/256.0
+        ]
+    }
+}
+
+
+const inputs = {
+    image: {
+        type: 'Image',
+    } as ImageVar,
+    preset: {
+        type: 'String',
+        default: 'sobel',
+        enum: Object.keys(kernels),
+    } as StringVar,
+    passes: {
+        type: 'Number',
+        default: 1,
+    } as NumberVar,
+    multiplier: {
+        type: 'Number',
+        default: 1,
+    } as NumberVar,
+}
+
+
+const outputs = {
+    image: {
+        type: 'Image',
+    } as ImageVar,
+    width: {
+        type: 'Number',
+    } as NumberVar,
+    height: {
+        type: 'Number',
+    } as NumberVar
+}
+
+export default class KernelPreset extends WebGL<typeof inputs> {
+
+    _lastKernelSize?: number;
+
+    constructor() {
+        super('KernelPreset', inputs)
+    }
+
+    _passes() {
+
+        const passes = Array(Math.max(this.in.passes.value, 0))
+            .fill(null)
+            .flatMap((x, i) => {
+                return [
+                    (gl, program) => {
+                        if (i === 0) {
+                            this._setParams(gl, program)
+                        }
+                    }
+                ]
+            });
+        return passes;
+    }
+
+    get currentPreset() {
+        return kernels[this.in.preset.value];
+    }
+
+    setup() {
+        if (this.currentPreset.size !== this._lastKernelSize) {
+            this.program = undefined;
+        }
+        super.setup();
+        this._lastKernelSize = this.currentPreset.size;
+    }
+
+    get frag() {
+        const KERNEL_SIZE = Math.ceil(this.currentPreset.size);
+        const shader = `
+        precision mediump float;
+
+        // our texture
+        uniform sampler2D u_image;
+        uniform vec2 u_textureSize;
+        uniform float u_kernel[${KERNEL_SIZE * KERNEL_SIZE}];
+
+        // the texCoords passed in from the vertex shader.
+        varying vec2 v_texCoord;
+
+        const int RADIUS = ${KERNEL_SIZE};
+
+        vec4 applyKernel(in vec2 texCoord) {
+            vec2 onePixel = vec2(1.0, 1.0) / u_textureSize;
+            vec4 colorSum = vec4(0.0);
+
+            for (int y = 0; y < RADIUS; y++) {
+                for (int x = 0; x < RADIUS; x++) {
+                    vec2 offset = vec2(float(x - RADIUS/2), float(y - RADIUS/2));
+                    colorSum += texture2D(u_image, texCoord + onePixel * offset) * u_kernel[y * RADIUS + x];
+                }
+            }
+
+            return colorSum;
+        }
+
+        void main() {
+            gl_FragColor = applyKernel(v_texCoord);
+        }
+        `;
+        console.log(shader);
+        return shader;
+    }
+
+
+    /**
+     *
+     * @param {WebGLRenderingContext} gl
+     * @param {*} program
+     */
+    _setParams(gl, program) {
+        const kernel = this.currentPreset.kernel;
+        if (kernel) {
+            gl.uniform1fv(
+                gl.getUniformLocation(program, 'u_kernel'),
+                kernel.map((x) => x * this.in.multiplier.value)
+            );
+        }
+    }
+}

src/nodes/index.ts

@@ -18,6 +18,7 @@ export {default as ColorLookupTable} from './WebGL/ColorLookupTable';
 export {default as Vignette} from './WebGL/Vignette';
 export {default as Blur} from './WebGL/Blur';
 export {default as Edge} from './WebGL/Edge';
+export {default as KernelPreset} from './WebGL/KernelPreset';
 // Math
 export {default as Numbers} from './Math/Numbers';
 export {default as NumberBinaryOperation} from './Math/NumberBinaryOperation';