From 3fd8f610fe35e01909aae045a2690404a8ebb341 Mon Sep 17 00:00:00 2001 From: Stanislav Fifik Date: Fri, 24 Mar 2023 14:48:44 +0100 Subject: [PATCH] :sparkles: blur and body segmentation tweaks, adding kernel presets --- package.json | 2 +- src/nodes/AI/BodySegmentation.ts | 54 ++------ src/nodes/Canvas2d/Flip.ts | 2 +- src/nodes/WebGL/Blur.ts | 8 +- src/nodes/WebGL/KernelPreset.ts | 216 +++++++++++++++++++++++++++++++ src/nodes/index.ts | 1 + 6 files changed, 234 insertions(+), 49 deletions(-) create mode 100644 src/nodes/WebGL/KernelPreset.ts diff --git a/package.json b/package.json index 108ed58..9f81e52 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "graphfx", - "version": "0.4.0", + "version": "0.4.1", "description": "Graph image processing pipeline", "main": "src/index.ts", "scripts": { diff --git a/src/nodes/AI/BodySegmentation.ts b/src/nodes/AI/BodySegmentation.ts index ecc7dd5..0423531 100644 --- a/src/nodes/AI/BodySegmentation.ts +++ b/src/nodes/AI/BodySegmentation.ts @@ -15,28 +15,17 @@ import { } from '@tensorflow-models/body-segmentation' import {canvasPool2D} from "../../canvas/CanvasPool"; import {waitForMedia} from "../../utils"; +import { mediaSize } from '../canvas'; const inputs = { image: { type: 'Image' } as ImageVar, - width: { - type: 'Number', - default: 100, - step: 1, - min: 1 - } as NumberVar, - height: { - type: 'Number', - default: 100, - step: 1, - min: 1, - } as NumberVar, // blur mask edges by number of pixels maskBlurAmount: { type: 'Number', - default: 0, - step: 1, + default: 1, + step: 0.1, } as NumberVar, // Default to 0.5. The minimum probability to color a pixel as foreground rather than background. The alpha channel integer values will be taken as the probabilities (for more information refer to Segmentation type's documentation). foregroundThreshold: { @@ -70,29 +59,6 @@ export default class BodySegmentation extends Node { } gl_FragColor = colorSum / totalWeight; - gl_FragColor.a = 1.0; } ` } diff --git a/src/nodes/WebGL/KernelPreset.ts b/src/nodes/WebGL/KernelPreset.ts new file mode 100644 index 0000000..c9df2e7 --- /dev/null +++ b/src/nodes/WebGL/KernelPreset.ts @@ -0,0 +1,216 @@ +import WebGL from './WebGl'; +import { + ImageVar, + NumberVar, + StringVar +} from '../io/AbstractIOSet'; + +const kernels = { + 'sobel': { + size: 3, + kernel: [ + -1.0, -2.0, -1.0, + 0.0, 0.0, 0.0, + 1.0, 2.0, 1.0 + ] + }, + 'laplacian': { + size: 3, + kernel: [ + 0.0, 1.0, 0.0, + 1.0, -4.0, 1.0, + 0.0, 1.0, 0.0 + ] + }, + 'laplacianOfGaussian': { + size: 5, + kernel: [ + 0.0, 0.0, -1.0, 0.0, 0.0, + 0.0, -1.0, -2.0, -1.0, 0.0, + -1.0, -2.0, 16.0, -2.0, -1.0, + 0.0, -1.0, -2.0, -1.0, 0.0, + 0.0, 0.0, -1.0, 0.0, 0.0 + ] + }, + 'emboss': { + size: 3, + kernel: [ + -2.0, -1.0, 0.0, + -1.0, 1.0, 1.0, + 0.0, 1.0, 2.0 + ] + }, + 'sharpen': { + size: 3, + kernel: [ + 0.0, -1.0, 0.0, + -1.0, 5.0, -1.0, + 0.0, -1.0, 0.0 + ] + }, + 'boxBlur': { + size: 3, + kernel: [ + 1.0/9.0, 1.0/9.0, 1.0/9.0, + 1.0/9.0, 1.0/9.0, 1.0/9.0, + 1.0/9.0, 1.0/9.0, 1.0/9.0 + ] + }, + 'outline': { + size: 3, + kernel: [ + -2.0, -1.0, 0.0, + -1.0, 1.0, 1.0, + 0.0, 1.0, 2.0 + ] + }, + 'erosion': { + size: 3, + kernel: [ + 0.0, 1.0, 0.0, + 1.0, 1.0, 1.0, + 0.0, 1.0, 0.0 + ] + }, + 'dilation': { + size: 3, + kernel: [ + 1.0, 1.0, 1.0, + 1.0, 0.0, 1.0, + 1.0, 1.0, 1.0 + ] + }, + 'unsharpMask': { + size: 5, + kernel: [ + -1.0/256.0, -4.0/256.0, -6.0/256.0, -4.0/256.0, -1.0/256.0, + -4.0/256.0, 1.0/256.0, 13.0/256.0, 1.0/256.0, -4.0/256.0, + -6.0/256.0, 13.0/256.0, 269.0/256.0, 13.0/256.0, -6.0/256.0, + -4.0/256.0, 1.0/256.0, 13.0/256.0, 1.0/256.0, -4.0/256.0, + -1.0/256.0, -4.0/256.0, -6.0/256.0, -4.0/256.0, -1.0/256.0 + ] + } +} + + +const inputs = { + image: { + type: 'Image', + } as ImageVar, + preset: { + type: 'String', + default: 'sobel', + enum: Object.keys(kernels), + } as StringVar, + passes: { + type: 'Number', + default: 1, + } as NumberVar, + multiplier: { + type: 'Number', + default: 1, + } as NumberVar, +} + + +const outputs = { + image: { + type: 'Image', + } as ImageVar, + width: { + type: 'Number', + } as NumberVar, + height: { + type: 'Number', + } as NumberVar +} + +export default class KernelPreset extends WebGL { + + _lastKernelSize?: number; + + constructor() { + super('KernelPreset', inputs) + } + + _passes() { + + const passes = Array(Math.max(this.in.passes.value, 0)) + .fill(null) + .flatMap((x, i) => { + return [ + (gl, program) => { + if (i === 0) { + this._setParams(gl, program) + } + } + ] + }); + return passes; + } + + get currentPreset() { + return kernels[this.in.preset.value]; + } + + setup() { + if (this.currentPreset.size !== this._lastKernelSize) { + this.program = undefined; + } + super.setup(); + this._lastKernelSize = this.currentPreset.size; + } + + get frag() { + const KERNEL_SIZE = Math.ceil(this.currentPreset.size); + const shader = ` + precision mediump float; + + // our texture + uniform sampler2D u_image; + uniform vec2 u_textureSize; + uniform float u_kernel[${KERNEL_SIZE * KERNEL_SIZE}]; + + // the texCoords passed in from the vertex shader. + varying vec2 v_texCoord; + + const int RADIUS = ${KERNEL_SIZE}; + + vec4 applyKernel(in vec2 texCoord) { + vec2 onePixel = vec2(1.0, 1.0) / u_textureSize; + vec4 colorSum = vec4(0.0); + + for (int y = 0; y < RADIUS; y++) { + for (int x = 0; x < RADIUS; x++) { + vec2 offset = vec2(float(x - RADIUS/2), float(y - RADIUS/2)); + colorSum += texture2D(u_image, texCoord + onePixel * offset) * u_kernel[y * RADIUS + x]; + } + } + + return colorSum; + } + + void main() { + gl_FragColor = applyKernel(v_texCoord); + } + `; + console.log(shader); + return shader; + } + + + /** + * + * @param {WebGLRenderingContext} gl + * @param {*} program + */ + _setParams(gl, program) { + const kernel = this.currentPreset.kernel; + if (kernel) { + gl.uniform1fv( + gl.getUniformLocation(program, 'u_kernel'), + kernel.map((x) => x * this.in.multiplier.value) + ); + } + } +} \ No newline at end of file diff --git a/src/nodes/index.ts b/src/nodes/index.ts index 4f31713..6e94166 100644 --- a/src/nodes/index.ts +++ b/src/nodes/index.ts @@ -18,6 +18,7 @@ export {default as ColorLookupTable} from './WebGL/ColorLookupTable'; export {default as Vignette} from './WebGL/Vignette'; export {default as Blur} from './WebGL/Blur'; export {default as Edge} from './WebGL/Edge'; +export {default as KernelPreset} from './WebGL/KernelPreset'; // Math export {default as Numbers} from './Math/Numbers'; export {default as NumberBinaryOperation} from './Math/NumberBinaryOperation';