gemma/graphfx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ blur and body segmentation tweaks, adding kernel presets

Browse Source
This commit is contained in:
Stanislav Fifik
2023-03-24 14:48:44 +01:00
parent ccbd3e4365
commit 3fd8f610fe
6 changed files with 234 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
src/nodes/AI/BodySegmentation.ts
View File

@@ -15,28 +15,17 @@ import {
} from '@tensorflow-models/body-segmentation'
import {canvasPool2D} from "../../canvas/CanvasPool";
import {waitForMedia} from "../../utils";
import { mediaSize } from '../canvas';
const inputs = {
image: {
type: 'Image'
} as ImageVar,
width: {
type: 'Number',
default: 100,
step: 1,
min: 1
} as NumberVar,
height: {
type: 'Number',
default: 100,
step: 1,
min: 1,
} as NumberVar,
// blur mask edges by number of pixels
maskBlurAmount: {
type: 'Number',
default: 0,
step: 1,
default: 1,
step: 0.1,
} as NumberVar,
// Default to 0.5. The minimum probability to color a pixel as foreground rather than background. The alpha channel integer values will be taken as the probabilities (for more information refer to Segmentation type's documentation).
foregroundThreshold: {
@@ -70,29 +59,6 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
super('BodySegmentation', inputs, outputs);
}
get width() {
return this.__in.width.value;
}
get height() {
return this.__in.height.value;
}
get image() {
if (!this.__in.image.value) {
return null;
} else {
const {width, height} = this.__in.image.value;
return {
image: this.__in.image.value,
top: 0,
left: 0,
width,
height,
};
}
}
async initSegmenter() {
if (this.segmenter) {
return;
@@ -108,13 +74,14 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
async segmentPeople() {
await this.initSegmenter();
await waitForMedia(this.in.image.value);
const {width, height} = mediaSize(this.__in.image.value);
if (!this.in.image.value) {
return;
}
const originalImageCanvas = canvasPool2D.createCanvas();
originalImageCanvas.width = this.in.width.value;
originalImageCanvas.height = this.in.height.value
originalImageCanvas.width = width;
originalImageCanvas.height = height;
originalImageCanvas.acquire();
const originalCanvasCtx = originalImageCanvas.getContext('2d');
@@ -137,16 +104,15 @@ export default class BodySegmentation extends Node<typeof inputs, typeof outputs
const canvas = canvasPool2D.createCanvas();
canvas.acquire();
const ctx = canvas.getContext('2d');
canvas.width = this.in.width.value;
canvas.height = this.in.height.value
canvas.width = width;
canvas.height = height;
const maskBlurAmount = this.in.maskBlurAmount.value;
await drawMask(canvas, canvas as CanvasImageSource, backgroundDarkeningMask, 1, maskBlurAmount);
this.out.image.value = canvas;
this.out.originalImage.value = originalImageCanvas;
this.out.width.value = this.in.width.value;
this.out.height.value = this.in.height.value;
this.out.width.value = width;
this.out.height.value = height;
}
async _update(){

2
src/nodes/Canvas2d/Flip.ts
View File

@@ -1,7 +1,7 @@
import Canvas2d from './Canvas2d';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
import {
BooleanVar
BooleanVar, ImageVar
} from '../io/AbstractIOSet';
const inputs = {

8
src/nodes/WebGL/Blur.ts
View File

@@ -10,11 +10,14 @@ const inputs = {
} as ImageVar,
radius: {
type: 'Number',
default: 0,
default: 8,
min: 0,
} as NumberVar,
sigma: {
type: 'Number',
default: 0,
default: 1,
step: 0.1,
min: 0,
} as NumberVar,
passes: {
type: 'Number',
@@ -105,7 +108,6 @@ export default class Blur extends WebGL<typeof inputs> {
}
gl_FragColor = colorSum / totalWeight;
gl_FragColor.a = 1.0;
}
`
}

216
src/nodes/WebGL/KernelPreset.ts Normal file
View File

@@ -0,0 +1,216 @@
import WebGL from './WebGl';
import {
ImageVar,
NumberVar,
StringVar
} from '../io/AbstractIOSet';
const kernels = {
'sobel': {
size: 3,
kernel: [
-1.0, -2.0, -1.0,
0.0, 0.0, 0.0,
1.0, 2.0, 1.0
]
},
'laplacian': {
size: 3,
kernel: [
0.0, 1.0, 0.0,
1.0, -4.0, 1.0,
0.0, 1.0, 0.0
]
},
'laplacianOfGaussian': {
size: 5,
kernel: [
0.0, 0.0, -1.0, 0.0, 0.0,
0.0, -1.0, -2.0, -1.0, 0.0,
-1.0, -2.0, 16.0, -2.0, -1.0,
0.0, -1.0, -2.0, -1.0, 0.0,
0.0, 0.0, -1.0, 0.0, 0.0
]
},
'emboss': {
size: 3,
kernel: [
-2.0, -1.0, 0.0,
-1.0, 1.0, 1.0,
0.0, 1.0, 2.0
]
},
'sharpen': {
size: 3,
kernel: [
0.0, -1.0, 0.0,
-1.0, 5.0, -1.0,
0.0, -1.0, 0.0
]
},
'boxBlur': {
size: 3,
kernel: [
1.0/9.0, 1.0/9.0, 1.0/9.0,
1.0/9.0, 1.0/9.0, 1.0/9.0,
1.0/9.0, 1.0/9.0, 1.0/9.0
]
},
'outline': {
size: 3,
kernel: [
-2.0, -1.0, 0.0,
-1.0, 1.0, 1.0,
0.0, 1.0, 2.0
]
},
'erosion': {
size: 3,
kernel: [
0.0, 1.0, 0.0,
1.0, 1.0, 1.0,
0.0, 1.0, 0.0
]
},
'dilation': {
size: 3,
kernel: [
1.0, 1.0, 1.0,
1.0, 0.0, 1.0,
1.0, 1.0, 1.0
]
},
'unsharpMask': {
size: 5,
kernel: [
-1.0/256.0, -4.0/256.0, -6.0/256.0, -4.0/256.0, -1.0/256.0,
-4.0/256.0, 1.0/256.0, 13.0/256.0, 1.0/256.0, -4.0/256.0,
-6.0/256.0, 13.0/256.0, 269.0/256.0, 13.0/256.0, -6.0/256.0,
-4.0/256.0, 1.0/256.0, 13.0/256.0, 1.0/256.0, -4.0/256.0,
-1.0/256.0, -4.0/256.0, -6.0/256.0, -4.0/256.0, -1.0/256.0
]
}
}
const inputs = {
image: {
type: 'Image',
} as ImageVar,
preset: {
type: 'String',
default: 'sobel',
enum: Object.keys(kernels),
} as StringVar,
passes: {
type: 'Number',
default: 1,
} as NumberVar,
multiplier: {
type: 'Number',
default: 1,
} as NumberVar,
}
const outputs = {
image: {
type: 'Image',
} as ImageVar,
width: {
type: 'Number',
} as NumberVar,
height: {
type: 'Number',
} as NumberVar
}
export default class KernelPreset extends WebGL<typeof inputs> {
_lastKernelSize?: number;
constructor() {
super('KernelPreset', inputs)
}
_passes() {
const passes = Array(Math.max(this.in.passes.value, 0))
.fill(null)
.flatMap((x, i) => {
return [
(gl, program) => {
if (i === 0) {
this._setParams(gl, program)
}
}
]
});
return passes;
}
get currentPreset() {
return kernels[this.in.preset.value];
}
setup() {
if (this.currentPreset.size !== this._lastKernelSize) {
this.program = undefined;
}
super.setup();
this._lastKernelSize = this.currentPreset.size;
}
get frag() {
const KERNEL_SIZE = Math.ceil(this.currentPreset.size);
const shader = `
precision mediump float;
// our texture
uniform sampler2D u_image;
uniform vec2 u_textureSize;
uniform float u_kernel[${KERNEL_SIZE * KERNEL_SIZE}];
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
const int RADIUS = ${KERNEL_SIZE};
vec4 applyKernel(in vec2 texCoord) {
vec2 onePixel = vec2(1.0, 1.0) / u_textureSize;
vec4 colorSum = vec4(0.0);
for (int y = 0; y < RADIUS; y++) {
for (int x = 0; x < RADIUS; x++) {
vec2 offset = vec2(float(x - RADIUS/2), float(y - RADIUS/2));
colorSum += texture2D(u_image, texCoord + onePixel * offset) * u_kernel[y * RADIUS + x];
}
}
return colorSum;
}
void main() {
gl_FragColor = applyKernel(v_texCoord);
}
`;
console.log(shader);
return shader;
}
/**
*
* @param {WebGLRenderingContext} gl
* @param {*} program
*/
_setParams(gl, program) {
const kernel = this.currentPreset.kernel;
if (kernel) {
gl.uniform1fv(
gl.getUniformLocation(program, 'u_kernel'),
kernel.map((x) => x * this.in.multiplier.value)
);
}
}
}

1
src/nodes/index.ts
View File

@@ -18,6 +18,7 @@ export {default as ColorLookupTable} from './WebGL/ColorLookupTable';
export {default as Vignette} from './WebGL/Vignette';
export {default as Blur} from './WebGL/Blur';
export {default as Edge} from './WebGL/Edge';
export {default as KernelPreset} from './WebGL/KernelPreset';
// Math
export {default as Numbers} from './Math/Numbers';
export {default as NumberBinaryOperation} from './Math/NumberBinaryOperation';