From 6394f2a869d0ab6fffb6ca38794b68009c94a4d8 Mon Sep 17 00:00:00 2001 From: Stanislav Fifik Date: Mon, 21 Jan 2019 20:39:56 +0100 Subject: [PATCH] :sparkles: basic ui, some filters --- package-lock.json | 8 +- package.json | 15 ++- src/nodes/Compose.js | 4 +- src/nodes/ImageIdentity.js | 2 +- src/nodes/Node.js | 37 +++++- src/nodes/Resize.js | 5 +- src/nodes/ToBlob.js | 2 +- src/nodes/WebGL/BrightnessContrast.js | 2 +- src/nodes/WebGL/Greenscreen.js | 106 +++++++++++++++++ src/nodes/WebGL/HSV.js | 4 +- src/nodes/WebGL/Sepia.js | 4 +- src/nodes/WebGL/WebGl.js | 59 ++++++---- src/nodes/Webcam.js | 41 +++++++ src/nodes/index.js | 4 +- src/nodes/io.js | 51 +++++++- ui/Components/App.vue | 77 ++++++++++++ ui/Components/ContextMenu.vue | 68 +++++++++++ ui/Components/Drag/Draggable.js | 24 ++++ ui/Components/Graph.vue | 162 +++++++++++++++++++++++++- ui/Components/Input.vue | 34 ++++++ ui/Components/Node.vue | 135 ++++++++++++++++++--- ui/Components/Value/Image.vue | 100 +++++++++++----- ui/Components/Value/Select.vue | 2 +- ui/Components/Value/String.vue | 13 ++- ui/Components/Value/index.js | 1 + ui/defaultGraph.js | 44 +++++++ ui/graph.js | 11 +- ui/index.js | 8 +- ui/persistent.js | 60 ++++++++++ webpack.config.js | 2 +- 30 files changed, 974 insertions(+), 111 deletions(-) create mode 100644 src/nodes/WebGL/Greenscreen.js create mode 100644 src/nodes/Webcam.js create mode 100644 ui/Components/App.vue create mode 100644 ui/Components/ContextMenu.vue create mode 100644 ui/Components/Drag/Draggable.js create mode 100644 ui/Components/Input.vue create mode 100644 ui/defaultGraph.js create mode 100644 ui/persistent.js diff --git a/package-lock.json b/package-lock.json index 52e3cd7..a35d2b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5095,6 +5095,11 @@ "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true }, + "interactjs": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/interactjs/-/interactjs-1.3.4.tgz", + "integrity": "sha512-AQ2CdPEyHqiEEQ1FFgMBj79UEsU1+rUwSXuhOkflvB65p4iECft28SN/PvhD/Y9OtNge8aH1qTibjAi+RXQMqQ==" + }, "internal-ip": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-3.0.1.tgz", @@ -9851,8 +9856,7 @@ "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "dev": true + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, "v8-compile-cache": { "version": "2.0.2", diff --git a/package.json b/package.json index b953c4e..f43264e 100644 --- a/package.json +++ b/package.json @@ -16,21 +16,24 @@ "@types/jest": "^23.3.10", "@types/webpack": "^4.4.22", "babel-jest": "^23.6.0", - "canvas": "^2.2.0", - "jest": "^23.6.0", - "webpack": "^4.28.3", - "webpack-cli": "^3.1.2", - "webpack-dev-server": "^3.1.14", "babel-loader": "^8.0.4", "babel-polyfill": "^6.26.0", + "canvas": "^2.2.0", "css-loader": "^1.0.1", "file-loader": "^2.0.0", "friendly-errors-webpack-plugin": "^1.7.0", "html-webpack-plugin": "^3.2.0", + "jest": "^23.6.0", "vue-loader": "^15.4.2", - "vue-template-compiler": "^2.5.17" + "vue-style-loader": "^4.1.2", + "vue-template-compiler": "^2.5.17", + "webpack": "^4.28.3", + "webpack-cli": "^3.1.2", + "webpack-dev-server": "^3.1.14" }, "dependencies": { + "interactjs": "^1.3.4", + "uuid": "^3.3.2", "vue": "^2.5.21", "webgl-utils": "^1.0.1" } diff --git a/src/nodes/Compose.js b/src/nodes/Compose.js index 4ff3797..88fc42b 100644 --- a/src/nodes/Compose.js +++ b/src/nodes/Compose.js @@ -1,10 +1,10 @@ import Node from './Node'; import {createCanvas, mediaSize, paintToCanvas} from './canvas'; -export default class Resize extends Node { +export default class Compose extends Node { constructor(options) { - super({ + super('Compose', { fg: 'Image', fgX: 'Number', fgY: 'Number', diff --git a/src/nodes/ImageIdentity.js b/src/nodes/ImageIdentity.js index 2abc7dc..53cba48 100644 --- a/src/nodes/ImageIdentity.js +++ b/src/nodes/ImageIdentity.js @@ -2,7 +2,7 @@ import Node from './Node'; export default class ImageIdentity extends Node { constructor() { - super({ + super('ImageIdentity', { image: 'Image', }, { image: 'Image', diff --git a/src/nodes/Node.js b/src/nodes/Node.js index 5414c28..c510aeb 100644 --- a/src/nodes/Node.js +++ b/src/nodes/Node.js @@ -1,10 +1,13 @@ import {Inputs, Outputs} from './io.js'; +import uuidv4 from 'uuid/v4'; export default class Node { - constructor(inputNames, outputNames) { - this.__in = new Inputs(inputNames); - this.__out = new Outputs(outputNames); + constructor(name, inputNames, outputNames) { + this.name = name; + this.id = uuidv4(); + this.__in = new Inputs(inputNames, this); + this.__out = new Outputs(outputNames, this); this.__in.update = (name) => this.__update([name]); } @@ -25,5 +28,33 @@ export default class Node { get out() { return this.__out; } + + serialize() { + return { + id: this.id, + name: this.name, + options: { + in: this.in.serialize(), + }, + } + } + + deserialize({id, options}) { + this.id = id || uuidv4(); + } + + reconnect({options}, outputs) { + console.log('outputs', outputs) + for (let name of Object.keys(options.in)) { + const {value, output} = options.in[name]; + if (value) { + this.in[name].value = value; + } + if (output) { + console.log('connecting', this.in[name].id, outputs[output]) + this.in[name].connect(outputs[output]); + } + } + } }; diff --git a/src/nodes/Resize.js b/src/nodes/Resize.js index 1d89af9..d2feb51 100644 --- a/src/nodes/Resize.js +++ b/src/nodes/Resize.js @@ -4,7 +4,7 @@ import {createCanvas, mediaSize, paintToCanvas} from './canvas'; export default class Resize extends Node { constructor(options) { - super({ + super('Resize', { image: 'Image', width: 'Number', height: 'Number', @@ -24,6 +24,9 @@ export default class Resize extends Node { __update() { const media = this.__in.image.value; + if (!media) { + return; + } const canvas = createCanvas(this.width, this.height); const {width: srcWidth, height: srcHeight} = mediaSize(media); diff --git a/src/nodes/ToBlob.js b/src/nodes/ToBlob.js index 042f6fe..9bde6bd 100644 --- a/src/nodes/ToBlob.js +++ b/src/nodes/ToBlob.js @@ -4,7 +4,7 @@ import {createCanvas} from './canvas'; export default class ToBlob extends Node { constructor(options) { - super({ + super('ToBlob', { image: 'Image', }, { image: 'Image', diff --git a/src/nodes/WebGL/BrightnessContrast.js b/src/nodes/WebGL/BrightnessContrast.js index 416addb..0213ffc 100644 --- a/src/nodes/WebGL/BrightnessContrast.js +++ b/src/nodes/WebGL/BrightnessContrast.js @@ -4,7 +4,7 @@ import WebGL from './WebGl'; export default class BrightnessContrast extends WebGL { constructor() { - super({ + super('BrightnessContrast', { brightness: 'Number', contrast: 'Number', }) diff --git a/src/nodes/WebGL/Greenscreen.js b/src/nodes/WebGL/Greenscreen.js new file mode 100644 index 0000000..9fb9536 --- /dev/null +++ b/src/nodes/WebGL/Greenscreen.js @@ -0,0 +1,106 @@ +import WebGL from './WebGl'; + +const hexColorTOvec3 = (val) => { + const match = val.match(/#?([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})/); + if (match) { + return [parseInt(match[1], 16) / 255, parseInt(match[2], 16) / 255, parseInt(match[3], 16) / 255]; + } else { + return null; + } +}; + +export default class GreenScreen extends WebGL { + + constructor() { + super('GreenScreen', { + balance: 'Number', + screen: 'Color', + screenWeight: 'Number', + clipBlack: 'Number', + clipWhite: 'Number', + }) + } + + get frag() { + return ` + precision mediump float; + + // our texture + uniform sampler2D u_image; + + // the texCoords passed in from the vertex shader. + varying vec2 v_texCoord; + + uniform float balance; + + uniform vec3 screen; + + uniform float clipBlack; + uniform float clipWhite; + uniform float screenWeight; + + void main() { + float pixelSat, secondaryComponents; + vec4 sourcePixel = texture2D(u_image, v_texCoord); + float fmin = min(min(sourcePixel.r, sourcePixel.g), sourcePixel.b); + float fmax = max(max(sourcePixel.r, sourcePixel.g), sourcePixel.b); + vec3 pixelPrimary = step(fmax, sourcePixel.rgb); + secondaryComponents = dot(1.0 - pixelPrimary, sourcePixel.rgb); + + // luminance = fmax + float screenFmin = min(min(screen.r, screen.g), screen.b); //Min. value of RGB + float screenFmax = max(max(screen.r, screen.g), screen.b); //Max. value of RGB + vec3 screenPrimary = step(screenFmax, screen.rgb); + float screenSecondaryComponents = dot(1.0 - screenPrimary, screen.rgb); + float screenSat = fmax - mix(secondaryComponents - fmin, secondaryComponents / 2.0, balance); + + pixelSat = fmax - mix(secondaryComponents - fmin, secondaryComponents / 2.0, balance); + + // solid pixel if primary color component is not the same as the screen color + float diffPrimary = dot(abs(pixelPrimary - screenPrimary), vec3(1.0)); + float solid = step(1.0, step(pixelSat, 0.1) + step(fmax, 0.1) + diffPrimary); + + /* + Semi-transparent pixel if the primary component matches but if saturation is less + than that of screen color. Otherwise totally transparent + */ + float alpha = max(0.0, 1.0 - pixelSat / screenSat); + alpha = smoothstep(clipBlack, clipWhite, alpha); + vec4 semiTransparentPixel = vec4((sourcePixel.rgb - (1.0 - alpha) * screen.rgb * screenWeight) / max(0.00001, alpha), alpha); + vec4 pixel = mix(semiTransparentPixel, sourcePixel, solid); + // vec4 pixel = vec4(sourcePixel.rgb, (1.0 - alpha)); + + gl_FragColor = vec4(pixel.rgb * pixel.a, pixel.a); + //gl_FragColor = vec4(min(1.0, max(pixel.r, 0.0)), min(1.0, max(pixel.g, 0.0)), min(1.0, max(pixel.b, 0.0)), min(1.0, max(pixel.a, 0.0))); + } + ` + } + + get clipBlack() { + return (this.in.clipBlack.value || 0); + } + + get clipWhite() { + return (this.in.clipWhite.value || 0); + } + + get screenWeight() { + return (this.in.screenWeight.value || 1); + } + + get screen() { + return hexColorTOvec3(this.in.screen.value || '#00FF00'); + } + + get balance() { + return (this.in.balance.value || 0.5); + } + + _setParams(gl, program) { + gl.uniform3f(gl.getUniformLocation(program, 'screen'), ...this.screen); + gl.uniform1f(gl.getUniformLocation(program, 'balance'), this.balance); + gl.uniform1f(gl.getUniformLocation(program, 'clipBlack'), this.clipBlack); + gl.uniform1f(gl.getUniformLocation(program, 'clipWhite'), this.clipWhite); + gl.uniform1f(gl.getUniformLocation(program, 'screenWeight'), this.screenWeight); + } +} \ No newline at end of file diff --git a/src/nodes/WebGL/HSV.js b/src/nodes/WebGL/HSV.js index b27c23a..2b234ef 100644 --- a/src/nodes/WebGL/HSV.js +++ b/src/nodes/WebGL/HSV.js @@ -1,10 +1,10 @@ import WebGL from './WebGl'; -export default class BrightnessContrast extends WebGL { +export default class HSV extends WebGL { constructor() { - super({ + super('HSV', { hue: 'Number', saturation: 'Number', value: 'Number', diff --git a/src/nodes/WebGL/Sepia.js b/src/nodes/WebGL/Sepia.js index 376d051..1e765c2 100644 --- a/src/nodes/WebGL/Sepia.js +++ b/src/nodes/WebGL/Sepia.js @@ -1,10 +1,10 @@ import WebGL from './WebGl'; -export default class BrightnessContrast extends WebGL { +export default class Sepia extends WebGL { constructor() { - super({ + super('Sepia', { amount: 'Number', }) } diff --git a/src/nodes/WebGL/WebGl.js b/src/nodes/WebGL/WebGl.js index 673afa1..3d6dfbd 100644 --- a/src/nodes/WebGL/WebGl.js +++ b/src/nodes/WebGL/WebGl.js @@ -6,8 +6,8 @@ console.log(webglUtils); export default class WebGL extends Node { - constructor(inputs={}) { - super(Object.assign({ + constructor(name, inputs={}) { + super(name, Object.assign({ image: 'Image', }, inputs), { image: 'Image' @@ -86,7 +86,7 @@ export default class WebGL extends Node { * * @param {WebGLRenderingContext} gl */ - program(gl) { + compileProgram(gl) { const program = gl.createProgram(); gl.attachShader(program, this.vertShader(gl)); @@ -115,10 +115,7 @@ export default class WebGL extends Node { gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - if (this.image) { - // Upload the image into the texture. - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.image); - } + return texture; } @@ -126,22 +123,46 @@ export default class WebGL extends Node { } + setup() { + if (!this.canvas) { + this.program = null; + this.canvas = createCanvas(0, 0); + } + const gl = this.canvas.getContext('webgl'); + if (!this.program) { + this.program = this.compileProgram(gl); + gl.useProgram(this.program); + this.createTexture(gl); + } + } + __update() { const image = this.in.image.value; if (!image) return; const {width, height} = mediaSize(image); - const canvas = createCanvas(width, height); + this.setup(); + const canvas = this.canvas; + canvas.width = width; + canvas.height = height; + const gl = canvas.getContext('webgl'); - const program = this.program(gl); + const program = this.program; + + gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + const positionLocation = gl.getAttribLocation(program, "a_position"); const texcoordLocation = gl.getAttribLocation(program, "a_texCoord"); const resolutionLocation = gl.getUniformLocation(program, "u_resolution"); - const positionBuffer = gl.createBuffer(); + if (this.image) { + // Upload the image into the texture. + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.image); + } + - gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); - this.setRectangle(gl, 0, 0, width, height); const texcoordBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([ @@ -153,17 +174,12 @@ export default class WebGL extends Node { 1.0, 1.0, ]), gl.STATIC_DRAW); - this.createTexture(gl); - - gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); - gl.clearColor(0, 0, 0, 0); - gl.clear(gl.COLOR_BUFFER_BIT); - - gl.useProgram(program); gl.enableVertexAttribArray(positionLocation); + const positionBuffer = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + this.setRectangle(gl, 0, 0, width, height); // Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER) var size = 2; // 2 components per iteration @@ -195,10 +211,7 @@ export default class WebGL extends Node { this._setParams(gl, program); // Draw the rectangle. - var primitiveType = gl.TRIANGLES; - var offset = 0; - var count = 6; - gl.drawArrays(primitiveType, offset, count); + gl.drawArrays(gl.TRIANGLES, 0, 6); this.out.image.value = canvas; } diff --git a/src/nodes/Webcam.js b/src/nodes/Webcam.js new file mode 100644 index 0000000..dcfc0eb --- /dev/null +++ b/src/nodes/Webcam.js @@ -0,0 +1,41 @@ +import Node from './Node'; + +export default class ToBlob extends Node { + + constructor(options) { + super('Webcam', {}, { + image: 'Image', + }); + this.start(); + } + + + async start() { + const devices = (await navigator.mediaDevices.enumerateDevices()).filter((d) => d.kind === 'videoinput') + const last = (ary) => ary[ary.length - 1]; + console.log(last(devices)); + const stream = await navigator.mediaDevices.getUserMedia({ + video: { + width: { min: 1024, ideal: 1280, max: 1920 }, + height: { min: 776, ideal: 720, max: 1080 }, + deviceId: last(devices).deviceId + }, + }); + console.log(stream) + /** @type {HTMLVideoElement} */ + + const video = document.createElement('video'); + video.playsinline = true; + video.srcObject = stream; + video.play(); + let lastFrameTime = null; + const feedLoop = () => { + if (lastFrameTime !== video.currentTime) { + lastFrameTime = video.currentTime; + this.out.image.value = video; + } + requestAnimationFrame(feedLoop); + }; + feedLoop(); + } +} \ No newline at end of file diff --git a/src/nodes/index.js b/src/nodes/index.js index 213d2fa..794584a 100644 --- a/src/nodes/index.js +++ b/src/nodes/index.js @@ -1,3 +1,4 @@ +export {default as Webcam} from './Webcam'; export {default as Resize} from './Resize'; export {default as ImageIdentity} from './ImageIdentity'; export {default as ToBlob} from './ToBlob'; @@ -5,4 +6,5 @@ export {default as Compose} from './Compose'; export {default as WebGL} from './WebGL/WebGl'; export {default as BrightnessContrast} from './WebGL/BrightnessContrast'; export {default as HSV} from './WebGL/HSV'; -export {default as Sepia} from './WebGL/Sepia'; \ No newline at end of file +export {default as Sepia} from './WebGL/Sepia'; +export {default as GreenScreen} from './WebGL/Greenscreen'; \ No newline at end of file diff --git a/src/nodes/io.js b/src/nodes/io.js index fd5a72a..0ef0a4f 100644 --- a/src/nodes/io.js +++ b/src/nodes/io.js @@ -15,6 +15,10 @@ export class Input { } } + get id() { + return `${this.__owner.id}-${this.__name}`; + } + get name() { return this.__name; } @@ -29,12 +33,20 @@ export class Input { } connect(output) { + this.disconnect(); this.__output = output this.__output.onchange(this.__onchangelistener); } disconnect(output) { - this.__output.offchage(this.__onchangelistener); + if (this.__output) { + this.__output.offchange(this.__onchangelistener); + this.__output = null; + } + } + + get output() { + return this.__output; } __notifyListeners() { @@ -51,6 +63,13 @@ export class Input { this.__listeners = this.__listeners .filter((l) => l !== listener); } + + serialize() { + return { + value: this.__output ? null : this.__value, + output: this.__output ? this.__output.id: null, + } + } } export class Output { @@ -62,6 +81,10 @@ export class Output { this.__listeners = []; } + get id() { + return `${this.__owner.id}-${this.__name}`; + } + get name() { return this.__name; } @@ -94,8 +117,9 @@ export class Output { export class Inputs { - constructor(variables) { + constructor(variables, owner) { this.__values = {}; + this.__owner = owner; this.variables = variables; for (let name of Object.keys(this.variables)) { @@ -108,14 +132,27 @@ export class Inputs { } } + get id() { + return `${this.__owner.id}-in`; + } + update(changed) { } + + serialize() { + const res = {}; + for (let name of Object.keys(this.variables)) { + res[name] = this.__values[name].serialize(); + } + return res; + } } export class Outputs { - constructor(variables) { + constructor(variables, owner) { this.__values = {}; + this.__owner = owner; this.variables = variables; for (let name of Object.keys(this.variables)) { @@ -127,4 +164,12 @@ export class Outputs { }) } } + + get id() { + return `${this.__owner.id}-out`; + } + + serialize() { + return null; + } } \ No newline at end of file diff --git a/ui/Components/App.vue b/ui/Components/App.vue new file mode 100644 index 0000000..b4e5638 --- /dev/null +++ b/ui/Components/App.vue @@ -0,0 +1,77 @@ + + + diff --git a/ui/Components/ContextMenu.vue b/ui/Components/ContextMenu.vue new file mode 100644 index 0000000..7e7074a --- /dev/null +++ b/ui/Components/ContextMenu.vue @@ -0,0 +1,68 @@ + + + diff --git a/ui/Components/Drag/Draggable.js b/ui/Components/Drag/Draggable.js new file mode 100644 index 0000000..884042d --- /dev/null +++ b/ui/Components/Drag/Draggable.js @@ -0,0 +1,24 @@ +import interact from 'interactjs'; + +export default { + abstract: true, + render() { + try { + return this.$slots.default[0]; + } catch (e) { + throw new Error('Exactly one child component needed.'); + } + return null; + }, + mounted () { + const el = this.$slots.default[0].elm; + this.$nextTick(() => { + interact(el) + .draggable({ + onmove: (event) => { + this.$emit('move', event); + } + }) + }) + } +} \ No newline at end of file diff --git a/ui/Components/Graph.vue b/ui/Components/Graph.vue index f537533..49ab0ea 100644 --- a/ui/Components/Graph.vue +++ b/ui/Components/Graph.vue @@ -1,17 +1,47 @@ + + diff --git a/ui/Components/Input.vue b/ui/Components/Input.vue new file mode 100644 index 0000000..d616815 --- /dev/null +++ b/ui/Components/Input.vue @@ -0,0 +1,34 @@ + + diff --git a/ui/Components/Node.vue b/ui/Components/Node.vue index 03916f1..15137c5 100644 --- a/ui/Components/Node.vue +++ b/ui/Components/Node.vue @@ -1,25 +1,52 @@