diff --git a/package-lock.json b/package-lock.json index a35d2b2..1efc5b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6185,8 +6185,7 @@ "lodash": { "version": "4.17.11", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", - "dev": true + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "lodash.debounce": { "version": "4.0.8", diff --git a/package.json b/package.json index f43264e..05f0c80 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "interactjs": "^1.3.4", + "lodash": "^4.17.11", "uuid": "^3.3.2", "vue": "^2.5.21", "webgl-utils": "^1.0.1" diff --git a/src/nodes/Canvas2d/Canvas2d.js b/src/nodes/Canvas2d/Canvas2d.js new file mode 100644 index 0000000..ee62776 --- /dev/null +++ b/src/nodes/Canvas2d/Canvas2d.js @@ -0,0 +1,44 @@ +import Node from '../Node'; +import {waitForMedia} from '../../utils'; + +export default class Canvas2d extends Node { + constructor(name, inputDefinition, outputDefiniton, options) { + super(name, + Object.assign({ + image: { + type: 'Image', + }, + }, inputDefinition), + Object.assign({ + image: { + type: 'Image', + } + }, outputDefiniton), + options + ); + this.__canvas = document.createElement('canvas'); + } + + async __update() { + const values = {}; + for (let name of Object.keys(this.in.variables)) { + values[name] = await waitForMedia(this.in[name].value); + } + const ctx = this.__canvas.getContext('2d'); + if (this.__canvas.width > 0 && this.__canvas.height > 0) { + ctx.clearRect(0,0,this.__canvas.width, this.__canvas.height); + } + await this.render(values, this.__canvas, ctx); + } + + + /** + * + * @param {any} values + * @param {HTMLCanvasElement} canvas + * @param {CanvasRenderingContext2D} ctx + */ + async render(values, canvas, ctx) { + + } +} \ No newline at end of file diff --git a/src/nodes/Compose.js b/src/nodes/Canvas2d/Compose.js similarity index 83% rename from src/nodes/Compose.js rename to src/nodes/Canvas2d/Compose.js index bcaa969..730e0af 100644 --- a/src/nodes/Compose.js +++ b/src/nodes/Canvas2d/Compose.js @@ -1,10 +1,11 @@ -import Node from './Node'; -import {createCanvas, mediaSize, paintToCanvas} from './canvas'; +import Canvas2d from './Canvas2d'; +import {createCanvas, mediaSize, paintToCanvas} from '../canvas'; -export default class Compose extends Node { +export default class Compose extends Canvas2d { constructor(options) { super('Compose', { + image: null, fg: { type: 'Image' }, @@ -120,23 +121,30 @@ export default class Compose extends Node { } } - __update() { - if (!this.width || !this.height) { + /** + * @param {any} values + * @param {HTMLCanvasElement} canvas + * @param {CanvasRenderingContext2D} ctx + */ + async render({width, height, mode}, canvas, ctx) { + if (!width || !height) { return; } - const canvas = createCanvas(this.width, this.height); + canvas.width = width; + canvas.height = height; + + ctx.globalCompositeOperation = 'source-over'; if (this.bg) { paintToCanvas(canvas, this.bg.image, this.bg); } - const ctx = canvas.getContext('2d'); - ctx.globalCompositeOperation = this.mode; + ctx.globalCompositeOperation = mode; if (this.fg) { paintToCanvas(canvas, this.fg.image, this.fg); } - this.__out.image.value = canvas; + this.__out.image.value = await createImageBitmap(canvas); } } \ No newline at end of file diff --git a/src/nodes/Canvas2d/ImageIdentity.js b/src/nodes/Canvas2d/ImageIdentity.js new file mode 100644 index 0000000..6a91d0f --- /dev/null +++ b/src/nodes/Canvas2d/ImageIdentity.js @@ -0,0 +1,26 @@ +import Node from './Canvas2d'; +import {waitForMedia} from '../../utils'; +import {createCanvas, mediaSize, paintToCanvas} from '../canvas'; + + +export default class ImageIdentity extends Node { + constructor() { + super('ImageIdentity', { + }, { + }); + } + + async render({image}) { + if (!image) { + return; + } + const {width, height} = mediaSize(image); + const canvas = createCanvas(width, height); + paintToCanvas(canvas, image, {width, height, top: 0, left: 0}); + const dataUrl = canvas.toDataURL() + const i = new Image(); + i.src = dataUrl; + await waitForMedia(i); + this.__out.image.value = i; + } +} \ No newline at end of file diff --git a/src/nodes/Resize.js b/src/nodes/Canvas2d/Resize.js similarity index 62% rename from src/nodes/Resize.js rename to src/nodes/Canvas2d/Resize.js index 6a81245..789861f 100644 --- a/src/nodes/Resize.js +++ b/src/nodes/Canvas2d/Resize.js @@ -1,13 +1,11 @@ -import Node from './Node'; -import {createCanvas, mediaSize, paintToCanvas} from './canvas'; +import Canvas2d from './Canvas2d'; +import {waitForMedia} from '../../utils'; +import {createCanvas, mediaSize, paintToCanvas} from '../canvas'; -export default class Resize extends Node { +export default class Resize extends Canvas2d { - constructor(options) { + constructor() { super('Resize', { - image: { - type: 'Image', - }, width: { type: 'Number', default: 100, @@ -18,28 +16,26 @@ export default class Resize extends Node { default: 100, min: 1, }, - }, { - image: { - type: 'Image', - } - }); - this.options = options; + }, {}); } - get width() { - return this.__in.width.value; - } - - get height() { - return this.__in.height.value; - } - - __update() { - const media = this.__in.image.value; + /** + * @param {any} values + * @param {HTMLCanvasElement} canvas + * @param {CanvasRenderingContext2D} ctx + */ + async render({image: media, width, height}, canvas, ctx) { + await waitForMedia(media) if (!media) { return; } - const canvas = createCanvas(this.width, this.height); + + if (!width || !height) { + return; + } + + canvas.width = width; + canvas.height = height; const {width: srcWidth, height: srcHeight} = mediaSize(media); const {width: destWidth, height: destHeight} = mediaSize(canvas); @@ -58,6 +54,6 @@ export default class Resize extends Node { newImage.left = (destWidth - newImage.width) / 2; paintToCanvas(canvas, media, newImage); - this.__out.image.value = canvas; + this.__out.image.value = await createImageBitmap(canvas); } } \ No newline at end of file diff --git a/src/nodes/Canvas2d/Text.js b/src/nodes/Canvas2d/Text.js new file mode 100644 index 0000000..84dfeec --- /dev/null +++ b/src/nodes/Canvas2d/Text.js @@ -0,0 +1,49 @@ +import Canvas2d from './Canvas2d'; +import {waitForMedia} from '../../utils'; +import {createCanvas, mediaSize, paintToCanvas} from '../canvas'; + +export default class Resize extends Canvas2d { + + constructor() { + super('Text', { + image: null, + text: { + type: 'String', + default: '', + }, + fontSize: { + type: 'Number', + min: 1, + default: 50, + }, + font: { + type: 'String', + default: 'Arial', + }, + width: { + type: 'Number', + default: 100, + min: 1, + }, + height: { + type: 'Number', + default: 100, + min: 1, + }, + color: { + type: 'Color', + default: '#FFFFFF' + } + }, {}); + } + + async render({font, fontSize, text, color, width, height}, canvas, ctx) { + canvas.width = width; + canvas.height = height; + ctx.font = `${fontSize}px ${font}`; + ctx.fillStyle = color; + ctx.textAlign = 'center'; + ctx.fillText(text, canvas.width/2, canvas.height/2); + this.__out.image.value = await createImageBitmap(canvas); + } +} \ No newline at end of file diff --git a/src/nodes/Node.js b/src/nodes/Node.js index 7d649b4..6807ea0 100644 --- a/src/nodes/Node.js +++ b/src/nodes/Node.js @@ -46,15 +46,18 @@ export default class Node { reconnect({options}, outputs) { console.log('outputs', outputs) for (let name of Object.keys(options.in)) { - const {value, output} = options.in[name]; - if (value) { - this.in[name].value = value; - } + const {output} = options.in[name]; if (output) { console.log('connecting', this.in[name].id, outputs[output]) this.in[name].connect(outputs[output]); } } + for (let name of Object.keys(options.in)) { + const {value} = options.in[name]; + if (value) { + this.in[name].deserialize(value); + } + } } }; diff --git a/src/nodes/WebGL/Channels.js b/src/nodes/WebGL/Channels.js new file mode 100644 index 0000000..77829f8 --- /dev/null +++ b/src/nodes/WebGL/Channels.js @@ -0,0 +1,57 @@ +import WebGL from './WebGl'; + + +export default class Channels extends WebGL { + + constructor() { + super('Greyscale by channel', { + red: { + type: 'Number', + default: 1, + min: 0, + max: 1, + step: 0.1, + }, + green: { + type: 'Number', + default: 1, + min: 0, + max: 1, + step: 0.1, + }, + blue: { + type: 'Number', + default: 1, + min: 0, + max: 1, + step: 0.1, + } + }) + } + + get frag() { + return ` + precision mediump float; + + // our texture + uniform sampler2D u_image; + + // the texCoords passed in from the vertex shader. + varying vec2 v_texCoord; + + uniform vec3 u_weights; + + void main() { + vec4 pixelColor = texture2D(u_image, v_texCoord).rgba; + vec3 weights = u_weights / max((u_weights.r + u_weights.g + u_weights.b), 1.0); + float grey = dot(pixelColor.rgb, weights); + + gl_FragColor = vec4(grey, grey, grey, pixelColor.a); + } + ` + } + + _setParams(gl, program) { + gl.uniform3f(gl.getUniformLocation(program, "u_weights"), this.in.red.value, this.in.green.value, this.in.blue.value); + } +} \ No newline at end of file diff --git a/src/nodes/WebGL/Greenscreen.js b/src/nodes/WebGL/Greenscreen.js index fdc5a44..861ab63 100644 --- a/src/nodes/WebGL/Greenscreen.js +++ b/src/nodes/WebGL/Greenscreen.js @@ -89,7 +89,7 @@ export default class GreenScreen extends WebGL { vec4 pixel = mix(semiTransparentPixel, sourcePixel, solid); // vec4 pixel = vec4(sourcePixel.rgb, (1.0 - alpha)); - gl_FragColor = vec4(pixel.rgb * pixel.a, pixel.a); + gl_FragColor = vec4(pixel.rgb * pixel.a, min(max(pixel.a, 0.0), 1.0)); //gl_FragColor = vec4(min(1.0, max(pixel.r, 0.0)), min(1.0, max(pixel.g, 0.0)), min(1.0, max(pixel.b, 0.0)), min(1.0, max(pixel.a, 0.0))); } ` @@ -116,7 +116,6 @@ export default class GreenScreen extends WebGL { } _setParams(gl, program) { - console.log(this.balance, this.clipBlack, this.clipWhite, this.screenWeight) gl.uniform3f(gl.getUniformLocation(program, 'screen'), ...this.screen); gl.uniform1f(gl.getUniformLocation(program, 'balance'), this.balance); gl.uniform1f(gl.getUniformLocation(program, 'clipBlack'), this.clipBlack); diff --git a/src/nodes/WebGL/WebGl.js b/src/nodes/WebGL/WebGl.js index 0452405..06cc51f 100644 --- a/src/nodes/WebGL/WebGl.js +++ b/src/nodes/WebGL/WebGl.js @@ -2,8 +2,6 @@ import Node from '../Node'; import webglUtils from '../lib/webgl-utils'; import {createCanvas, mediaSize, paintToCanvas} from '../canvas'; -console.log(webglUtils); - export default class WebGL extends Node { constructor(name, inputs={}) { @@ -140,7 +138,7 @@ export default class WebGL extends Node { } } - __update() { + async __update() { const image = this.in.image.value; if (!image) return; const {width, height} = mediaSize(image); @@ -217,7 +215,7 @@ export default class WebGL extends Node { // Draw the rectangle. gl.drawArrays(gl.TRIANGLES, 0, 6); - this.out.image.value = canvas; + this.out.image.value = await createImageBitmap(canvas); } setRectangle(gl, x, y, width, height) { diff --git a/src/nodes/Webcam.js b/src/nodes/Webcam.js index 3dbd986..446f9dd 100644 --- a/src/nodes/Webcam.js +++ b/src/nodes/Webcam.js @@ -18,8 +18,8 @@ export default class ToBlob extends Node { console.log(last(devices)); const stream = await navigator.mediaDevices.getUserMedia({ video: { - width: { min: 1024, ideal: 1920, max: 1920 }, - height: { min: 776, ideal: 1080, max: 1080 }, + width: { min: 1024, ideal: 1280, max: 1920 }, + height: { min: 776, ideal: 720, max: 1080 }, deviceId: last(devices).deviceId }, }); @@ -32,8 +32,9 @@ export default class ToBlob extends Node { video.play(); let lastFrameTime = null; const feedLoop = () => { - if (lastFrameTime !== video.currentTime) { - lastFrameTime = video.currentTime; + const currentTime = video.currentTime; + if (lastFrameTime !== currentTime) { + lastFrameTime = currentTime; this.out.image.value = video; } requestAnimationFrame(feedLoop); diff --git a/src/nodes/index.js b/src/nodes/index.js index 794584a..ced9acb 100644 --- a/src/nodes/index.js +++ b/src/nodes/index.js @@ -1,10 +1,10 @@ export {default as Webcam} from './Webcam'; -export {default as Resize} from './Resize'; -export {default as ImageIdentity} from './ImageIdentity'; -export {default as ToBlob} from './ToBlob'; -export {default as Compose} from './Compose'; -export {default as WebGL} from './WebGL/WebGl'; +export {default as Resize} from './Canvas2d/Resize'; +export {default as Compose} from './Canvas2d/Compose'; +// export {default as ImageIdentity} from './Canvas2d/ImageIdentity'; + export {default as Text} from './Canvas2d/Text'; export {default as BrightnessContrast} from './WebGL/BrightnessContrast'; export {default as HSV} from './WebGL/HSV'; export {default as Sepia} from './WebGL/Sepia'; -export {default as GreenScreen} from './WebGL/Greenscreen'; \ No newline at end of file +export {default as GreenScreen} from './WebGL/Greenscreen'; +export {default as Channels} from './WebGL/Channels'; diff --git a/src/nodes/io/AbstractIO.js b/src/nodes/io/AbstractIO.js index 0020195..90841a7 100644 --- a/src/nodes/io/AbstractIO.js +++ b/src/nodes/io/AbstractIO.js @@ -1,4 +1,4 @@ -const isNil = (val) => val === null || val === undefined; +import {isNil} from '../../utils'; export const defaultConstrainSatisfied = (value) => { return !isNil(value); diff --git a/src/nodes/io/AbstractIOSet.js b/src/nodes/io/AbstractIOSet.js index 3b0be57..1ce7c12 100644 --- a/src/nodes/io/AbstractIOSet.js +++ b/src/nodes/io/AbstractIOSet.js @@ -8,6 +8,10 @@ export default class AbstractIOSet { this.variables = variables; for (let name of Object.keys(this.variables)) { + if (variables[name] === null) { + delete this.variables[name]; + continue; + } this.__values[name] = this.__createProperty(name, variables[name]); Object.defineProperty(this, name, { get() { diff --git a/src/nodes/io/Input.js b/src/nodes/io/Input.js index 223b3c3..e5b93bb 100644 --- a/src/nodes/io/Input.js +++ b/src/nodes/io/Input.js @@ -1,4 +1,5 @@ import AbstractIO from './AbstractIO'; +import {serialize, deserialize} from './serializer'; export default class Input extends AbstractIO { @@ -19,6 +20,7 @@ export default class Input extends AbstractIO { set value(value) { this.__setValue(value); this.__owner.update(this.name); + this.__notifyListeners(); } get output() { @@ -30,6 +32,7 @@ export default class Input extends AbstractIO { this.disconnect(); this.__output = output this.__output.onchange(this.__onchangelistener); + this.value = this.__output.value; } } @@ -42,8 +45,15 @@ export default class Input extends AbstractIO { serialize() { return { - value: this.__output ? null : this.__value, + value: this.__output ? null : serialize(this.__value), output: this.__output ? this.__output.id: null, } } + + deserialize(value) { + const deserializedValue = deserialize(value); + if (deserializedValue) { + this.value = deserializedValue; + } + } } \ No newline at end of file diff --git a/src/nodes/io/serializer.js b/src/nodes/io/serializer.js new file mode 100644 index 0000000..d2b6dfa --- /dev/null +++ b/src/nodes/io/serializer.js @@ -0,0 +1,23 @@ +import {isNil} from '../../utils'; + +const serializeImage = (image) => { + return {type: 'image', src: image.src}; +} + +export const serialize = (value) => { + return value instanceof Image ? serializeImage(value) : + value +} + + +const deserializeImage = ({src}) => { + const img = new Image(); + img.src = src; + return img; +} + +export const deserialize = (value) => { + return isNil(value) ? value : + value.type === 'image' ? deserializeImage(value) : + value; +} \ No newline at end of file diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..1857edb --- /dev/null +++ b/src/utils.js @@ -0,0 +1,10 @@ +export const isNil = (val) => val === null || val === undefined; + +const waitForImage = (img) => new Promise((resolve) => img.addEventListener('load', resolve, {once: true})); + +export const waitForMedia = async (media) => { + if (media instanceof Image && !media.complete) { + await waitForImage(media); + } + return media; +} \ No newline at end of file diff --git a/ui/Components/App.vue b/ui/Components/App.vue index b4e5638..ceee48c 100644 --- a/ui/Components/App.vue +++ b/ui/Components/App.vue @@ -68,6 +68,8 @@ export default { bottom: 0; top: 0; right: 0; + font-family: sans-serif; + font-size: 12px; } .app__toolbar { diff --git a/ui/Components/Node.vue b/ui/Components/Node.vue index b029319..6946804 100644 --- a/ui/Components/Node.vue +++ b/ui/Components/Node.vue @@ -2,7 +2,10 @@
-
+
{{node.name}}
+
-
-
- {{node.name}} -
+
- -