⚡ reusing canvas, passing imageBitmap instead

package-lock.json

@@ -6185,8 +6185,7 @@
     "lodash": {
       "version": "4.17.11",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
+      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
-      "dev": true
     },
     "lodash.debounce": {
       "version": "4.0.8",

package.json

@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "interactjs": "^1.3.4",
+    "lodash": "^4.17.11",
     "uuid": "^3.3.2",
     "vue": "^2.5.21",
     "webgl-utils": "^1.0.1"

src/nodes/Canvas2d/Canvas2d.js

@@ -0,0 +1,44 @@
+import Node from '../Node';
+import {waitForMedia} from '../../utils';
+
+export default class Canvas2d extends Node {
+    constructor(name, inputDefinition, outputDefiniton, options) {
+        super(name,
+            Object.assign({
+                image: {
+                    type: 'Image',
+                },
+            }, inputDefinition),
+            Object.assign({
+                image: {
+                    type: 'Image',
+                }
+            }, outputDefiniton),
+            options
+        );
+        this.__canvas = document.createElement('canvas');
+    }
+
+    async __update() {
+        const values = {};
+        for (let name of Object.keys(this.in.variables)) {
+            values[name] = await waitForMedia(this.in[name].value);
+        }
+        const ctx = this.__canvas.getContext('2d');
+        if (this.__canvas.width > 0 && this.__canvas.height > 0) {
+            ctx.clearRect(0,0,this.__canvas.width, this.__canvas.height);
+        }
+        await this.render(values, this.__canvas, ctx);
+    }
+
+
+    /**
+     *
+     * @param {any} values
+     * @param {HTMLCanvasElement} canvas
+     * @param {CanvasRenderingContext2D} ctx
+     */
+    async render(values, canvas, ctx) {
+
+    }
+}

src/nodes/Canvas2d/Compose.js

@@ -1,10 +1,11 @@
-import Node from './Node';
+import Canvas2d from './Canvas2d';
-import {createCanvas, mediaSize, paintToCanvas} from './canvas';
+import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
 
-export default class Compose extends Node {
+export default class Compose extends Canvas2d {
 
     constructor(options) {
         super('Compose', {
+            image: null,
             fg: {
                 type: 'Image'
             },
@@ -120,23 +121,30 @@ export default class Compose extends Node {
         }
     }
 
-    __update() {
+    /**
-        if (!this.width || !this.height) {
+     * @param {any} values
+     * @param {HTMLCanvasElement} canvas
+     * @param {CanvasRenderingContext2D} ctx
+     */
+    async render({width, height, mode}, canvas, ctx) {
+        if (!width || !height) {
             return;
         }
-        const canvas = createCanvas(this.width, this.height);
+        canvas.width = width;
+        canvas.height = height;
+
+        ctx.globalCompositeOperation = 'source-over';
 
         if (this.bg) {
             paintToCanvas(canvas, this.bg.image, this.bg);
         }
 
-        const ctx = canvas.getContext('2d');
+        ctx.globalCompositeOperation = mode;
-        ctx.globalCompositeOperation = this.mode;
 
         if (this.fg) {
             paintToCanvas(canvas, this.fg.image, this.fg);
         }
 
-        this.__out.image.value = canvas;
+        this.__out.image.value = await createImageBitmap(canvas);
     }
 }

src/nodes/Canvas2d/ImageIdentity.js

@@ -0,0 +1,26 @@
+import Node from './Canvas2d';
+import {waitForMedia} from '../../utils';
+import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
+
+
+export default class ImageIdentity extends Node {
+    constructor() {
+        super('ImageIdentity', {
+        }, {
+        });
+    }
+
+    async render({image}) {
+        if (!image) {
+            return;
+        }
+        const {width, height} = mediaSize(image);
+        const canvas = createCanvas(width, height);
+        paintToCanvas(canvas, image, {width, height, top: 0, left: 0});
+        const dataUrl = canvas.toDataURL()
+        const i = new Image();
+        i.src = dataUrl;
+        await waitForMedia(i);
+        this.__out.image.value = i;
+    }
+}

src/nodes/Canvas2d/Resize.js

@@ -1,13 +1,11 @@
-import Node from './Node';
+import Canvas2d from './Canvas2d';
-import {createCanvas, mediaSize, paintToCanvas} from './canvas';
+import {waitForMedia} from '../../utils';
+import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
 
-export default class Resize extends Node {
+export default class Resize extends Canvas2d {
 
-    constructor(options) {
+    constructor() {
         super('Resize', {
-            image: {
-                type: 'Image',
-            },
             width: {
                 type: 'Number',
                 default: 100,
@@ -18,28 +16,26 @@ export default class Resize extends Node {
                 default: 100,
                 min: 1,
             },
-        }, {
+        }, {});
-            image: {
-                type: 'Image',
-            }
-        });
-        this.options = options;
     }
 
-    get width() {
+    /**
-        return this.__in.width.value;
+     * @param {any} values
-    }
+     * @param {HTMLCanvasElement} canvas
-
+     * @param {CanvasRenderingContext2D} ctx
-    get height() {
+     */
-        return this.__in.height.value;
+    async render({image: media, width, height}, canvas, ctx) {
-    }
+        await waitForMedia(media)
-
-    __update() {
-        const media = this.__in.image.value;
         if (!media) {
             return;
         }
-        const canvas = createCanvas(this.width, this.height);
+
+        if (!width || !height) {
+            return;
+        }
+
+        canvas.width = width;
+        canvas.height = height;
 
         const {width: srcWidth, height: srcHeight} = mediaSize(media);
         const {width: destWidth, height: destHeight} = mediaSize(canvas);
@@ -58,6 +54,6 @@ export default class Resize extends Node {
         newImage.left = (destWidth - newImage.width) / 2;
 
         paintToCanvas(canvas, media, newImage);
-        this.__out.image.value = canvas;
+        this.__out.image.value = await createImageBitmap(canvas);
     }
 }

src/nodes/Canvas2d/Text.js

@@ -0,0 +1,49 @@
+import Canvas2d from './Canvas2d';
+import {waitForMedia} from '../../utils';
+import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
+
+export default class Resize extends Canvas2d {
+
+    constructor() {
+        super('Text', {
+            image: null,
+            text: {
+                type: 'String',
+                default: '',
+            },
+            fontSize: {
+                type: 'Number',
+                min: 1,
+                default: 50,
+            },
+            font: {
+                type: 'String',
+                default: 'Arial',
+            },
+            width: {
+                type: 'Number',
+                default: 100,
+                min: 1,
+            },
+            height: {
+                type: 'Number',
+                default: 100,
+                min: 1,
+            },
+            color: {
+                type: 'Color',
+                default: '#FFFFFF'
+            }
+        }, {});
+    }
+
+    async render({font, fontSize, text, color, width, height}, canvas, ctx) {
+        canvas.width = width;
+        canvas.height = height;
+        ctx.font = `${fontSize}px ${font}`;
+        ctx.fillStyle = color;
+        ctx.textAlign = 'center';
+        ctx.fillText(text, canvas.width/2, canvas.height/2);
+        this.__out.image.value = await createImageBitmap(canvas);
+    }
+}

src/nodes/Node.js

@@ -46,15 +46,18 @@ export default class Node {
     reconnect({options}, outputs) {
         console.log('outputs', outputs)
         for (let name of Object.keys(options.in)) {
-            const {value, output} = options.in[name];
+            const {output} = options.in[name];
-            if (value) {
-                this.in[name].value = value;
-            }
             if (output) {
                 console.log('connecting', this.in[name].id, outputs[output])
                 this.in[name].connect(outputs[output]);
             }
         }
+        for (let name of Object.keys(options.in)) {
+            const {value} = options.in[name];
+            if (value) {
+                this.in[name].deserialize(value);
+            }
+        }
     }
 };
 

src/nodes/WebGL/Channels.js

@@ -0,0 +1,57 @@
+import WebGL from './WebGl';
+
+
+export default class Channels extends WebGL {
+
+    constructor() {
+        super('Greyscale by channel', {
+            red: {
+                type: 'Number',
+                default: 1,
+                min: 0,
+                max: 1,
+                step: 0.1,
+            },
+            green: {
+                type: 'Number',
+                default: 1,
+                min: 0,
+                max: 1,
+                step: 0.1,
+            },
+            blue: {
+                type: 'Number',
+                default: 1,
+                min: 0,
+                max: 1,
+                step: 0.1,
+            }
+        })
+    }
+
+    get frag() {
+        return `
+        precision mediump float;
+
+        // our texture
+        uniform sampler2D u_image;
+
+        // the texCoords passed in from the vertex shader.
+        varying vec2 v_texCoord;
+
+        uniform vec3 u_weights;
+
+        void main() {
+            vec4 pixelColor = texture2D(u_image, v_texCoord).rgba;
+            vec3 weights = u_weights / max((u_weights.r + u_weights.g + u_weights.b), 1.0);
+            float grey = dot(pixelColor.rgb, weights);
+
+            gl_FragColor = vec4(grey, grey, grey, pixelColor.a);
+        }
+        `
+    }
+
+    _setParams(gl, program) {
+        gl.uniform3f(gl.getUniformLocation(program, "u_weights"), this.in.red.value, this.in.green.value, this.in.blue.value);
+    }
+}

src/nodes/WebGL/Greenscreen.js

@@ -89,7 +89,7 @@ export default class GreenScreen extends WebGL {
             vec4 pixel = mix(semiTransparentPixel, sourcePixel, solid);
             // vec4 pixel = vec4(sourcePixel.rgb, (1.0 - alpha));
 
-            gl_FragColor = vec4(pixel.rgb * pixel.a, pixel.a);
+            gl_FragColor = vec4(pixel.rgb * pixel.a, min(max(pixel.a, 0.0), 1.0));
             //gl_FragColor = vec4(min(1.0, max(pixel.r, 0.0)), min(1.0, max(pixel.g, 0.0)), min(1.0, max(pixel.b, 0.0)), min(1.0, max(pixel.a, 0.0)));
         }
         `
@@ -116,7 +116,6 @@ export default class GreenScreen extends WebGL {
     }
 
     _setParams(gl, program) {
-        console.log(this.balance, this.clipBlack, this.clipWhite, this.screenWeight)
         gl.uniform3f(gl.getUniformLocation(program, 'screen'), ...this.screen);
         gl.uniform1f(gl.getUniformLocation(program, 'balance'), this.balance);
         gl.uniform1f(gl.getUniformLocation(program, 'clipBlack'), this.clipBlack);

src/nodes/WebGL/WebGl.js

@@ -2,8 +2,6 @@ import Node from '../Node';
 import webglUtils from '../lib/webgl-utils';
 import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
 
-console.log(webglUtils);
-
 export default class WebGL extends Node {
 
     constructor(name, inputs={}) {
@@ -140,7 +138,7 @@ export default class WebGL extends Node {
         }
     }
 
-    __update() {
+    async __update() {
         const image = this.in.image.value;
         if (!image) return;
         const {width, height} = mediaSize(image);
@@ -217,7 +215,7 @@ export default class WebGL extends Node {
         // Draw the rectangle.
         gl.drawArrays(gl.TRIANGLES, 0, 6);
 
-        this.out.image.value = canvas;
+        this.out.image.value = await createImageBitmap(canvas);
     }
 
     setRectangle(gl, x, y, width, height) {

src/nodes/Webcam.js

@@ -18,8 +18,8 @@ export default class ToBlob extends Node {
         console.log(last(devices));
         const stream = await navigator.mediaDevices.getUserMedia({
             video: {
-                width: { min: 1024, ideal: 1920, max: 1920 },
+                width: { min: 1024, ideal: 1280, max: 1920 },
-                height: { min: 776, ideal: 1080, max: 1080 },
+                height: { min: 776, ideal: 720, max: 1080 },
                 deviceId: last(devices).deviceId
             },
         });
@@ -32,8 +32,9 @@ export default class ToBlob extends Node {
         video.play();
         let lastFrameTime = null;
         const feedLoop = () => {
-            if (lastFrameTime !== video.currentTime) {
+            const currentTime = video.currentTime;
-                lastFrameTime = video.currentTime;
+            if (lastFrameTime !== currentTime) {
+                lastFrameTime = currentTime;
                 this.out.image.value = video;
             }
             requestAnimationFrame(feedLoop);

src/nodes/index.js

@@ -1,10 +1,10 @@
 export {default as Webcam} from './Webcam';
-export {default as Resize} from './Resize';
+export {default as Resize} from './Canvas2d/Resize';
-export {default as ImageIdentity} from './ImageIdentity';
+export {default as Compose} from './Canvas2d/Compose';
-export {default as ToBlob} from './ToBlob';
+// export {default as ImageIdentity} from './Canvas2d/ImageIdentity';
-export {default as Compose} from './Compose';
+ export {default as Text} from './Canvas2d/Text';
-export {default as WebGL} from './WebGL/WebGl';
 export {default as BrightnessContrast} from './WebGL/BrightnessContrast';
 export {default as HSV} from './WebGL/HSV';
 export {default as Sepia} from './WebGL/Sepia';
 export {default as GreenScreen} from './WebGL/Greenscreen';
+export {default as Channels} from './WebGL/Channels';

src/nodes/io/AbstractIO.js

@@ -1,4 +1,4 @@
-const isNil = (val) => val === null || val === undefined;
+import {isNil} from '../../utils';
 
 export const defaultConstrainSatisfied = (value) => {
     return !isNil(value);

src/nodes/io/AbstractIOSet.js

@@ -8,6 +8,10 @@ export default class AbstractIOSet {
         this.variables = variables;
 
         for (let name of Object.keys(this.variables)) {
+            if (variables[name] === null) {
+                delete this.variables[name];
+                continue;
+            }
             this.__values[name] = this.__createProperty(name, variables[name]);
             Object.defineProperty(this, name, {
                 get() {

src/nodes/io/Input.js

@@ -1,4 +1,5 @@
 import AbstractIO from './AbstractIO';
+import {serialize, deserialize} from './serializer';
 
 export default class Input extends AbstractIO {
 
@@ -19,6 +20,7 @@ export default class Input extends AbstractIO {
     set value(value) {
         this.__setValue(value);
         this.__owner.update(this.name);
+        this.__notifyListeners();
     }
 
     get output() {
@@ -30,6 +32,7 @@ export default class Input extends AbstractIO {
             this.disconnect();
             this.__output = output
             this.__output.onchange(this.__onchangelistener);
+            this.value = this.__output.value;
         }
     }
 
@@ -42,8 +45,15 @@ export default class Input extends AbstractIO {
 
     serialize() {
         return {
-            value: this.__output ? null : this.__value,
+            value: this.__output ? null : serialize(this.__value),
             output: this.__output ? this.__output.id: null,
         }
     }
+
+    deserialize(value) {
+        const deserializedValue = deserialize(value);
+        if (deserializedValue) {
+            this.value = deserializedValue;
+        }
+    }
 }

src/nodes/io/serializer.js

@@ -0,0 +1,23 @@
+import {isNil} from '../../utils';
+
+const serializeImage = (image) => {
+    return {type: 'image', src: image.src};
+}
+
+export const serialize = (value) => {
+    return value instanceof Image ? serializeImage(value) :
+        value
+}
+
+
+const deserializeImage = ({src}) => {
+    const img = new Image();
+    img.src = src;
+    return img;
+}
+
+export const deserialize = (value) => {
+    return isNil(value) ? value :
+        value.type === 'image' ? deserializeImage(value) :
+        value;
+}

src/utils.js

@@ -0,0 +1,10 @@
+export const isNil = (val) => val === null || val === undefined;
+
+const waitForImage = (img) => new Promise((resolve) => img.addEventListener('load', resolve, {once: true}));
+
+export const waitForMedia = async (media) => {
+    if (media instanceof Image && !media.complete) {
+        await waitForImage(media);
+    }
+    return media;
+}

ui/Components/App.vue

@@ -68,6 +68,8 @@ export default {
     bottom: 0;
     top: 0;
     right: 0;
+    font-family: sans-serif;
+    font-size: 12px;
 }
 
 .app__toolbar {

ui/Components/Node.vue

@@ -2,7 +2,10 @@
     <div
         class="node"
     >
-        <div style="flex: 1" class="node__vars node__vars--input">
+        <div
+            class="node__title"
+        >{{node.name}}</div>
+        <div class="node__vars node__vars--input">
             <div
                 v-for="(type, name) in node.in.variables"
                 :key="name"
@@ -28,10 +31,7 @@
                 />
             </div>
         </div>
-        <div style="flex: 1">
+        <div  class="node__vars node__vars--output">
-            <div>
-                {{node.name}}
-            </div>
             <div
                 v-for="(type, name) in node.out.variables"
                 :key="name"
@@ -144,17 +144,34 @@ export default {
 .node {
     margin: 5px;
     padding: 5px 0;
-    display: inline-flex;
+    display: grid;
     flex-direction: row;
     background-color: #EEE;
     border-radius: 5px;
     box-shadow: 1px 1px 5px rgba(0,0,0,0.35);
+    grid-gap: 5px;
+    grid-template-areas:
+        "title title"
+        "in  out";
+}
+.node__title {
+    grid-area: title;
+    text-align: center;
+    border-bottom: 1px solid black;
 }
 
 .node__var {
     margin-bottom: 2px
 }
 
+.node__vars--input {
+    grid-area: in;
+}
+
+.node__vars--output {
+    grid-area: out;
+}
+
 .node__remove {
     position: absolute;
     top: -20px;

ui/Components/Value/Image.vue

@@ -2,15 +2,16 @@
     <div
         class="image-value"
         @click="onClick"
+        @mouseenter="isPreviewVisible = true"
+        @mouseleave="isPreviewVisible = false"
     >
         <canvas class="image-value__thumb" ref="thumb" />
         <canvas class="image-value__preview" ref="preview" />
     </div>
-
-
 </template>
 <script>
 import {paintToCanvas, mediaSize} from '../../../src/nodes/canvas.js'
+import _ from 'lodash';
 
 export default {
     props: {
@@ -27,25 +28,28 @@ export default {
             required: true,
         }
     },
-    watch: {
-        value: {
-            handler(value) {
-                this.redraw();
-            }
-        }
-    },
     data() {
         return {
             isRunning: true,
+            isPreviewVisible: false,
+            /*redrawThumb: _.throttle(() => {
+                console.log('redrawThum')
+            }, 500, {leading: true, trailing: true}),*/
+            redrawThumb: _.throttle(() => this.redrawCanvas(this.$refs.thumb), 500, {leading: true, trailing: true}),
         };
     },
     mounted() {
         this.$nextTick(() => {
             this.redraw();
-            this.io.onchange(this.redraw);
+            this.io.onchange(this.onChange);
         })
     },
     watch: {
+        isPreviewVisible(isVisible) {
+            if (isVisible) {
+                this.redraw();
+            }
+        },
         isRunning: {
             handler() {
                 this.redraw();
@@ -54,10 +58,16 @@ export default {
     },
     beforeDestroy() {
         this.isRunning = false;
+        if (this.io) {
+            this.io.offchange(this.onChange)
+        }
     },
     methods: {
+        onChange() {
+            this.redraw();
+        },
         redrawCanvas(canvas) {
-            const value = this.value;
+            const value = this.io.value;
             /** @type {HTMLCanvasElement} */
             const ctx = canvas.getContext('2d');
             if (!value) {
@@ -77,9 +87,12 @@ export default {
         },
         redraw() {
             if (this.isRunning) {
-                this.redrawCanvas(this.$refs.thumb);
+                this.redrawThumb();
+                // this.redrawCanvas(this.$refs.thumb);
+                if (this.isPreviewVisible) {
                     this.redrawCanvas(this.$refs.preview)
                 }
+            }
         },
         onClick() {
             if (this.direction === 'output') {
@@ -95,13 +108,13 @@ export default {
             input.accept = 'image/*';
             input.onchange = (event) => {
                 const files = event.target.files;
-                console.log(event);
                 if (files.length) {
                     const fr = new FileReader()
                     fr.onload = (event) => {
                         const image = new Image();
                         image.onload = () => {
                             this.$emit('change', image);
+                            this.onChange();
                         }
                         image.src = event.target.result;
                     }
@@ -124,11 +137,16 @@ export default {
 .image-value {
     position: relative;
 }
+.image-value__thumb, .image-value__preview {
+    background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAJ0lEQVQoU2NsaGj4z4AG6uvr0YUYGIeCwv///2N4prGxEdMzQ0AhAChTL3KV95+lAAAAAElFTkSuQmCC);
+    background-repeat: repeat;
+}
+
 .image-value__thumb {
     width: 50px;
     height: 50px;
     object-fit: contain;
-    background-color: black;
+
     box-shadow: 0 0 6px;
 }
 

ui/Components/Value/index.js

@@ -15,10 +15,11 @@ export default {
     },
     render(h) {
         return h(
+            this.io.definition.enum ? SelectValue :
             this.io.type === 'Image' ? ImageValue :
+            this.io.type === 'String' ? StringValue :
             this.io.type === 'Number' ? StringValue :
             this.io.type === 'Color' ? StringValue :
-            this.io.definition.enum ? SelectValue :
             null,
         {
             props: {