gemma/graphfx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

⚡ reusing canvas, passing imageBitmap instead

Browse Source
This commit is contained in:
Stanislav Fifik
2019-01-22 18:53:21 +01:00
parent abd29d2999
commit a8bced5b9e
22 changed files with 346 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
package-lock.json generated
View File

@@ -6185,8 +6185,7 @@
"lodash": {
"version": "4.17.11",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
"dev": true
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
},
"lodash.debounce": {
"version": "4.0.8",

1
package.json
View File

@@ -33,6 +33,7 @@
},
"dependencies": {
"interactjs": "^1.3.4",
"lodash": "^4.17.11",
"uuid": "^3.3.2",
"vue": "^2.5.21",
"webgl-utils": "^1.0.1"

44
src/nodes/Canvas2d/Canvas2d.js Normal file
View File

@@ -0,0 +1,44 @@
import Node from '../Node';
import {waitForMedia} from '../../utils';
export default class Canvas2d extends Node {
constructor(name, inputDefinition, outputDefiniton, options) {
super(name,
Object.assign({
image: {
type: 'Image',
},
}, inputDefinition),
Object.assign({
image: {
type: 'Image',
}
}, outputDefiniton),
options
);
this.__canvas = document.createElement('canvas');
}
async __update() {
const values = {};
for (let name of Object.keys(this.in.variables)) {
values[name] = await waitForMedia(this.in[name].value);
}
const ctx = this.__canvas.getContext('2d');
if (this.__canvas.width > 0 && this.__canvas.height > 0) {
ctx.clearRect(0,0,this.__canvas.width, this.__canvas.height);
}
await this.render(values, this.__canvas, ctx);
}
/**
*
* @param {any} values
* @param {HTMLCanvasElement} canvas
* @param {CanvasRenderingContext2D} ctx
*/
async render(values, canvas, ctx) {
}
}

26
src/nodes/Compose.js → src/nodes/Canvas2d/Compose.js
View File

@@ -1,10 +1,11 @@
import Node from './Node';
import {createCanvas, mediaSize, paintToCanvas} from './canvas';
import Canvas2d from './Canvas2d';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
export default class Compose extends Node {
export default class Compose extends Canvas2d {
constructor(options) {
super('Compose', {
image: null,
fg: {
type: 'Image'
},
@@ -120,23 +121,30 @@ export default class Compose extends Node {
}
}
__update() {
if (!this.width || !this.height) {
/**
* @param {any} values
* @param {HTMLCanvasElement} canvas
* @param {CanvasRenderingContext2D} ctx
*/
async render({width, height, mode}, canvas, ctx) {
if (!width || !height) {
return;
}
const canvas = createCanvas(this.width, this.height);
canvas.width = width;
canvas.height = height;
ctx.globalCompositeOperation = 'source-over';
if (this.bg) {
paintToCanvas(canvas, this.bg.image, this.bg);
}
const ctx = canvas.getContext('2d');
ctx.globalCompositeOperation = this.mode;
ctx.globalCompositeOperation = mode;
if (this.fg) {
paintToCanvas(canvas, this.fg.image, this.fg);
}
this.__out.image.value = canvas;
this.__out.image.value = await createImageBitmap(canvas);
}
}

26
src/nodes/Canvas2d/ImageIdentity.js Normal file
View File

@@ -0,0 +1,26 @@
import Node from './Canvas2d';
import {waitForMedia} from '../../utils';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
export default class ImageIdentity extends Node {
constructor() {
super('ImageIdentity', {
}, {
});
}
async render({image}) {
if (!image) {
return;
}
const {width, height} = mediaSize(image);
const canvas = createCanvas(width, height);
paintToCanvas(canvas, image, {width, height, top: 0, left: 0});
const dataUrl = canvas.toDataURL()
const i = new Image();
i.src = dataUrl;
await waitForMedia(i);
this.__out.image.value = i;
}
}

46
src/nodes/Resize.js → src/nodes/Canvas2d/Resize.js
View File

@@ -1,13 +1,11 @@
import Node from './Node';
import {createCanvas, mediaSize, paintToCanvas} from './canvas';
import Canvas2d from './Canvas2d';
import {waitForMedia} from '../../utils';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
export default class Resize extends Node {
export default class Resize extends Canvas2d {
constructor(options) {
constructor() {
super('Resize', {
image: {
type: 'Image',
},
width: {
type: 'Number',
default: 100,
@@ -18,28 +16,26 @@ export default class Resize extends Node {
default: 100,
min: 1,
},
}, {
image: {
type: 'Image',
}
});
this.options = options;
}, {});
}
get width() {
return this.__in.width.value;
}
get height() {
return this.__in.height.value;
}
__update() {
const media = this.__in.image.value;
/**
* @param {any} values
* @param {HTMLCanvasElement} canvas
* @param {CanvasRenderingContext2D} ctx
*/
async render({image: media, width, height}, canvas, ctx) {
await waitForMedia(media)
if (!media) {
return;
}
const canvas = createCanvas(this.width, this.height);
if (!width || !height) {
return;
}
canvas.width = width;
canvas.height = height;
const {width: srcWidth, height: srcHeight} = mediaSize(media);
const {width: destWidth, height: destHeight} = mediaSize(canvas);
@@ -58,6 +54,6 @@ export default class Resize extends Node {
newImage.left = (destWidth - newImage.width) / 2;
paintToCanvas(canvas, media, newImage);
this.__out.image.value = canvas;
this.__out.image.value = await createImageBitmap(canvas);
}
}

49
src/nodes/Canvas2d/Text.js Normal file
View File

@@ -0,0 +1,49 @@
import Canvas2d from './Canvas2d';
import {waitForMedia} from '../../utils';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
export default class Resize extends Canvas2d {
constructor() {
super('Text', {
image: null,
text: {
type: 'String',
default: '',
},
fontSize: {
type: 'Number',
min: 1,
default: 50,
},
font: {
type: 'String',
default: 'Arial',
},
width: {
type: 'Number',
default: 100,
min: 1,
},
height: {
type: 'Number',
default: 100,
min: 1,
},
color: {
type: 'Color',
default: '#FFFFFF'
}
}, {});
}
async render({font, fontSize, text, color, width, height}, canvas, ctx) {
canvas.width = width;
canvas.height = height;
ctx.font = `${fontSize}px ${font}`;
ctx.fillStyle = color;
ctx.textAlign = 'center';
ctx.fillText(text, canvas.width/2, canvas.height/2);
this.__out.image.value = await createImageBitmap(canvas);
}
}

11
src/nodes/Node.js
View File

@@ -46,15 +46,18 @@ export default class Node {
reconnect({options}, outputs) {
console.log('outputs', outputs)
for (let name of Object.keys(options.in)) {
const {value, output} = options.in[name];
if (value) {
this.in[name].value = value;
}
const {output} = options.in[name];
if (output) {
console.log('connecting', this.in[name].id, outputs[output])
this.in[name].connect(outputs[output]);
}
}
for (let name of Object.keys(options.in)) {
const {value} = options.in[name];
if (value) {
this.in[name].deserialize(value);
}
}
}
};

57
src/nodes/WebGL/Channels.js Normal file
View File

@@ -0,0 +1,57 @@
import WebGL from './WebGl';
export default class Channels extends WebGL {
constructor() {
super('Greyscale by channel', {
red: {
type: 'Number',
default: 1,
min: 0,
max: 1,
step: 0.1,
},
green: {
type: 'Number',
default: 1,
min: 0,
max: 1,
step: 0.1,
},
blue: {
type: 'Number',
default: 1,
min: 0,
max: 1,
step: 0.1,
}
})
}
get frag() {
return `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
uniform vec3 u_weights;
void main() {
vec4 pixelColor = texture2D(u_image, v_texCoord).rgba;
vec3 weights = u_weights / max((u_weights.r + u_weights.g + u_weights.b), 1.0);
float grey = dot(pixelColor.rgb, weights);
gl_FragColor = vec4(grey, grey, grey, pixelColor.a);
}
`
}
_setParams(gl, program) {
gl.uniform3f(gl.getUniformLocation(program, "u_weights"), this.in.red.value, this.in.green.value, this.in.blue.value);
}
}

3
src/nodes/WebGL/Greenscreen.js
View File

@@ -89,7 +89,7 @@ export default class GreenScreen extends WebGL {
vec4 pixel = mix(semiTransparentPixel, sourcePixel, solid);
// vec4 pixel = vec4(sourcePixel.rgb, (1.0 - alpha));
gl_FragColor = vec4(pixel.rgb * pixel.a, pixel.a);
gl_FragColor = vec4(pixel.rgb * pixel.a, min(max(pixel.a, 0.0), 1.0));
//gl_FragColor = vec4(min(1.0, max(pixel.r, 0.0)), min(1.0, max(pixel.g, 0.0)), min(1.0, max(pixel.b, 0.0)), min(1.0, max(pixel.a, 0.0)));
}
`
@@ -116,7 +116,6 @@ export default class GreenScreen extends WebGL {
}
_setParams(gl, program) {
console.log(this.balance, this.clipBlack, this.clipWhite, this.screenWeight)
gl.uniform3f(gl.getUniformLocation(program, 'screen'), ...this.screen);
gl.uniform1f(gl.getUniformLocation(program, 'balance'), this.balance);
gl.uniform1f(gl.getUniformLocation(program, 'clipBlack'), this.clipBlack);

6
src/nodes/WebGL/WebGl.js
View File

@@ -2,8 +2,6 @@ import Node from '../Node';
import webglUtils from '../lib/webgl-utils';
import {createCanvas, mediaSize, paintToCanvas} from '../canvas';
console.log(webglUtils);
export default class WebGL extends Node {
constructor(name, inputs={}) {
@@ -140,7 +138,7 @@ export default class WebGL extends Node {
}
}
__update() {
async __update() {
const image = this.in.image.value;
if (!image) return;
const {width, height} = mediaSize(image);
@@ -217,7 +215,7 @@ export default class WebGL extends Node {
// Draw the rectangle.
gl.drawArrays(gl.TRIANGLES, 0, 6);
this.out.image.value = canvas;
this.out.image.value = await createImageBitmap(canvas);
}
setRectangle(gl, x, y, width, height) {

9
src/nodes/Webcam.js
View File

@@ -18,8 +18,8 @@ export default class ToBlob extends Node {
console.log(last(devices));
const stream = await navigator.mediaDevices.getUserMedia({
video: {
width: { min: 1024, ideal: 1920, max: 1920 },
height: { min: 776, ideal: 1080, max: 1080 },
width: { min: 1024, ideal: 1280, max: 1920 },
height: { min: 776, ideal: 720, max: 1080 },
deviceId: last(devices).deviceId
},
});
@@ -32,8 +32,9 @@ export default class ToBlob extends Node {
video.play();
let lastFrameTime = null;
const feedLoop = () => {
if (lastFrameTime !== video.currentTime) {
lastFrameTime = video.currentTime;
const currentTime = video.currentTime;
if (lastFrameTime !== currentTime) {
lastFrameTime = currentTime;
this.out.image.value = video;
}
requestAnimationFrame(feedLoop);

12
src/nodes/index.js
View File

@@ -1,10 +1,10 @@
export {default as Webcam} from './Webcam';
export {default as Resize} from './Resize';
export {default as ImageIdentity} from './ImageIdentity';
export {default as ToBlob} from './ToBlob';
export {default as Compose} from './Compose';
export {default as WebGL} from './WebGL/WebGl';
export {default as Resize} from './Canvas2d/Resize';
export {default as Compose} from './Canvas2d/Compose';
// export {default as ImageIdentity} from './Canvas2d/ImageIdentity';
export {default as Text} from './Canvas2d/Text';
export {default as BrightnessContrast} from './WebGL/BrightnessContrast';
export {default as HSV} from './WebGL/HSV';
export {default as Sepia} from './WebGL/Sepia';
export {default as GreenScreen} from './WebGL/Greenscreen';
export {default as GreenScreen} from './WebGL/Greenscreen';
export {default as Channels} from './WebGL/Channels';

2
src/nodes/io/AbstractIO.js
View File

@@ -1,4 +1,4 @@
const isNil = (val) => val === null || val === undefined;
import {isNil} from '../../utils';
export const defaultConstrainSatisfied = (value) => {
return !isNil(value);

4
src/nodes/io/AbstractIOSet.js
View File

@@ -8,6 +8,10 @@ export default class AbstractIOSet {
this.variables = variables;
for (let name of Object.keys(this.variables)) {
if (variables[name] === null) {
delete this.variables[name];
continue;
}
this.__values[name] = this.__createProperty(name, variables[name]);
Object.defineProperty(this, name, {
get() {

12
src/nodes/io/Input.js
View File

@@ -1,4 +1,5 @@
import AbstractIO from './AbstractIO';
import {serialize, deserialize} from './serializer';
export default class Input extends AbstractIO {
@@ -19,6 +20,7 @@ export default class Input extends AbstractIO {
set value(value) {
this.__setValue(value);
this.__owner.update(this.name);
this.__notifyListeners();
}
get output() {
@@ -30,6 +32,7 @@ export default class Input extends AbstractIO {
this.disconnect();
this.__output = output
this.__output.onchange(this.__onchangelistener);
this.value = this.__output.value;
}
}
@@ -42,8 +45,15 @@ export default class Input extends AbstractIO {
serialize() {
return {
value: this.__output ? null : this.__value,
value: this.__output ? null : serialize(this.__value),
output: this.__output ? this.__output.id: null,
}
}
deserialize(value) {
const deserializedValue = deserialize(value);
if (deserializedValue) {
this.value = deserializedValue;
}
}
}

23
src/nodes/io/serializer.js Normal file
View File

@@ -0,0 +1,23 @@
import {isNil} from '../../utils';
const serializeImage = (image) => {
return {type: 'image', src: image.src};
}
export const serialize = (value) => {
return value instanceof Image ? serializeImage(value) :
value
}
const deserializeImage = ({src}) => {
const img = new Image();
img.src = src;
return img;
}
export const deserialize = (value) => {
return isNil(value) ? value :
value.type === 'image' ? deserializeImage(value) :
value;
}

10
src/utils.js Normal file
View File

@@ -0,0 +1,10 @@
export const isNil = (val) => val === null || val === undefined;
const waitForImage = (img) => new Promise((resolve) => img.addEventListener('load', resolve, {once: true}));
export const waitForMedia = async (media) => {
if (media instanceof Image && !media.complete) {
await waitForImage(media);
}
return media;
}

2
ui/Components/App.vue
View File

@@ -68,6 +68,8 @@ export default {
bottom: 0;
top: 0;
right: 0;
font-family: sans-serif;
font-size: 12px;
}
.app__toolbar {

29
ui/Components/Node.vue
View File

@@ -2,7 +2,10 @@
<div
class="node"
>
<div style="flex: 1" class="node__vars node__vars--input">
<div
class="node__title"
>{{node.name}}</div>
<div class="node__vars node__vars--input">
<div
v-for="(type, name) in node.in.variables"
:key="name"
@@ -28,10 +31,7 @@
/>
</div>
</div>
<div style="flex: 1">
<div>
{{node.name}}
</div>
<div class="node__vars node__vars--output">
<div
v-for="(type, name) in node.out.variables"
:key="name"
@@ -144,17 +144,34 @@ export default {
.node {
margin: 5px;
padding: 5px 0;
display: inline-flex;
display: grid;
flex-direction: row;
background-color: #EEE;
border-radius: 5px;
box-shadow: 1px 1px 5px rgba(0,0,0,0.35);
grid-gap: 5px;
grid-template-areas:
"title title"
"in out";
}
.node__title {
grid-area: title;
text-align: center;
border-bottom: 1px solid black;
}
.node__var {
margin-bottom: 2px
}
.node__vars--input {
grid-area: in;
}
.node__vars--output {
grid-area: out;
}
.node__remove {
position: absolute;
top: -20px;

48
ui/Components/Value/Image.vue
View File

@@ -2,15 +2,16 @@
<div
class="image-value"
@click="onClick"
@mouseenter="isPreviewVisible = true"
@mouseleave="isPreviewVisible = false"
>
<canvas class="image-value__thumb" ref="thumb" />
<canvas class="image-value__preview" ref="preview" />
</div>
</template>
<script>
import {paintToCanvas, mediaSize} from '../../../src/nodes/canvas.js'
import _ from 'lodash';
export default {
props: {
@@ -27,25 +28,28 @@ export default {
required: true,
}
},
watch: {
value: {
handler(value) {
this.redraw();
}
}
},
data() {
return {
isRunning: true,
isPreviewVisible: false,
/*redrawThumb: _.throttle(() => {
console.log('redrawThum')
}, 500, {leading: true, trailing: true}),*/
redrawThumb: _.throttle(() => this.redrawCanvas(this.$refs.thumb), 500, {leading: true, trailing: true}),
};
},
mounted() {
this.$nextTick(() => {
this.redraw();
this.io.onchange(this.redraw);
this.io.onchange(this.onChange);
})
},
watch: {
isPreviewVisible(isVisible) {
if (isVisible) {
this.redraw();
}
},
isRunning: {
handler() {
this.redraw();
@@ -54,10 +58,16 @@ export default {
},
beforeDestroy() {
this.isRunning = false;
if (this.io) {
this.io.offchange(this.onChange)
}
},
methods: {
onChange() {
this.redraw();
},
redrawCanvas(canvas) {
const value = this.value;
const value = this.io.value;
/** @type {HTMLCanvasElement} */
const ctx = canvas.getContext('2d');
if (!value) {
@@ -77,8 +87,11 @@ export default {
},
redraw() {
if (this.isRunning) {
this.redrawCanvas(this.$refs.thumb);
this.redrawCanvas(this.$refs.preview)
this.redrawThumb();
// this.redrawCanvas(this.$refs.thumb);
if (this.isPreviewVisible) {
this.redrawCanvas(this.$refs.preview)
}
}
},
onClick() {
@@ -95,13 +108,13 @@ export default {
input.accept = 'image/*';
input.onchange = (event) => {
const files = event.target.files;
console.log(event);
if (files.length) {
const fr = new FileReader()
fr.onload = (event) => {
const image = new Image();
image.onload = () => {
this.$emit('change', image);
this.onChange();
}
image.src = event.target.result;
}
@@ -124,11 +137,16 @@ export default {
.image-value {
position: relative;
}
.image-value__thumb, .image-value__preview {
background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAJ0lEQVQoU2NsaGj4z4AG6uvr0YUYGIeCwv///2N4prGxEdMzQ0AhAChTL3KV95+lAAAAAElFTkSuQmCC);
background-repeat: repeat;
}
.image-value__thumb {
width: 50px;
height: 50px;
object-fit: contain;
background-color: black;
box-shadow: 0 0 6px;
}

3
ui/Components/Value/index.js
View File

@@ -15,10 +15,11 @@ export default {
},
render(h) {
return h(
this.io.definition.enum ? SelectValue :
this.io.type === 'Image' ? ImageValue :
this.io.type === 'String' ? StringValue :
this.io.type === 'Number' ? StringValue :
this.io.type === 'Color' ? StringValue :
this.io.definition.enum ? SelectValue :
null,
{
props: {