From 17c585434035cff1a400c01e692e7ec4130d81f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Kol=C3=A1=C5=99?= Date: Fri, 24 Mar 2023 11:34:31 +0100 Subject: [PATCH] BodySegmentation node --- package-lock.json | 154 ++++++++++++++++++++++++++---- package.json | 11 ++- src/nodes/AI/BodySegmentation.ts | 155 +++++++++++++++++++++++++++++++ src/nodes/index.ts | 1 + 4 files changed, 298 insertions(+), 23 deletions(-) create mode 100644 src/nodes/AI/BodySegmentation.ts diff --git a/package-lock.json b/package-lock.json index b26e8bc..6c02202 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "graphfx", - "version": "0.3.3", + "version": "0.4.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -3806,6 +3806,11 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "@mediapipe/selfie_segmentation": { + "version": "0.1.1675465747", + "resolved": "https://registry.npmjs.org/@mediapipe/selfie_segmentation/-/selfie_segmentation-0.1.1675465747.tgz", + "integrity": "sha512-IxYxNhwE5VwOm52L1yoFWYLP7q9Pd+NJjzOC5tlepfvEGaY3o9hslhUrx9BgseqdfZtKSDtd/4NfCSMjNzQalA==" + }, "@sinclair/typebox": { "version": "0.24.51", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.51.tgz", @@ -3830,6 +3835,62 @@ "@sinonjs/commons": "^1.7.0" } }, + "@tensorflow-models/body-segmentation": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@tensorflow-models/body-segmentation/-/body-segmentation-1.0.1.tgz", + "integrity": "sha512-IPQ+a848KDZ/yzxJwP+GDPGwa6RLJ3+VbFraxlSWxAfygqTKx0EQQWHUawbCKxILtlzcfeinW+Gdcyiq0Onkfw==", + "requires": { + "rimraf": "^3.0.2" + } + }, + "@tensorflow/tfjs-backend-cpu": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.2.0.tgz", + "integrity": "sha512-8HWg9J69m0Ovc6w8TVhhixMOcwA3t/NPXLblOA/sgJ+/JD5gsbpLWJk4QISQyb1RnpSVzw6PX3BSMTJU7hWVOg==", + "requires": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + } + }, + "@tensorflow/tfjs-backend-webgl": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-4.2.0.tgz", + "integrity": "sha512-Qvf+hD5pSh+xi48kChSGzcDKJemkc4EKfoVVjuxl4k25ZUPwuEd7zZUAtinkLu1dzgHNyvePZY8k+9rVm59HJA==", + "requires": { + "@tensorflow/tfjs-backend-cpu": "4.2.0", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "seedrandom": "^3.0.5" + }, + "dependencies": { + "@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==" + } + } + }, + "@tensorflow/tfjs-converter": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-4.2.0.tgz", + "integrity": "sha512-m+E2KJM6yGQdi8ElzWpChdD/JaqhWMCi9yK70v/ndkOaCL2q2UN48nYP2T5S15vkDvMIgzAQyZfh7hxQsMuvRQ==" + }, + "@tensorflow/tfjs-core": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.2.0.tgz", + "integrity": "sha512-uuHkiWVC8b00ngFbHvAV7J7haRlN/9PEdeenCi0CzBjgKd7aN25wPWaoN0TSQcU+GT4FJ8mofMZ9VBYZ/s/WLg==", + "requires": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.7.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@webgpu/types": "0.1.21", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + } + }, "@types/babel__core": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.3.tgz", @@ -3925,6 +3986,11 @@ "integrity": "sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ==", "dev": true }, + "@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "@types/node": { "version": "12.12.16", "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.16.tgz", @@ -3933,8 +3999,7 @@ "@types/offscreencanvas": { "version": "2019.7.0", "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.0.tgz", - "integrity": "sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg==", - "dev": true + "integrity": "sha512-PGcyveRIpL1XIqK8eBsmRBt76eFgtzuPiSTyKHZxnGemp2yzGzWpjYKAfK3wIMiU7eH+851yEpiuP8JZerTmWg==" }, "@types/prettier": { "version": "2.7.2", @@ -3942,6 +4007,11 @@ "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==", "dev": true }, + "@types/seedrandom": { + "version": "2.4.30", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.30.tgz", + "integrity": "sha512-AnxLHewubLVzoF/A4qdxBGHCKifw8cY32iro3DQX9TPcetE95zBeVt3jnsvtvAUf1vwzMfwzp4t/L2yqPlnjkQ==" + }, "@types/stack-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-1.0.1.tgz", @@ -3956,6 +4026,11 @@ "@types/node": "*" } }, + "@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==" + }, "@types/yargs": { "version": "13.0.3", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-13.0.3.tgz", @@ -3971,6 +4046,11 @@ "integrity": "sha512-gCubfBUZ6KxzoibJ+SCUc/57Ms1jz5NjHe4+dI2krNmU5zCPAphyLJYyTOg06ueIyfj+SaCUqmzun7ImlxDcKg==", "dev": true }, + "@webgpu/types": { + "version": "0.1.21", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.21.tgz", + "integrity": "sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow==" + }, "ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -4210,8 +4290,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base": { "version": "0.11.2", @@ -4272,7 +4351,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4487,8 +4565,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "convert-source-map": { "version": "1.7.0", @@ -5230,8 +5307,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "1.2.9", @@ -5803,7 +5879,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -5916,7 +5991,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -5925,8 +5999,7 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { "version": "1.3.8", @@ -11900,6 +11973,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -11996,7 +12074,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -12081,6 +12158,14 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -12216,7 +12301,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1" } @@ -12285,8 +12369,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-key": { "version": "2.0.1", @@ -12610,6 +12693,14 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, "rsvp": { "version": "4.8.5", "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", @@ -12656,6 +12747,11 @@ } } }, + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", @@ -13105,6 +13201,11 @@ "repeat-string": "^1.6.1" } }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -13270,6 +13371,20 @@ "makeerror": "1.0.x" } }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -13319,8 +13434,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write-file-atomic": { "version": "2.4.1", diff --git a/package.json b/package.json index 3b1eff5..108ed58 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "graphfx", - "version": "0.3.3", + "version": "0.4.0", "description": "Graph image processing pipeline", "main": "src/index.ts", "scripts": { @@ -29,10 +29,15 @@ "typescript": "^3.7.3" }, "dependencies": { + "@mediapipe/selfie_segmentation": "^0.1.1675465747", + "@tensorflow-models/body-segmentation": "^1.0.1", + "@tensorflow/tfjs-backend-webgl": "^4.2.0", + "@tensorflow/tfjs-converter": "^4.2.0", + "@tensorflow/tfjs-core": "^4.2.0", "@types/css-font-loading-module": "0.0.2", "@types/uuid": "^3.4.6", - "uuid": "^3.3.3", - "lodash": "^4.17.21" + "lodash": "^4.17.21", + "uuid": "^3.3.3" }, "jest": { "setupFiles": [ diff --git a/src/nodes/AI/BodySegmentation.ts b/src/nodes/AI/BodySegmentation.ts new file mode 100644 index 0000000..ecc7dd5 --- /dev/null +++ b/src/nodes/AI/BodySegmentation.ts @@ -0,0 +1,155 @@ +import { + ImageVar, + NumberVar +} from '../io/AbstractIOSet'; +import Node from "../Node"; +import '@tensorflow/tfjs-backend-webgl'; +import { + SupportedModels, + createSegmenter, + toBinaryMask, + drawMask, + BodySegmenter, + MediaPipeSelfieSegmentationMediaPipeModelConfig, + BodySegmenterInput, +} from '@tensorflow-models/body-segmentation' +import {canvasPool2D} from "../../canvas/CanvasPool"; +import {waitForMedia} from "../../utils"; + +const inputs = { + image: { + type: 'Image' + } as ImageVar, + width: { + type: 'Number', + default: 100, + step: 1, + min: 1 + } as NumberVar, + height: { + type: 'Number', + default: 100, + step: 1, + min: 1, + } as NumberVar, + // blur mask edges by number of pixels + maskBlurAmount: { + type: 'Number', + default: 0, + step: 1, + } as NumberVar, + // Default to 0.5. The minimum probability to color a pixel as foreground rather than background. The alpha channel integer values will be taken as the probabilities (for more information refer to Segmentation type's documentation). + foregroundThreshold: { + type: 'Number', + default: 0.5, + step: 0.1, + min: 0, + max: 1, + } as NumberVar, +} + +const outputs = { + image: { + type: 'Image' + } as ImageVar, + originalImage: { + type: 'Image' + } as ImageVar, + width: { + type: 'Number', + } as NumberVar, + height: { + type: 'Number', + } as NumberVar, +} + +export default class BodySegmentation extends Node { + private segmenter: BodySegmenter + + constructor(options={}) { + super('BodySegmentation', inputs, outputs); + } + + get width() { + return this.__in.width.value; + } + + get height() { + return this.__in.height.value; + } + + get image() { + if (!this.__in.image.value) { + return null; + } else { + const {width, height} = this.__in.image.value; + return { + image: this.__in.image.value, + top: 0, + left: 0, + width, + height, + }; + } + } + + async initSegmenter() { + if (this.segmenter) { + return; + } + const model = SupportedModels.MediaPipeSelfieSegmentation; + const segmenterConfig = { + runtime: 'tfjs', + modelType: 'general' + } as MediaPipeSelfieSegmentationMediaPipeModelConfig; + this.segmenter = await createSegmenter(model, segmenterConfig); + } + + async segmentPeople() { + await this.initSegmenter(); + await waitForMedia(this.in.image.value); + if (!this.in.image.value) { + return; + } + + const originalImageCanvas = canvasPool2D.createCanvas(); + originalImageCanvas.width = this.in.width.value; + originalImageCanvas.height = this.in.height.value + originalImageCanvas.acquire(); + const originalCanvasCtx = originalImageCanvas.getContext('2d'); + + originalCanvasCtx.drawImage(this.in.image.value as CanvasImageSource | OffscreenCanvas, 0, 0); + + const segmentations = await this.segmenter.segmentPeople(this.in.image.value as BodySegmenterInput); + + const foregroundColor = {r: 0, g: 0, b: 0, a: 0}; + const backgroundColor = {r: 0, g: 0, b: 0, a: 255}; + const foregroundThreshold = this.in.maskBlurAmount.value; + const backgroundDarkeningMask = await toBinaryMask( + segmentations, + foregroundColor, + backgroundColor, + false, + foregroundThreshold + ); + + + const canvas = canvasPool2D.createCanvas(); + canvas.acquire(); + const ctx = canvas.getContext('2d'); + canvas.width = this.in.width.value; + canvas.height = this.in.height.value + + const maskBlurAmount = this.in.maskBlurAmount.value; + await drawMask(canvas, canvas as CanvasImageSource, backgroundDarkeningMask, 1, maskBlurAmount); + + this.out.image.value = canvas; + this.out.originalImage.value = originalImageCanvas; + this.out.width.value = this.in.width.value; + this.out.height.value = this.in.height.value; + } + + async _update(){ + await this.segmentPeople(); + } +} \ No newline at end of file diff --git a/src/nodes/index.ts b/src/nodes/index.ts index 1c46617..13c06bb 100644 --- a/src/nodes/index.ts +++ b/src/nodes/index.ts @@ -26,3 +26,4 @@ export {default as Font} from './Font'; export {default as Disable} from './Disable'; // export {default as Synchronize} from './Synchronize'; export {default as Api} from './Api'; +export {default as BodySegmentation} from './AI/BodySegmentation';