From 432e0d1dcd6a9148c54683c97f4eacbec8388de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Kol=C3=A1=C5=99?= Date: Mon, 24 Apr 2023 16:24:42 +0200 Subject: [PATCH] face feature position --- package-lock.json | 15 +++- package.json | 2 + src/nodes/AI/FaceFeaturePosition.ts | 133 ++++++++++++++++++++++++++++ src/nodes/index.ts | 1 + 4 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 src/nodes/AI/FaceFeaturePosition.ts diff --git a/package-lock.json b/package-lock.json index 6c02202..3c28fb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "graphfx", - "version": "0.4.0", + "version": "0.4.4", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -3806,6 +3806,11 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "@mediapipe/face_detection": { + "version": "0.4.1646425229", + "resolved": "https://registry.npmjs.org/@mediapipe/face_detection/-/face_detection-0.4.1646425229.tgz", + "integrity": "sha512-aeCN+fRAojv9ch3NXorP6r5tcGVLR3/gC1HmtqB0WEZBRXrdP6/3W/sGR0dHr1iT6ueiK95G9PVjbzFosf/hrg==" + }, "@mediapipe/selfie_segmentation": { "version": "0.1.1675465747", "resolved": "https://registry.npmjs.org/@mediapipe/selfie_segmentation/-/selfie_segmentation-0.1.1675465747.tgz", @@ -3843,6 +3848,14 @@ "rimraf": "^3.0.2" } }, + "@tensorflow-models/face-detection": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@tensorflow-models/face-detection/-/face-detection-1.0.1.tgz", + "integrity": "sha512-oIEEqHy4qbkGn5xT1ldzwxK5s948+t7ts0+WoIfTPKwScpNa8YY4GJf+fpyhtOxUGGAZymXx25jrvF9YlAt9dg==", + "requires": { + "rimraf": "^3.0.2" + } + }, "@tensorflow/tfjs-backend-cpu": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.2.0.tgz", diff --git a/package.json b/package.json index 4f5b2e6..cb6c430 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,10 @@ "typescript": "^3.7.3" }, "dependencies": { + "@mediapipe/face_detection": "^0.4.1646425229", "@mediapipe/selfie_segmentation": "^0.1.1675465747", "@tensorflow-models/body-segmentation": "^1.0.1", + "@tensorflow-models/face-detection": "^1.0.1", "@tensorflow/tfjs-backend-webgl": "^4.2.0", "@tensorflow/tfjs-converter": "^4.2.0", "@tensorflow/tfjs-core": "^4.2.0", diff --git a/src/nodes/AI/FaceFeaturePosition.ts b/src/nodes/AI/FaceFeaturePosition.ts new file mode 100644 index 0000000..f8f27c9 --- /dev/null +++ b/src/nodes/AI/FaceFeaturePosition.ts @@ -0,0 +1,133 @@ +import { + ImageVar, + NumberVar +} from '../io/AbstractIOSet'; +import Node from "../Node"; +import '@tensorflow/tfjs-backend-webgl'; +import { + Face, + FaceDetector, + FaceDetectorInput, + SupportedModels, + createDetector, + MediaPipeFaceDetectorTfjsModelConfig +} from '@tensorflow-models/face-detection'; +import {canvasPool2D} from "../../canvas/CanvasPool"; +import {waitForMedia} from "../../utils"; +import { mediaSize } from '../canvas'; +import {first} from 'lodash/array'; +import keyBy from 'lodash/keyBy'; + +const inputs = { + image: { + type: 'Image' + } as ImageVar, +} + +const outputs = { + image: { + type: 'Image' + } as ImageVar, + rightEyeX: { + type: 'Number', + } as NumberVar, + rightEyeY: { + type: 'Number', + } as NumberVar, + leftEyeX: { + type: 'Number', + } as NumberVar, + leftEyeY: { + type: 'Number', + } as NumberVar, + noseTipX: { + type: 'Number', + } as NumberVar, + noseTipY: { + type: 'Number', + } as NumberVar, + mouthCenterX: { + type: 'Number', + } as NumberVar, + mouthCenterY: { + type: 'Number', + } as NumberVar, + rightEarTragionX: { + type: 'Number', + } as NumberVar, + rightEarTragionY: { + type: 'Number', + } as NumberVar, + leftEarTragionX: { + type: 'Number', + } as NumberVar, + leftEarTragionY: { + type: 'Number', + } as NumberVar, +} + +export default class FaceFeaturePosition extends Node { + private detector: FaceDetector + + constructor(options={}) { + super('FaceFeaturePosition', inputs, outputs); + } + + async initDetector() { + if (this.detector) { + return; + } + const model = SupportedModels.MediaPipeFaceDetector; + const detectorConfig = { + maxFaces: 1, + modelType: 'short', + runtime: 'tfjs', + } as MediaPipeFaceDetectorTfjsModelConfig; + this.detector = await createDetector(model, detectorConfig); + } + + async detectFeatures() { + await this.initDetector(); + await waitForMedia(this.in.image.value); + if (!this.in.image.value) { + return; + } + const {width, height} = mediaSize(this.__in.image.value); + + const originalImageCanvas = canvasPool2D.createCanvas(); + originalImageCanvas.width = width; + originalImageCanvas.height = height; + originalImageCanvas.acquire(); + const originalCanvasCtx = originalImageCanvas.getContext('2d'); + + originalCanvasCtx.drawImage(this.in.image.value as CanvasImageSource | OffscreenCanvas, 0, 0); + + const detections = await this.detector.estimateFaces(this.in.image.value as FaceDetectorInput, {flipHorizontal: false}); + + if (!detections.length) { + this.out.image.value = this.in.image.value; + return; + } + + const detection: Face = first(detections); + const keyPoints = keyBy(detection.keypoints, (keyPoint) => keyPoint.name) + + this.out.image.value = this.in.image.value; + this.out.rightEyeX.value = keyPoints['rightEye'].x; + this.out.rightEyeY.value = keyPoints['rightEye'].y; + this.out.leftEyeX.value = keyPoints['leftEye'].x; + this.out.leftEyeY.value = keyPoints['leftEye'].y; + this.out.noseTipX.value = keyPoints['noseTip'].x; + this.out.noseTipY.value = keyPoints['noseTip'].y; + this.out.mouthCenterX.value = keyPoints['mouthCenter'].x; + this.out.mouthCenterY.value = keyPoints['mouthCenter'].y; + this.out.rightEarTragionX.value = keyPoints['rightEarTragion'].x; + this.out.rightEarTragionY.value = keyPoints['rightEarTragion'].y; + this.out.leftEarTragionX.value = keyPoints['leftEarTragion'].x; + this.out.leftEarTragionY.value = keyPoints['leftEarTragion'].y; + } + + async _update(){ + await this.detectFeatures(); + } +} \ No newline at end of file diff --git a/src/nodes/index.ts b/src/nodes/index.ts index 6e94166..2f3e273 100644 --- a/src/nodes/index.ts +++ b/src/nodes/index.ts @@ -28,3 +28,4 @@ export {default as Disable} from './Disable'; // export {default as Synchronize} from './Synchronize'; export {default as Api} from './Api'; export {default as BodySegmentation} from './AI/BodySegmentation'; +export {default as FaceFeaturePosition} from './AI/FaceFeaturePosition';