face feature position

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "graphfx",
-  "version": "0.4.0",
+  "version": "0.4.4",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -3806,6 +3806,11 @@
         "@jridgewell/sourcemap-codec": "1.4.14"
       }
     },
+    "@mediapipe/face_detection": {
+      "version": "0.4.1646425229",
+      "resolved": "https://registry.npmjs.org/@mediapipe/face_detection/-/face_detection-0.4.1646425229.tgz",
+      "integrity": "sha512-aeCN+fRAojv9ch3NXorP6r5tcGVLR3/gC1HmtqB0WEZBRXrdP6/3W/sGR0dHr1iT6ueiK95G9PVjbzFosf/hrg=="
+    },
     "@mediapipe/selfie_segmentation": {
       "version": "0.1.1675465747",
       "resolved": "https://registry.npmjs.org/@mediapipe/selfie_segmentation/-/selfie_segmentation-0.1.1675465747.tgz",
@@ -3843,6 +3848,14 @@
         "rimraf": "^3.0.2"
       }
     },
+    "@tensorflow-models/face-detection": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@tensorflow-models/face-detection/-/face-detection-1.0.1.tgz",
+      "integrity": "sha512-oIEEqHy4qbkGn5xT1ldzwxK5s948+t7ts0+WoIfTPKwScpNa8YY4GJf+fpyhtOxUGGAZymXx25jrvF9YlAt9dg==",
+      "requires": {
+        "rimraf": "^3.0.2"
+      }
+    },
     "@tensorflow/tfjs-backend-cpu": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-4.2.0.tgz",

package.json

@@ -29,8 +29,10 @@
     "typescript": "^3.7.3"
   },
   "dependencies": {
+    "@mediapipe/face_detection": "^0.4.1646425229",
     "@mediapipe/selfie_segmentation": "^0.1.1675465747",
     "@tensorflow-models/body-segmentation": "^1.0.1",
+    "@tensorflow-models/face-detection": "^1.0.1",
     "@tensorflow/tfjs-backend-webgl": "^4.2.0",
     "@tensorflow/tfjs-converter": "^4.2.0",
     "@tensorflow/tfjs-core": "^4.2.0",

src/nodes/AI/FaceFeaturePosition.ts

@@ -0,0 +1,133 @@
+import {
+    ImageVar,
+    NumberVar
+} from '../io/AbstractIOSet';
+import Node from "../Node";
+import '@tensorflow/tfjs-backend-webgl';
+import {
+    Face,
+    FaceDetector,
+    FaceDetectorInput,
+    SupportedModels,
+    createDetector,
+    MediaPipeFaceDetectorTfjsModelConfig
+} from '@tensorflow-models/face-detection';
+import {canvasPool2D} from "../../canvas/CanvasPool";
+import {waitForMedia} from "../../utils";
+import { mediaSize } from '../canvas';
+import {first} from 'lodash/array';
+import keyBy from 'lodash/keyBy';
+
+const inputs = {
+    image: {
+        type: 'Image'
+    } as ImageVar,
+}
+
+const outputs = {
+    image: {
+        type: 'Image'
+    } as ImageVar,
+    rightEyeX: {
+        type: 'Number',
+    } as NumberVar,
+    rightEyeY: {
+        type: 'Number',
+    } as NumberVar,
+    leftEyeX: {
+        type: 'Number',
+    } as NumberVar,
+    leftEyeY: {
+        type: 'Number',
+    } as NumberVar,
+    noseTipX: {
+        type: 'Number',
+    } as NumberVar,
+    noseTipY: {
+        type: 'Number',
+    } as NumberVar,
+    mouthCenterX: {
+        type: 'Number',
+    } as NumberVar,
+    mouthCenterY: {
+        type: 'Number',
+    } as NumberVar,
+    rightEarTragionX: {
+        type: 'Number',
+    } as NumberVar,
+    rightEarTragionY: {
+        type: 'Number',
+    } as NumberVar,
+    leftEarTragionX: {
+        type: 'Number',
+    } as NumberVar,
+    leftEarTragionY: {
+        type: 'Number',
+    } as NumberVar,
+}
+
+export default class FaceFeaturePosition extends Node<typeof inputs, typeof outputs> {
+    private detector: FaceDetector
+
+    constructor(options={}) {
+        super('FaceFeaturePosition', inputs, outputs);
+    }
+
+    async initDetector() {
+        if (this.detector) {
+            return;
+        }
+        const model = SupportedModels.MediaPipeFaceDetector;
+        const detectorConfig = {
+            maxFaces: 1,
+            modelType: 'short',
+            runtime: 'tfjs',
+        } as MediaPipeFaceDetectorTfjsModelConfig;
+        this.detector = await createDetector(model, detectorConfig);
+    }
+
+    async detectFeatures() {
+        await this.initDetector();
+        await waitForMedia(this.in.image.value);
+        if (!this.in.image.value) {
+            return;
+        }
+        const {width, height} = mediaSize(this.__in.image.value);
+
+        const originalImageCanvas = canvasPool2D.createCanvas();
+        originalImageCanvas.width = width;
+        originalImageCanvas.height = height;
+        originalImageCanvas.acquire();
+        const originalCanvasCtx = originalImageCanvas.getContext('2d');
+
+        originalCanvasCtx.drawImage(this.in.image.value as CanvasImageSource | OffscreenCanvas, 0, 0);
+
+        const detections = await this.detector.estimateFaces(this.in.image.value as FaceDetectorInput, {flipHorizontal: false});
+
+        if (!detections.length) {
+            this.out.image.value = this.in.image.value;
+            return;
+        }
+
+        const detection: Face = first(detections);
+        const keyPoints = keyBy(detection.keypoints, (keyPoint) => keyPoint.name)
+
+        this.out.image.value = this.in.image.value;
+        this.out.rightEyeX.value = keyPoints['rightEye'].x;
+        this.out.rightEyeY.value = keyPoints['rightEye'].y;
+        this.out.leftEyeX.value = keyPoints['leftEye'].x;
+        this.out.leftEyeY.value = keyPoints['leftEye'].y;
+        this.out.noseTipX.value = keyPoints['noseTip'].x;
+        this.out.noseTipY.value = keyPoints['noseTip'].y;
+        this.out.mouthCenterX.value = keyPoints['mouthCenter'].x;
+        this.out.mouthCenterY.value = keyPoints['mouthCenter'].y;
+        this.out.rightEarTragionX.value = keyPoints['rightEarTragion'].x;
+        this.out.rightEarTragionY.value = keyPoints['rightEarTragion'].y;
+        this.out.leftEarTragionX.value = keyPoints['leftEarTragion'].x;
+        this.out.leftEarTragionY.value = keyPoints['leftEarTragion'].y;
+    }
+
+    async _update(){
+        await this.detectFeatures();
+    }
+}

src/nodes/index.ts

@@ -28,3 +28,4 @@ export {default as Disable} from './Disable';
 // export {default as Synchronize} from './Synchronize';
 export {default as Api} from './Api';
 export {default as BodySegmentation} from './AI/BodySegmentation';
+export {default as FaceFeaturePosition} from './AI/FaceFeaturePosition';