import * as scrawl from '../source/scrawl.js';
import { reportSpeed, initializeDomInputs } from './utilities.js';
import * as scrawl from '../source/scrawl.js';
import { reportSpeed, initializeDomInputs } from './utilities.js';
const canvas = scrawl.findCanvas('mycanvas');
Namespacing boilerplate
const namespace = canvas.name;
const name = (n) => `${namespace}-${n}`;
Magic numbers
const width = 1280,
height = 720;
Create and manipulate the face worm let wormStart = 220, wormEnd = 270;
const dom = initializeDomInputs([
['input', 'wormstart', '220'],
['input', 'wormend', '270'],
]);
const worm = scrawl.makePolyline({
name: name('face-worm'),
strokeStyle: 'red',
lineJoin: 'round',
lineCap: 'round',
lineWidth: 3,
method: 'draw',
mapToPins: true,
tension: 0.3,
});
const updateLabelsAndWorm = function (asset) {
const { entitys, mesh } = asset
if (mesh.length && Array.isArray(mesh[0])) {
const face = mesh[0];
We only need to create the labels once
if (!entitys.length) {
face.forEach((coord, index) => {
entitys.push(scrawl.makeLabel({
name: name(`label-${index}`),
text: `${index}`,
handle: ['center', 'center'],
fontString: '10px Arial',
textIsAccessible: false,
}));
});
}
Update label coordinates with new mesh data
entitys.forEach((e, index) => {
const coord = face[index];
const {x, y} = coord;
e.set({
startX: `${x * 100}%`,
startY: `${y * 100}%`,
});
});
let wormStart = parseInt(dom.wormstart.value, 10),
wormEnd = parseInt(dom.wormend.value, 10);
Check for perilous user input
if (!Number.isFinite(wormStart)) wormStart = 0;
if (wormStart < 0) wormStart = 0;
if (wormStart > 468) wormStart = 468;
if (!Number.isFinite(wormEnd)) wormEnd = 0;
if (wormEnd < 0) wormEnd = 0;
if (wormEnd > 468) wormEnd = 468;
if (wormEnd < wormStart) {
const temp = wormStart;
wormStart = wormEnd;
wormEnd = temp;
}
dom.wormstart.value = `${wormStart}`;
dom.wormend.value = `${wormEnd}`;
Update the worm’s pins
worm.set({
pins: entitys.slice(wormStart, wormEnd),
});
}
};
const myAsset = scrawl.makeRawAsset({
name: name('mediapipe-model-interpreter'),
userAttributes: [{
key: 'mesh',
defaultValue: false,
setter: function (item) {
if (item) {
const {image:img, multiFaceLandmarks:mesh} = item;
if (img) {
/** @ts-expect-error */
this.canvasWidth = img.width;
/** @ts-expect-error */
this.canvasHeight = img.height;
}
/** @ts-expect-error */
if (mesh) this.mesh = mesh;
/** @ts-expect-error */
this.dirtyData = true;
}
},
},{
key: 'entitys',
defaultValue: [],
setter: () => {},
},{
key: 'canvasWidth',
defaultValue: 0,
setter: () => {},
},{
key: 'canvasHeight',
defaultValue: 0,
setter: () => {},
}],
updateSource: function (assetWrapper) {
const { mesh } = assetWrapper;
if (mesh && mesh.length) updateLabelsAndWorm(this);
},
});
The forever loop function, which captures the MediaPipe model’s output and passes it on to our raw asset for processing
const perform = function (mesh) {
myAsset.set({ mesh });
To get the raw asset working, something needs to subscribe to it - even though we’re not using the asset to output any graphics in this demo. We only need to create the Picture entity once.
if (!output) output = scrawl.makePicture({
name: name('output'),
asset: name('mediapipe-model-interpreter'),
});
};
let video, model, output;
Capture the media stream
scrawl.importMediaStream({
name: name('device-camera'),
audio: false,
})
.then(mycamera => {
video = mycamera;
/** @ts-expect-error */
video.source.width = width;
/** @ts-expect-error */
video.source.height = height;
scrawl.makePicture({
name: 'background',
asset: mycamera.name,
dimensions: ['100%', '100%'],
copyDimensions: ['100%', '100%'],
globalAlpha: 0.2,
});
Start the MediaPipe model
/* eslint-disable */
/** @ts-expect-error */
model = new FaceMesh({
/* eslint-enable */
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`,
});
model.setOptions({
maxNumFaces: 1,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5
});
model.onResults(perform);
Use MediaPipe’s camera functionality to get updates to the forever loop
/* eslint-disable */
/** @ts-expect-error */
const mediaPipeCamera = new Camera(video.source, {
/* eslint-enable */
onFrame: async () => {
await model.send({image: video.source});
},
width,
height,
});
mediaPipeCamera.start();
})
.catch(err => console.log(err.message));
Function to display frames-per-second data, and other information relevant to the demo
const report = reportSpeed('#reportmessage');
Create the Display cycle animation
scrawl.makeRender({
name: name('animation'),
target: canvas,
afterShow: report,
});
console.log(scrawl.library);