import * as scrawl from '../source/scrawl.js';
import { reportSpeed, initializeDomInputs } from './utilities.js';
import * as scrawl from '../source/scrawl.js';
import { reportSpeed, initializeDomInputs } from './utilities.js';
const canvas = scrawl.findCanvas('mycanvas');
Namespacing boilerplate
const namespace = canvas.name;
const name = (n) => `${namespace}-${n}`;
Build out the filters
scrawl.makeFilter({
name: name('grayscale'),
method: 'grayscale',
}).clone({
name: name('sepia'),
method: 'sepia',
}).clone({
name: name('invert'),
method: 'invert',
}).clone({
name: name('red'),
method: 'red',
});
scrawl.makeFilter({
name: name('pixelate'),
method: 'pixelate',
tileWidth: 20,
tileHeight: 20,
offsetX: 8,
offsetY: 8,
});
scrawl.makeFilter({
name: name('background-blur'),
method: 'gaussianBlur',
radius: 20,
});
scrawl.makeFilter({
name: name('body-blur'),
method: 'gaussianBlur',
radius: 10,
});
We’ll handle everything in a raw asset object, which a Picture entity can then use as its source
const myAsset = scrawl.makeRawAsset({
name: name('mediapipe-model-interpreter'),
userAttributes: [{
MediaPipe gives us imageData objects which we can drawImaghe into the RawAsset canvas element
key: 'mask',
defaultValue: [],
setter: function (item) {
item = (item.segmentationMask) ? item.segmentationMask : false;
if (item) {
/** @ts-expect-error */
this.canvasWidth = item.width;
/** @ts-expect-error */
this.canvasHeight = item.height;
/** @ts-expect-error */
this.mask = item;
/** @ts-expect-error */
this.dirtyData = true;
}
},
},{
key: 'canvasWidth',
defaultValue: 0,
setter: () => {},
},{
key: 'canvasHeight',
defaultValue: 0,
setter: () => {},
}],
updateSource: function (assetWrapper) {
const { element, engine, canvasWidth, canvasHeight, mask } = assetWrapper;
if (canvasWidth && canvasHeight && mask) {
Clear the canvas, resizing it if required
element.width = canvasWidth;
element.height = canvasHeight;
engine.drawImage(mask, 0, 0, canvasWidth, canvasHeight);
}
},
});
The forever loop function, which captures the MediaPipe model’s output and passes it on to our raw asset for processing
const perform = function (mask) {
myAsset.set({ mask });
if (!myOutline) {
Display the visual generated by our raw asset
myOutline = scrawl.makePicture({
name: name('outline'),
asset: name('mediapipe-model-interpreter'),
order: 0,
width: '100%',
height: '100%',
copyWidth: '80%',
copyHeight: '80%',
copyStartX: '10%',
copyStartY: '10%',
We blur here to make the outline merge into the background
filters: [name('body-blur')],
});
}
};
let video, model, myBackground, myOutline;
Capture the media stream
scrawl.importMediaStream({
name: name('device-camera'),
audio: false,
})
.then(mycamera => {
video = mycamera;
This fixes the issue in Firefox where the media stream will crash Tensorflow if the stream’s video element’s dimensions have not been set
/** @ts-expect-error */
video.source.width = "1280";
/** @ts-expect-error */
video.source.height = "720";
Take the media stream and display it in our canvas element
myBackground = scrawl.makePicture({
name: name('background'),
asset: mycamera.name,
order: 2,
width: '100%',
height: '100%',
copyWidth: '80%',
copyHeight: '80%',
copyStartX: '10%',
copyStartY: '10%',
visibility: false,
globalCompositeOperation: 'destination-over',
});
myBackground.clone({
name: name('body'),
order: 1,
visibility: true,
globalCompositeOperation: 'source-in',
});
Start the MediaPipe model
/* eslint-disable */
/** @ts-expect-error */
model = new SelfieSegmentation({
/* eslint-enable */
locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`
});
model.setOptions({ modelSelection: 1 });
model.onResults(perform);
Use MediaPipe’s camera functionality to get updates to the forever loop
/* eslint-disable */
/** @ts-expect-error */
const mediaPipeCamera = new Camera(video.source, {
/* eslint-enable */
onFrame: async () => {
await model.send({image: video.source});
},
width: 1280,
height: 720,
});
mediaPipeCamera.start();
})
.catch(err => console.log(err.message));
Function to display frames-per-second data, and other information relevant to the demo
const report = reportSpeed('#reportmessage');
Create the Display cycle animation
scrawl.makeRender({
name: name('animation'),
target: canvas,
afterShow: report,
});
initializeDomInputs([
['select', 'backgroundFilter', 0],
['select', 'outlineFilter', 1],
]);
Event listeners
scrawl.addNativeListener(['input', 'change'], (e) => {
e.preventDefault();
e.returnValue = false;
if (e && e.target) {
const id = e.target.id,
val = e.target.value;
if ('backgroundFilter' === id) {
myBackground.clearFilters();
if (val) {
myBackground.set({
visibility: true,
});
myBackground.addFilters(name(val));
}
else {
myBackground.set({
visibility: false,
});
}
}
else {
if ('1' === val) myOutline.addFilters(name('body-blur'));
else myOutline.clearFilters();
}
}
}, '.controlItem');
console.log(scrawl.library);