/* Fil-ready Mouth Synth. Generated from the source project scripts; keep as a classic script. */ window.fil = window.fil || { url: function(name) { return "assets/" + encodeURIComponent(name); } }; /* scripts/uimanager.js */ const uiManager = { font: undefined, messages: undefined, message: undefined, textColor: undefined, showFrameRate: undefined, refSize: undefined, preload: function (options) { this.font = p.loadFont(options.fontUrl); this.messages = {}; this.messages.loading = options?.messages?.loading ?? "loading..."; this.messages.welcome = options?.messages?.welcome ?? "welcome!"; this.messages.running = options?.messages?.running ?? ""; this.textColor = options?.textColor ?? "#fff"; this.showFrameRate = options?.showFrameRate ?? false; // message won't be displayed because // 1. font is not loaded yet // 2. message is displayed in draw() this.message = this.messages.loading; }, // todo: video takes even longer to load setup: function (canvasWidth, canvasHeight, options) { this.updateRefSize(canvasWidth, canvasHeight); }, updateRefSize: function (canvasWidth, canvasHeight) { const canvasShort = p.min(canvasWidth, canvasHeight); this.refSize = canvasShort; }, update: function (doneLoading, soundActivated, faceDetected) { if (doneLoading) this.message = this.messages.welcome; if (faceDetected && soundActivated) this.message = this.messages.running; }, display: function () { this.displayMessage(); }, displayMessage: function () { const textSize = this.refSize / (this.message.length + 1); p.textFont(this.font); p.push(); p.textSize(textSize); p.textAlign(p.CENTER, p.CENTER); p.fill(this.textColor); p.text(this.message, 0, 0); if (this.showFrameRate) { const frameRateTextSize = this.refSize / 16; p.textSize(frameRateTextSize); p.fill(255, 200); p.text(p.floor(p.frameRate()), 0, 100); } p.pop(); }, clearMessage() { this.message = ""; }, mousePressed() { this.clearMessage(); }, }; /* scripts/videomanager.js */ const videoManager = { capture: undefined, captureGraphics: undefined, pixelationSourceGraphics: undefined, pixelationShortSideNum: undefined, loaded: undefined, setup: function (canvasWidth, canvasHeight, options) { this.pixelationShortSideNum = options?.pixelationShortSideNum ?? 16; this.width = options?.width ?? 640; this.height = options?.height ?? 480; this.updateDimensions(canvasWidth, canvasHeight, options); this.loaded = false; this.capture = p.createCapture(options.video, { flipped: options.flipped }, () => { this.loaded = true; }); this.capture.hide(); }, update: function (options) { const flipVertical = options?.flipVertical ?? false; this.captureGraphics.push(); if (flipVertical) { this.captureGraphics.translate(this.captureGraphics.width, this.captureGraphics.height); this.captureGraphics.scale(-1, -1); } this.captureGraphics.image(this.capture, 0, 0, this.captureGraphics.width, this.captureGraphics.height, 0, 0, this.capture.width, this.capture.height, options.fit); this.captureGraphics.pop(); }, display: function (options) { p.push(); p.translate(-p.width / 2, -p.height / 2); this.imagePixelated(this.captureGraphics, this.pixelationSourceGraphics, options); p.pop(); }, updateDimensions: function (canvasWidth, canvasHeight, options) { if (this.captureGraphics === undefined) this.captureGraphics = p.createGraphics(this.width, this.height); this.captureGraphics.pixelDensity(1); let pixelationRowsNum; let pixelationColsNum; if (canvasWidth > canvasHeight) { pixelationRowsNum = Math.floor(this.pixelationShortSideNum); pixelationColsNum = Math.floor((this.pixelationShortSideNum * canvasWidth) / canvasHeight); } else { pixelationColsNum = Math.floor(this.pixelationShortSideNum); pixelationRowsNum = Math.floor((this.pixelationShortSideNum * canvasHeight) / canvasWidth); } this.pixelationSourceGraphics ? this.pixelationSourceGraphics.resizeCanvas(pixelationColsNum, pixelationRowsNum) : (this.pixelationSourceGraphics = p.createGraphics(pixelationColsNum, pixelationRowsNum)); this.pixelationSourceGraphics.pixelDensity(1); }, // draw image as pixelated image (use smaller graphics, this cause blurry edges) imagePixelated: function (sourceImg, tempPixelatedGraphics, options = {}) { // todo: this might be intialized in setup so it does not recreate new graphics every frame const rows = tempPixelatedGraphics.height; const cols = tempPixelatedGraphics.width; const waveformValues = options?.waveformValues ?? []; tempPixelatedGraphics.push(); tempPixelatedGraphics.image(sourceImg, 0, 0, tempPixelatedGraphics.width, tempPixelatedGraphics.height, 0, 0, sourceImg.width, sourceImg.height, p.COVER); tempPixelatedGraphics.loadPixels(); tempPixelatedGraphics.pop(); const colWidth = p.width / cols; const rowHeight = p.height / rows; p.background(options.backgroundColor); for (let c = 0; c < cols; c++) { for (let r = 0; r < rows; r++) { const x = c * colWidth + colWidth / 2; const y = r * rowHeight + rowHeight / 2; const index = 4 * (p.floor(tempPixelatedGraphics.width) * r + c); const valueRed = tempPixelatedGraphics.pixels[index]; const valueGreen = tempPixelatedGraphics.pixels[index + 1]; const valueBlue = tempPixelatedGraphics.pixels[index + 2]; const valueAlpha = tempPixelatedGraphics.pixels[index + 3]; const colorPixel = p.color(valueRed, valueGreen, valueBlue, valueAlpha); p.noStroke(); p.fill(colorPixel); // ellipses if (options.pixelationStyle === 1) { p.ellipseMode(p.CENTER); p.ellipse(x, y, p.ceil(colWidth), p.ceil(rowHeight)); // ellipses in varied sizes } else if (options.pixelationStyle === 2) { const factorSize = p.lightness(colorPixel) / colorPixel.maxes.hsl[2]; p.ellipseMode(p.CENTER); p.ellipse(x, y, p.ceil(colWidth * factorSize), p.ceil(rowHeight) * factorSize); // rectangles with varied corners } else if (options.pixelationStyle === 3) { const factorCorner = p.lightness(colorPixel) / colorPixel.maxes.hsl[2]; p.rectMode(p.CENTER); p.rect(x, y, p.ceil(colWidth), p.ceil(rowHeight), (factorCorner * p.min(colWidth, rowHeight)) / 2); // rectangles } else if (options.pixelationStyle === 4) { // customization const numFeaturesMax = options.numFeaturesMax ?? 16; const numVertices = 256; const amp = 0.3; const factorFeatures = p.lightness(colorPixel) / colorPixel.maxes.hsl[2]; const numFeatures = p.floor(numFeaturesMax * factorFeatures); const vertices = this.generateFlowerVertices(numFeatures, numVertices, amp); p.push(); p.translate(x, y); p.beginShape(); for (let vtx of vertices) { p.vertex(vtx.x * colWidth, vtx.y * rowHeight); } p.endShape(); p.pop(); } else if (options.pixelationStyle === 5) { // customization const numFeaturesMax = options.numFeaturesMax ?? 16; const numVertices = 64; const amp = 0.4; const factorFeatures = p.lightness(colorPixel) / colorPixel.maxes.hsl[2]; const numFeatures = p.floor(numFeaturesMax * factorFeatures); const vertices = this.generateStarVertices(numFeatures, numVertices, amp); p.push(); p.translate(x, y); p.beginShape(); for (let vtx of vertices) { p.vertex(vtx.x * colWidth, vtx.y * rowHeight); } p.endShape(); p.pop(); } else if (options.pixelationStyle === 6) { const minFactorSize = options?.minFactorSize ?? 0.2; const maxFactorSize = options?.maxFactorSize ?? 1; const waveformProgress = p.constrain(index / 4 / cols / rows, 0, 1); const waveformIndex = Math.floor(waveformProgress * waveformValues.length); const factorSize = p.constrain(p.map(waveformValues[waveformIndex], -0.02, 0.02, minFactorSize, maxFactorSize), minFactorSize, maxFactorSize); p.ellipseMode(p.CENTER); p.ellipse(x, y, p.ceil(colWidth * factorSize), p.ceil(rowHeight) * factorSize); } else { p.rectMode(p.CENTER); p.rect(x, y, p.ceil(colWidth), p.ceil(rowHeight)); } } } p.push(); p.rectMode(p.CORNER); p.fill(options.overlayColor); p.rect(0, 0, p.width, p.height); p.pop(); }, generateFlowerVertices: function (numPeriods, numVertices, amp) { const vertices = []; const angleOffest = 0; for (let k = 0; k < numVertices; k++) { const angle = (k / numVertices) * p.TWO_PI + angleOffest; const radius = p.map(p.sin(angle * numPeriods), -1, 1, 0.5 - amp / 2, 0.5); const vtx = p.createVector(radius * p.sin(angle), radius * p.cos(angle)); vertices.push(vtx); } return vertices; }, generateStarVertices: function (numPoints, numVertices, amp = 1) { const vertices = []; const angleOffset = 0; for (let k = 0; k < numVertices; k++) { const numSegments = numPoints * 2; const index = p.constrain(p.floor(k / p.floor(numVertices / numSegments)), 0, numSegments - 1); const radius = p.map(index % 2, 0, 1, 0.5 - amp / 2, 0.5); const angle = (index / numSegments) * p.TWO_PI + angleOffset; const vtx = p.createVector(radius * p.sin(angle), radius * p.cos(angle)); vertices.push(vtx); } return vertices; }, }; /* scripts/ml5manager.js */ const ml5Manager = { graphics: undefined, smoothness: undefined, loaded: undefined, faceMesh: undefined, faces: undefined, pfaces: undefined, detectedFaces: undefined, upperLipIndex: undefined, lowerLipIndex: undefined, lipLeftIndex: undefined, lipRightIndex: undefined, lipInteriorLength: undefined, lipsDistance: undefined, lipsRotation: undefined, lipsTipHeadingX: undefined, lipsTipHeadingY: undefined, lipsStatus: undefined, plipsStatus: undefined, lipsIndices: undefined, lipsIndicesExterior: undefined, lipsIndicesInterior: undefined, lipsCloseThreshold: undefined, lipsOpenThreshold: undefined, refSize: undefined, placeholderFacesIsActive: undefined, idleVolumeFactor: undefined, // placeholderFaces: undefined, placeholderFaces: { sequence: [], frameCount: 0, framesPerFace: 5, index: 0, getNextFrame: function () { this.frameCount++; this.index = Math.floor(this.frameCount / this.framesPerFace); if (this.index >= Object.keys(this.sequence).length) { this.index = 0; this.frameCount = 0; } return this.sequence[this.index]; }, }, preload: function (options) { this.placeholderFaces.sequence = p.loadJSON(options.placeholderFacesPath); // this.placeholderFaces = [p.loadJSON(options.placeholderFacePath)]; }, setup: function (canvasWidth, canvasHeight, options) { this.updateRefSize(canvasWidth, canvasHeight); this.lipsCloseThreshold = options?.lipsCloseThreshold ?? 8; this.lipsOpenThreshold = options?.lipsOpenThreshold ?? 16; this.idleVolumeFactor = options?.idleVolumeFactor ?? 0.5; // lips indices without repetition this.lipsIndices = [267, 312, 269, 311, 270, 310, 409, 415, 291, 308, 375, 324, 321, 318, 405, 402, 314, 317, 14, 17, 87, 84, 178, 181, 88, 91, 95, 146, 78, 61, 191, 185, 80, 40, 81, 39, 82, 37, 13, 0]; // lips contour this.lipsIndicesExterior = [267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146, 61, 185, 40, 39, 37, 0]; this.lipsIndicesInterior = [13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95, 78, 191, 80, 81, 82]; this.upperLipIndex = 13; this.lowerLipIndex = 14; this.lipLeftIndex = 78; this.lipRightIndex = 308; this.lipInteriorLength = Infinity; this.loaded = false; this.detectedFaces = []; this.faces = []; this.pfaces = []; this.lipsDistance = -Infinity; this.lipsRotation = 0; this.lipsStatus = "closed"; this.plipsStatus = "closed"; this.graphics = options.graphics; this.faceMesh = ml5.faceMesh(options.faceMesh, (faceMesh, error) => { if (error) { console.error(error); } else { this.loaded = true; faceMesh.detectStart(this.graphics, (results) => { this.updateFaces(results); }); } }); this.smoothness = options?.smoothness ?? 0.5; this.isDebugging = options?.isDebugging ?? false; }, updateFaces: function (faces) { this.detectedFaces = faces; if (faces.length < 1) { this.placeholderFacesIsActive = true; const placeholderFaces = this.placeholderFaces.getNextFrame(); faces = JSON.parse(JSON.stringify(placeholderFaces)); // faces = JSON.parse(JSON.stringify(this.placeholderFaces)); } else { this.placeholderFacesIsActive = false; } if (faces.length > 0 && this.faces.length > 0) { this.pfaces = this.faces; this.faces = this.smoothFaces(this.pfaces, faces, this.smoothness); } else if (faces.length > 0 && !this.faces.length > 0) { this.pfaces = faces; this.faces = this.smoothFaces(this.pfaces, faces, this.smoothness); } for (let face of this.faces) { face.lipsBox = this.getLipsBox(face.keypoints); face.lipsExterior = this.getKeypointsSubset(face.keypoints, this.lipsIndicesExterior); face.lipsInterior = this.getKeypointsSubset(face.keypoints, this.lipsIndicesInterior); if (this.isDebugging) this.drawKeypoints(face.keypoints); const upperLipKeypoint = face.keypoints[this.upperLipIndex]; const lowerLipKeypoint = face.keypoints[this.lowerLipIndex]; const upperLipVector3D = p.createVector(upperLipKeypoint.x, upperLipKeypoint.y, upperLipKeypoint.z); const lowerLipVector3D = p.createVector(lowerLipKeypoint.x, lowerLipKeypoint.y, lowerLipKeypoint.z); const lipsOpeningVector = upperLipVector3D.copy().sub(lowerLipVector3D); this.lipsDistance = lipsOpeningVector.mag(); this.lipsRotation = lipsOpeningVector.heading() + Math.PI; if (this.lipsStatus === "closed" && this.lipsDistance > this.lipsOpenThreshold) { this.lipsStatus = "open"; this.lipsOpened(); } else if (this.lipsStatus === "open" && this.lipsDistance > this.lipsCloseThreshold) { this.lipsMovedWhileOpen(); } else if (this.lipsStatus === "open" && this.lipsDistance < this.lipsCloseThreshold) { this.lipsStatus = "closed"; this.lipsClosed(); } const lipLeftKeypoint = face.keypoints[this.lipLeftIndex]; const lipRightKeypoint = face.keypoints[this.lipRightIndex]; const lipLeftVector3D = p.createVector(lipLeftKeypoint.x, lipLeftKeypoint.y, lipLeftKeypoint.z); const lipRightVector3D = p.createVector(lipRightKeypoint.x, lipRightKeypoint.y, lipRightKeypoint.z); const lipLengthVector = lipLeftVector3D.copy().sub(lipRightVector3D); this.lipInteriorLength = lipLengthVector.mag(); const lipsCenterReferenceVector = p5.Vector.lerp(lipLeftVector3D, lipRightVector3D, 0.5); const lipsTipVector = p5.Vector.lerp(upperLipVector3D, lowerLipVector3D, 0.5); const LipsCenterRefToLipsTipVector = lipsTipVector.copy().sub(lipsCenterReferenceVector); this.lipsTipHeadingX = p.createVector(LipsCenterRefToLipsTipVector.x, LipsCenterRefToLipsTipVector.z).heading() + Math.PI / 2; this.lipsTipHeadingY = p.createVector(LipsCenterRefToLipsTipVector.y, LipsCenterRefToLipsTipVector.z).heading() + Math.PI / 2; } }, update: function () {}, display: function (options) { const meter = options?.meter ?? 0; const blurriness = options?.blurriness ?? 0; for (let face of this.faces) { p.push(); const scaleX = (this.refSize / face.lipsBox.width) * options.lipsSize; const scaleY = (this.refSize / face.lipsBox.height) * options.lipsSize; const scale = scaleX > scaleY ? scaleY : scaleX; p.scale(scale); p.translate(-face.lipsBox.x, -face.lipsBox.y); if (blurriness > 0) { const glowingStrokeWeight = options?.glowingStrokeWeight ?? 0.02; const glowingStrokeColor = options?.glowingStrokeColor ?? options?.strokeColor ?? "#fff"; this.drawLips(face.lipsExterior, { ...options, strokeWeight: glowingStrokeWeight, strokeColor: glowingStrokeColor }); this.drawLips(face.lipsInterior, { ...options, strokeWeight: glowingStrokeWeight, strokeColor: glowingStrokeColor }); p.filter(p.BLUR, Math.round(blurriness * this.refSize)); } this.drawLips(face.lipsExterior, options); this.drawLips(face.lipsInterior, options); p.pop(); } }, smoothData: function (prevData, newData, prevScore, newScore, smoothness) { return (prevData * smoothness * prevScore + newData * (1 - smoothness) * newScore) / ((prevScore + newScore) / 2); }, // only smoothes lips smoothFaces: function (pfaces, faces, smoothness) { for (let i = 0; i < faces.length; i++) { const face = faces[i]; const pface = pfaces[i]; const score = 1; const pscore = 1; const keypoints = face.keypoints; const pkeypoints = pface.keypoints; for (let j of this.lipsIndices) { const keypoint = keypoints[j]; const pkeypoint = pkeypoints[j]; faces[i].keypoints[j].x = this.smoothData(pkeypoint.x, keypoint.x, pscore, score, smoothness); faces[i].keypoints[j].y = this.smoothData(pkeypoint.y, keypoint.y, pscore, score, smoothness); } } return faces; }, drawKeypoints: function (keypoints) { p.push(); p.translate(-p.width / 2, -p.height / 2); p.stroke(0, 255, 0); p.beginShape(p.POINTS); for (let i = 0; i < keypoints.length; i++) { const keypoint = keypoints[i]; p.vertex(keypoint.x, keypoint.y); p.textSize(10); p.fill(255); p.text(i, keypoint.x, keypoint.y); } p.endShape(); p.pop(); }, getKeypointsSubset: function (keypoints, indices) { const keypointsSubset = []; for (let index of indices) { const keypoint = keypoints[index]; keypointsSubset.push(keypoint); } return keypointsSubset; }, drawLips: function (keypoints, options) { p.push(); p.stroke(options.strokeColor); p.strokeWeight(options.strokeWeight * this.refSize); p.strokeJoin(p.ROUND); p.noFill(); p.beginShape(); for (let i = 0; i < keypoints.length + 3; i++) { const keypoint = keypoints[i % keypoints.length]; p.curveVertex(keypoint.x, keypoint.y); } p.endShape(); p.pop(); }, getLipsBox: function (keypoints) { const lipsBox = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity, x: 0, y: 0, width: 0, height: 0, }; for (let index of this.lipsIndices) { const keypoint = keypoints[index]; if (keypoint.x < lipsBox.minX) lipsBox.minX = keypoint.x; if (keypoint.y < lipsBox.minY) lipsBox.minY = keypoint.y; if (keypoint.x > lipsBox.maxX) lipsBox.maxX = keypoint.x; if (keypoint.y > lipsBox.maxY) lipsBox.maxY = keypoint.y; } lipsBox.width = lipsBox.maxX - lipsBox.minX; lipsBox.height = lipsBox.maxY - lipsBox.minY; lipsBox.x = lipsBox.minX + lipsBox.width / 2; lipsBox.y = lipsBox.minY + lipsBox.height / 2; return lipsBox; }, updateRefSize: function (canvasWidth, canvasHeight) { const canvasShort = p.min(canvasWidth, canvasHeight); this.refSize = canvasShort; }, getLipsRotationPercentage: function () { return this.lipsRotation / Math.PI; }, getLipsOpeningPercentage: function () { const lipsOpeningPercentage = p.constrain((this.lipsDistance / this.lipInteriorLength) * 2, 0, this.placeholderFacesIsActive ? this.idleVolumeFactor : 1); return lipsOpeningPercentage; }, getLipsTipHeadingXPercentage: function () { return this.lipsTipHeadingX / Math.PI; }, getLipsTipHeadingYPercentage: function () { return this.lipsTipHeadingY / Math.PI; }, lipsClosed: function (event) { console.log("lipsClosed"); }, lipsOpened: function (event) { console.log("lipsOpened"); }, lipsMovedWhileOpen: function (event) { console.log("lipsMovedWhileOpen"); }, hasDetectedFaces: function () { return this.detectedFaces.length > 0; }, }; /* scripts/soundmanager.js */ const soundManager = { synth: undefined, effect: undefined, notes: undefined, baseVolume: undefined, volumeAmount: undefined, startPercentage: undefined, detuneAmount: undefined, effectFrequencyAmount: undefined, harmonicityAmount: undefined, harmoncity: undefined, modulationAmount: undefined, modulationIndex: undefined, waveformSmoothness: undefined, smoothedWaveformValues: [], loaded: false, started: false, preload: function (options) { Tone.loaded().then(() => { this.loaded = true; }); }, setup: function (options) { const waveformSize = options?.waveform?.size ?? 256; this.waveformSmoothness = options?.waveform?.smoothness ?? 0.5; this.waveform = new Tone.Waveform(waveformSize); this.synth = new Tone.FMSynth(options.synth).toDestination(); this.synth.connect(this.waveform); this.baseVolume = options?.synth?.volume ?? -3; this.volumeAmount = options?.volumeAmount ?? 8; this.effect = new Tone.Vibrato(options.effect).toDestination(); // this.effect = new Tone.Tremolo(options.effect).toDestination(); this.synth.connect(this.effect); this.notes = options.noteIndices.map((index) => Tone.Frequency(options.midiScale[index], "midi")); this.detuneAmount = options?.detuneAmount ?? 256; this.effectFrequencyAmount = options?.effectFrequencyAmount ?? 8; this.harmoncity = options?.synth?.harmoncity ?? 3; this.harmonicityAmount = options?.harmonicityAmount ?? 16; this.modulationIndex = options?.modulationIndex ?? 10; this.modulationAmount = options?.modulationAmount ?? 10; }, mousePressed: function () { if (this.loaded === true && this.started !== true) { Tone.start().then(() => { this.started = true; Tone.Transport.start(); }); } }, getElementAtPercentage: function (elements, percentage) { const constrainedPercentage = p.constrain(percentage, 0, 1); const index = Math.floor(constrainedPercentage * (this.notes.length - 1)); return elements[index]; }, triggerStart: function (percentage = 0.1) { this.startPercentage = percentage; const note = this.getElementAtPercentage(this.notes, percentage); if (this.started) this.synth.triggerAttack(note); }, triggerEnd: function () { if (this.started) this.synth.triggerRelease(); }, changeVolume: function (percentage = 1) { const constrainedPercentage = p.constrain(percentage, 0, 1); // volume gain must be negative const volumeGain = -Math.abs(this.volumeAmount * p.log(constrainedPercentage)); const volume = this.baseVolume + volumeGain; this.synth.volume.value = volume; }, detune: function (percentage = 0) { const percentageOffset = percentage - this.startPercentage; const newDetune = percentageOffset * this.detuneAmount; this.synth.set({ detune: newDetune, }); }, changeEffectFrequency: function (percentage = 0) { const percentageOffset = percentage - this.startPercentage; const newFrequency = Math.abs(percentageOffset * this.effectFrequencyAmount) + 1; this.effect.set({ frequency: newFrequency, }); }, moveHarmonicity: function (percentage = 0) { const newHarmonicity = Math.abs(this.harmoncity + percentage * this.harmonicityAmount); this.synth.set({ harmonicity: newHarmonicity, }); }, changeModulationIndex: function (percentage = 0) { const newModulationIndex = Math.abs(this.modulationIndex - percentage * this.modulationAmount); this.synth.set({ modulationIndex: newModulationIndex, }); }, getWaveformValues: function () { const waveformValues = this.waveform?.getValue() ?? []; for (let i = 0; i < waveformValues.length; i++) { if (this.smoothedWaveformValues[i]) { this.smoothedWaveformValues[i] = p.lerp(waveformValues[i], this.smoothedWaveformValues[i], this.waveformSmoothness); } else { this.smoothedWaveformValues[i] = waveformValues[i]; } } return this.smoothedWaveformValues; }, }; /* scripts/sketch.js */ /* Main Sketch * Jack B. Du (github@jackbdu.com) * https://instagram.com/jackbdu/ */ const sketch = (p) => { p.specs = { fps: 60, colorBackground: 0, outputWidth: "auto", outputHeight: "auto", }; p.options = { exhibit: { reloadTotalMillis: 3600000, reloadAbsenceMillis: 10000, }, sound: { midiScale: [60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84], noteIndices: [0, 2, 4, 5, 7, 9, 11, 12, 14, 16, 17, 19, 21, 23, 24], // major 2 octaves detuneAmount: 512, effectFrequencyAmount: 16, harmonicityAmount: 16, modulationAmount: 64, volumeAmount: 16, effect: { frequency: 5, depth: 1, }, synth: { harmonicity: 3, modulationIndex: 10, envelope: { attack: 0.1, decay: 0.1, sustain: 1, release: 1, }, modulationEnvelope: { attack: 0.1, decay: 0.1, sustain: 1, release: 1, }, volume: -16, }, waveform: { smoothness: 0.7, size: 128, }, }, ml5: { lipsCloseThreshold: 4, lipsOpenThreshold: 8, idleVolumeFactor: 0.5, smoothness: 0.75, // isDebugging: true, facemesh: { maxFaces: 1, refineLandmarks: false, flipHorizontal: false, }, strokeWeight: 0.01, strokeColor: "#fff", glowingStrokeWeight: 0.015, glowingStrokeColor: "#fff", lipsSize: 0.75, // placeholderFacePath: fil.url("neutral-face.json"), placeholderFacesPath: fil.url("demo-faces.json"), blurriness: 0, }, video: { width: 480, height: 480, pixelationShortSideNum: 24, pixelationStyle: 6, flipped: true, fit: p.COVER, backgroundColor: "#0000", overlayColor: [0, 100], minFactorSize: 0.3, maxFactorSize: 0.9, }, ui: { fontUrl: fil.url("Ubuntu-Bold.ttf"), messages: { loading: "Loading...".toUpperCase(), welcome: "Click here to activate audio".toUpperCase(), running: "", }, textColor: "#fff", // showFrameRate: true, }, }; p.uiManager = uiManager; p.ml5Manager = ml5Manager; p.videoManager = videoManager; p.soundManager = soundManager; p.preload = () => { p.uiManager.preload(p.options.ui); p.ml5Manager.preload(p.options.ml5); p.soundManager.preload(p.options.sound); }; p.setup = () => { p.updateCanvas(p.specs.outputWidth, p.specs.outputHeight); p.frameRate(p.specs.fps); p.smooth(); p.videoManager.setup(p.width, p.height, { ...p.options.video, video: p.VIDEO }); p.ml5Manager.setup(p.width, p.height, { ...p.options.ml5, graphics: p.videoManager.captureGraphics }); p.uiManager.setup(p.width, p.height, p.options.ui); p.soundManager.setup(p.options.sound); p.ml5Manager.lipsOpened = () => { p.soundManager.triggerStart(p.ml5Manager.getLipsRotationPercentage()); }; p.ml5Manager.lipsClosed = () => { p.soundManager.triggerEnd(); }; p.ml5Manager.lipsMovedWhileOpen = () => { p.soundManager.changeVolume(p.ml5Manager.getLipsOpeningPercentage()); p.soundManager.detune(p.ml5Manager.getLipsRotationPercentage()); p.soundManager.changeEffectFrequency(p.ml5Manager.getLipsRotationPercentage()); p.soundManager.moveHarmonicity(p.ml5Manager.getLipsTipHeadingXPercentage()); p.soundManager.changeModulationIndex(p.ml5Manager.getLipsTipHeadingYPercentage()); }; if (p.specs.exhibit) { p.soundManager.mousePressed(); p.noCursor(); } }; p.mousePressed = () => { p.soundManager.mousePressed(); }; p.touchStarted = () => { p.soundManager.mousePressed(); }; p.beforeDraw = () => { if (p.beginCapture) p.beginCapture(); p.videoManager.update(p.options.video); p.uiManager.update(p.videoManager.loaded && p.ml5Manager.loaded && p.soundManager.loaded, p.soundManager.started, p.ml5Manager.faces.length > 0); p.background(p.specs.colorBackground); }; p.draw = () => { if (p.beforeDraw) p.beforeDraw(); p.videoManager.display({ ...p.options.video, waveformValues: p.soundManager.getWaveformValues() }); p.ml5Manager.display(p.options.ml5); p.uiManager.display(); if (p.afterDraw) p.afterDraw(); }; p.afterDraw = () => { if (p.endCapture) p.endCapture(); if (p.specs.exhibit) { const millis = p.millis(); if (millis > p.options.exhibit.reloadTotalMillis && !p.ml5Manager.hasDetectedFaces()) { if (!p.absenceMillis) { p.absenceMillis = millis; } if (millis - p.absenceMillis > p.options.exhibit.reloadAbsenceMillis) { window.location.reload(); } } else { p.absenceMillis = undefined; } } }; p.windowResized = () => { p.updateCanvas(p.specs.outputWidth, p.specs.outputHeight); p.uiManager.updateRefSize(p.width, p.height); p.ml5Manager.updateRefSize(p.width, p.height); p.videoManager.updateDimensions(p.width, p.height); }; p.updateCanvas = (outputWidth = "auto", outputHeight = "auto") => { const pd = p.pixelDensity(); const canvasWidth = outputWidth && outputWidth !== "auto" ? outputWidth / pd : p.windowWidth; const canvasHeight = outputHeight && outputHeight !== "auto" ? outputHeight / pd : p.windowHeight; if (canvasWidth !== p.width || canvasHeight !== p.height) { if (!p.hasCanvas) { p.createCanvas(canvasWidth, canvasHeight, p.WEBGL); p.hasCanvas = true; } else { p.resizeCanvas(canvasWidth, canvasHeight); } } }; }; let p = new p5(sketch); // https://stackoverflow.com/questions/901115/how-can-i-get-query-string-values-in-javascript const params = new Proxy(new URLSearchParams(window.location.search), { get: (searchParams, prop) => searchParams.get(prop), }); if (params.width && parseInt(params.width) > 0) p.specs.outputWidth = params.width; if (params.height && parseInt(params.height) > 0) p.specs.outputHeight = params.height; if (params.flipvertical) { p.options.video.flipVertical = params.flipvertical === "true"; } if (params.exhibit) { p.specs.exhibit = params.exhibit === "true"; if (p.specs.exhibit) { p.options.ui.messages = { loading: "", welcome: "", running: "", }; } } if (params.theme) { p.specs.theme = params.theme; document.body.classList.add(p.specs.theme); switch (p.specs.theme) { case "light": p.options.ml5.strokeColor = "#000"; p.options.ml5.glowingStrokeColor = "#000"; p.options.video.overlayColor = [255, 100]; p.options.ui.textColor = "#000"; break; case "dark": p.options.ml5.strokeColor = "#fff"; p.options.ml5.glowingStrokeColor = "#fff"; p.options.video.overlayColor = [0, 100]; p.options.ui.textColor = "#fff"; break; } }