You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
331 lines
9.6 KiB
HTML
331 lines
9.6 KiB
HTML
2 months ago
|
<!DOCTYPE html>
|
||
|
|
<html lang="en">
|
||
|
|
<head>
|
||
|
|
<title>three.js webgl - morph targets - webcam</title>
|
||
|
|
<meta charset="utf-8">
|
||
|
|
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
|
||
|
|
<link type="text/css" rel="stylesheet" href="main.css">
|
||
|
|
<style>
|
||
|
|
body {
|
||
|
|
background-color: #666666;
|
||
|
|
}
|
||
|
|
</style>
|
||
|
|
</head>
|
||
|
|
<body>
|
||
|
|
|
||
|
|
<div id="info">
|
||
|
|
<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> webgl - morph targets - webcam<br/>
|
||
|
|
model by <a href="https://www.bannaflak.com/face-cap" target="_blank" rel="noopener">Face Cap</a>
|
||
|
|
</div>
|
||
|
|
|
||
|
|
<script type="importmap">
|
||
|
|
{
|
||
|
|
"imports": {
|
||
|
|
"three": "../build/three.module.js",
|
||
|
|
"three/addons/": "./jsm/"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
</script>
|
||
|
|
|
||
|
|
<script type="module">
|
||
|
|
|
||
|
|
import * as THREE from 'three';
|
||
|
|
|
||
|
|
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
|
||
|
|
import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js';
|
||
|
|
|
||
|
|
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
||
|
|
import { KTX2Loader } from 'three/addons/loaders/KTX2Loader.js';
|
||
|
|
import { MeshoptDecoder } from 'three/addons/libs/meshopt_decoder.module.js';
|
||
|
|
|
||
|
|
import { GUI } from 'three/addons/libs/lil-gui.module.min.js';
|
||
|
|
|
||
|
|
// Mediapipe
|
||
|
|
|
||
|
|
import vision from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0';
|
||
|
|
|
||
|
|
const { FaceLandmarker, FilesetResolver } = vision;
|
||
|
|
|
||
|
|
const blendshapesMap = {
|
||
|
|
// '_neutral': '',
|
||
|
|
'browDownLeft': 'browDown_L',
|
||
|
|
'browDownRight': 'browDown_R',
|
||
|
|
'browInnerUp': 'browInnerUp',
|
||
|
|
'browOuterUpLeft': 'browOuterUp_L',
|
||
|
|
'browOuterUpRight': 'browOuterUp_R',
|
||
|
|
'cheekPuff': 'cheekPuff',
|
||
|
|
'cheekSquintLeft': 'cheekSquint_L',
|
||
|
|
'cheekSquintRight': 'cheekSquint_R',
|
||
|
|
'eyeBlinkLeft': 'eyeBlink_L',
|
||
|
|
'eyeBlinkRight': 'eyeBlink_R',
|
||
|
|
'eyeLookDownLeft': 'eyeLookDown_L',
|
||
|
|
'eyeLookDownRight': 'eyeLookDown_R',
|
||
|
|
'eyeLookInLeft': 'eyeLookIn_L',
|
||
|
|
'eyeLookInRight': 'eyeLookIn_R',
|
||
|
|
'eyeLookOutLeft': 'eyeLookOut_L',
|
||
|
|
'eyeLookOutRight': 'eyeLookOut_R',
|
||
|
|
'eyeLookUpLeft': 'eyeLookUp_L',
|
||
|
|
'eyeLookUpRight': 'eyeLookUp_R',
|
||
|
|
'eyeSquintLeft': 'eyeSquint_L',
|
||
|
|
'eyeSquintRight': 'eyeSquint_R',
|
||
|
|
'eyeWideLeft': 'eyeWide_L',
|
||
|
|
'eyeWideRight': 'eyeWide_R',
|
||
|
|
'jawForward': 'jawForward',
|
||
|
|
'jawLeft': 'jawLeft',
|
||
|
|
'jawOpen': 'jawOpen',
|
||
|
|
'jawRight': 'jawRight',
|
||
|
|
'mouthClose': 'mouthClose',
|
||
|
|
'mouthDimpleLeft': 'mouthDimple_L',
|
||
|
|
'mouthDimpleRight': 'mouthDimple_R',
|
||
|
|
'mouthFrownLeft': 'mouthFrown_L',
|
||
|
|
'mouthFrownRight': 'mouthFrown_R',
|
||
|
|
'mouthFunnel': 'mouthFunnel',
|
||
|
|
'mouthLeft': 'mouthLeft',
|
||
|
|
'mouthLowerDownLeft': 'mouthLowerDown_L',
|
||
|
|
'mouthLowerDownRight': 'mouthLowerDown_R',
|
||
|
|
'mouthPressLeft': 'mouthPress_L',
|
||
|
|
'mouthPressRight': 'mouthPress_R',
|
||
|
|
'mouthPucker': 'mouthPucker',
|
||
|
|
'mouthRight': 'mouthRight',
|
||
|
|
'mouthRollLower': 'mouthRollLower',
|
||
|
|
'mouthRollUpper': 'mouthRollUpper',
|
||
|
|
'mouthShrugLower': 'mouthShrugLower',
|
||
|
|
'mouthShrugUpper': 'mouthShrugUpper',
|
||
|
|
'mouthSmileLeft': 'mouthSmile_L',
|
||
|
|
'mouthSmileRight': 'mouthSmile_R',
|
||
|
|
'mouthStretchLeft': 'mouthStretch_L',
|
||
|
|
'mouthStretchRight': 'mouthStretch_R',
|
||
|
|
'mouthUpperUpLeft': 'mouthUpperUp_L',
|
||
|
|
'mouthUpperUpRight': 'mouthUpperUp_R',
|
||
|
|
'noseSneerLeft': 'noseSneer_L',
|
||
|
|
'noseSneerRight': 'noseSneer_R',
|
||
|
|
// '': 'tongueOut'
|
||
|
|
};
|
||
|
|
|
||
|
|
//
|
||
|
|
|
||
|
|
const renderer = new THREE.WebGLRenderer( { antialias: true } );
|
||
|
|
renderer.setPixelRatio( window.devicePixelRatio );
|
||
|
|
renderer.setSize( window.innerWidth, window.innerHeight );
|
||
|
|
renderer.toneMapping = THREE.ACESFilmicToneMapping;
|
||
|
|
document.body.appendChild( renderer.domElement );
|
||
|
|
|
||
|
|
const camera = new THREE.PerspectiveCamera( 60, window.innerWidth / window.innerHeight, 1, 100 );
|
||
|
|
camera.position.z = 5;
|
||
|
|
|
||
|
|
const scene = new THREE.Scene();
|
||
|
|
scene.scale.x = - 1;
|
||
|
|
|
||
|
|
const environment = new RoomEnvironment();
|
||
|
|
const pmremGenerator = new THREE.PMREMGenerator( renderer );
|
||
|
|
|
||
|
|
scene.background = new THREE.Color( 0x666666 );
|
||
|
|
scene.environment = pmremGenerator.fromScene( environment ).texture;
|
||
|
|
|
||
|
|
const controls = new OrbitControls( camera, renderer.domElement );
|
||
|
|
|
||
|
|
// Face
|
||
|
|
|
||
|
|
let face, eyeL, eyeR;
|
||
|
|
const eyeRotationLimit = THREE.MathUtils.degToRad( 30 );
|
||
|
|
|
||
|
|
const ktx2Loader = new KTX2Loader()
|
||
|
|
.setTranscoderPath( 'jsm/libs/basis/' )
|
||
|
|
.detectSupport( renderer );
|
||
|
|
|
||
|
|
new GLTFLoader()
|
||
|
|
.setKTX2Loader( ktx2Loader )
|
||
|
|
.setMeshoptDecoder( MeshoptDecoder )
|
||
|
|
.load( 'models/gltf/facecap.glb', ( gltf ) => {
|
||
|
|
|
||
|
|
const mesh = gltf.scene.children[ 0 ];
|
||
|
|
scene.add( mesh );
|
||
|
|
|
||
|
|
const head = mesh.getObjectByName( 'mesh_2' );
|
||
|
|
head.material = new THREE.MeshNormalMaterial();
|
||
|
|
|
||
|
|
face = mesh.getObjectByName( 'mesh_2' );
|
||
|
|
eyeL = mesh.getObjectByName( 'eyeLeft' );
|
||
|
|
eyeR = mesh.getObjectByName( 'eyeRight' );
|
||
|
|
|
||
|
|
// GUI
|
||
|
|
|
||
|
|
const gui = new GUI();
|
||
|
|
gui.close();
|
||
|
|
|
||
|
|
const influences = head.morphTargetInfluences;
|
||
|
|
|
||
|
|
for ( const [ key, value ] of Object.entries( head.morphTargetDictionary ) ) {
|
||
|
|
|
||
|
|
gui.add( influences, value, 0, 1, 0.01 )
|
||
|
|
.name( key.replace( 'blendShape1.', '' ) )
|
||
|
|
.listen( influences );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
renderer.setAnimationLoop( animate );
|
||
|
|
|
||
|
|
} );
|
||
|
|
|
||
|
|
// Video Texture
|
||
|
|
|
||
|
|
const video = document.createElement( 'video' );
|
||
|
|
|
||
|
|
const texture = new THREE.VideoTexture( video );
|
||
|
|
texture.colorSpace = THREE.SRGBColorSpace;
|
||
|
|
|
||
|
|
const geometry = new THREE.PlaneGeometry( 1, 1 );
|
||
|
|
const material = new THREE.MeshBasicMaterial( { map: texture, depthWrite: false } );
|
||
|
|
const videomesh = new THREE.Mesh( geometry, material );
|
||
|
|
scene.add( videomesh );
|
||
|
|
|
||
|
|
// MediaPipe
|
||
|
|
|
||
|
|
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||
|
|
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm'
|
||
|
|
);
|
||
|
|
|
||
|
|
const faceLandmarker = await FaceLandmarker.createFromOptions( filesetResolver, {
|
||
|
|
baseOptions: {
|
||
|
|
modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
|
||
|
|
delegate: 'GPU'
|
||
|
|
},
|
||
|
|
outputFaceBlendshapes: true,
|
||
|
|
outputFacialTransformationMatrixes: true,
|
||
|
|
runningMode: 'VIDEO',
|
||
|
|
numFaces: 1
|
||
|
|
} );
|
||
|
|
|
||
|
|
if ( navigator.mediaDevices && navigator.mediaDevices.getUserMedia ) {
|
||
|
|
|
||
|
|
navigator.mediaDevices.getUserMedia( { video: { facingMode: 'user' } } )
|
||
|
|
.then( function ( stream ) {
|
||
|
|
|
||
|
|
video.srcObject = stream;
|
||
|
|
video.play();
|
||
|
|
|
||
|
|
} )
|
||
|
|
.catch( function ( error ) {
|
||
|
|
|
||
|
|
console.error( 'Unable to access the camera/webcam.', error );
|
||
|
|
|
||
|
|
} );
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
const transform = new THREE.Object3D();
|
||
|
|
|
||
|
|
function animate() {
|
||
|
|
|
||
|
|
if ( video.readyState >= HTMLMediaElement.HAVE_METADATA ) {
|
||
|
|
|
||
|
|
const results = faceLandmarker.detectForVideo( video, Date.now() );
|
||
|
|
|
||
|
|
if ( results.facialTransformationMatrixes.length > 0 ) {
|
||
|
|
|
||
|
|
const facialTransformationMatrixes = results.facialTransformationMatrixes[ 0 ].data;
|
||
|
|
|
||
|
|
transform.matrix.fromArray( facialTransformationMatrixes );
|
||
|
|
transform.matrix.decompose( transform.position, transform.quaternion, transform.scale );
|
||
|
|
|
||
|
|
const object = scene.getObjectByName( 'grp_transform' );
|
||
|
|
|
||
|
|
object.position.x = transform.position.x;
|
||
|
|
object.position.y = transform.position.z + 40;
|
||
|
|
object.position.z = - transform.position.y;
|
||
|
|
|
||
|
|
object.rotation.x = transform.rotation.x;
|
||
|
|
object.rotation.y = transform.rotation.z;
|
||
|
|
object.rotation.z = - transform.rotation.y;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
if ( results.faceBlendshapes.length > 0 ) {
|
||
|
|
|
||
|
|
const faceBlendshapes = results.faceBlendshapes[ 0 ].categories;
|
||
|
|
|
||
|
|
// Morph values does not exist on the eye meshes, so we map the eyes blendshape score into rotation values
|
||
|
|
const eyeScore = {
|
||
|
|
leftHorizontal: 0,
|
||
|
|
rightHorizontal: 0,
|
||
|
|
leftVertical: 0,
|
||
|
|
rightVertical: 0,
|
||
|
|
};
|
||
|
|
|
||
|
|
for ( const blendshape of faceBlendshapes ) {
|
||
|
|
|
||
|
|
const categoryName = blendshape.categoryName;
|
||
|
|
const score = blendshape.score;
|
||
|
|
|
||
|
|
const index = face.morphTargetDictionary[ blendshapesMap[ categoryName ] ];
|
||
|
|
|
||
|
|
if ( index !== undefined ) {
|
||
|
|
|
||
|
|
face.morphTargetInfluences[ index ] = score;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
// There are two blendshape for movement on each axis (up/down , in/out)
|
||
|
|
// Add one and subtract the other to get the final score in -1 to 1 range
|
||
|
|
switch ( categoryName ) {
|
||
|
|
|
||
|
|
case 'eyeLookInLeft':
|
||
|
|
eyeScore.leftHorizontal += score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookOutLeft':
|
||
|
|
eyeScore.leftHorizontal -= score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookInRight':
|
||
|
|
eyeScore.rightHorizontal -= score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookOutRight':
|
||
|
|
eyeScore.rightHorizontal += score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookUpLeft':
|
||
|
|
eyeScore.leftVertical -= score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookDownLeft':
|
||
|
|
eyeScore.leftVertical += score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookUpRight':
|
||
|
|
eyeScore.rightVertical -= score;
|
||
|
|
break;
|
||
|
|
case 'eyeLookDownRight':
|
||
|
|
eyeScore.rightVertical += score;
|
||
|
|
break;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
eyeL.rotation.z = eyeScore.leftHorizontal * eyeRotationLimit;
|
||
|
|
eyeR.rotation.z = eyeScore.rightHorizontal * eyeRotationLimit;
|
||
|
|
eyeL.rotation.x = eyeScore.leftVertical * eyeRotationLimit;
|
||
|
|
eyeR.rotation.x = eyeScore.rightVertical * eyeRotationLimit;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
videomesh.scale.x = video.videoWidth / 100;
|
||
|
|
videomesh.scale.y = video.videoHeight / 100;
|
||
|
|
|
||
|
|
renderer.render( scene, camera );
|
||
|
|
|
||
|
|
controls.update();
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
window.addEventListener( 'resize', function () {
|
||
|
|
|
||
|
|
camera.aspect = window.innerWidth / window.innerHeight;
|
||
|
|
camera.updateProjectionMatrix();
|
||
|
|
|
||
|
|
renderer.setSize( window.innerWidth, window.innerHeight );
|
||
|
|
|
||
|
|
} );
|
||
|
|
|
||
|
|
</script>
|
||
|
|
</body>
|
||
|
|
</html>
|