关于前端:web端实现AR人脸特效

前言
直播、短视频、在线会议等利用越来越多地进入人们的生存，随之诞生的是丰盛的各类创意玩法与陈腐体验，其中大量利用了以 AI 检测和图形渲染为根底的 AR 技术。

而随着 Web 技术的一直成熟，AR 技术在 Web 上的实现成为了一种可能。明天就总结了在 Web 端实现此性能的几个技术要点，跟大家一起探讨一下。

架构和概念
形象整体的实现思路如下

调取 Camera 取得相机画面应用 tensorflow 加载人脸识别模型生成 FaceMesh 依据 FaceMesh 生成三角网格并进行 UV 贴图
FaceMesh
MediaPipe Face Mesh 是一种脸部几何解决方案，即便在挪动设施上，也能够实时预计 468 个 3D 脸部界标。它采纳 机器学习（ML）来推断 3D 外表几何形态，只须要单个摄像机输出，而无需专用的深度传感器。该解决方案利用轻量级的模型架构以及整个管线中的 GPU 减速，可提供对实时体验至关重要的实时性能。

UVMap
UV 是二维纹理坐标，U 代表程度方向，V 代表垂直方向。UV Map 用来形容三维物体外表与图像纹理(Texture) 的映射关系，有了 UV Map，咱们就能够将二维的图像纹理粘贴到三维的物体外表。

image.png

矩形贴图和球面的映射图
技术实现
调取 Camera 取得相机画面
通过 navigator.mediaDevices.getUserMedia 获取 stream，放到 video 查看。

async function setupWebcam() {

return new Promise(( resolve, reject) => {const webcamElement = document.getElementById( "webcam");
    const navigatorAny = navigator;
    navigator.getUserMedia = navigator.getUserMedia ||
    navigatorAny.webkitGetUserMedia || navigatorAny.mozGetUserMedia ||
    navigatorAny.msGetUserMedia;
    if(navigator.getUserMedia) {navigator.getUserMedia( { video: true},
            stream => {
                webcamElement.srcObject = stream;
                webcamElement.addEventListener("loadeddata", resolve, false);
            },
        error => reject());
    }
    else {reject();
    }
});

}
复制代码
人脸识别
// 创立模型
createModel() {

return new Promise(async resolve => {await tf.setBackend('webgl')
    const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
    const detectorConfig = {
        maxFaces: 1, // 检测到的最大面部数量
        refineLandmarks: true, // 能够欠缺眼睛和嘴唇四周的地标坐标，并在虹膜四周输入其余地标
        runtime: 'mediapipe',
        solutionPath: 'https://unpkg.com/@mediapipe/face_mesh', //WASM 二进制文件和模型文件所在的门路
    };
    this.model = await faceLandmarksDetection.createDetector(model, detectorConfig);
    resolve(this.model);
})

},
// 辨认
async recognition() {

try {
    const video = this.$refs.video;
    const faces = await this.model.estimateFaces(video, {flipHorizontal: false, // 镜像});
    if (faces.length > 0) {const keypoints = faces[0].keypoints;
        this.render3D({scaledMesh:keypoints.reduce((acc, pos) =>{acc.push([pos.x,pos.y,pos.z])
                return acc
            }, [])
        });
    }else{this.render3D({scaledMesh:[]})
    }
} catch (error) {console.log(error);
}

}
复制代码
3D 场景贴图
TRIANGULATION
UV_COORDS

//3D 场景
const scene = new THREE.Scene();

// 增加一些光照
scene.add(new THREE.AmbientLight( 0xcccccc, 0.4) );
camera.add(new THREE.PointLight( 0xffffff, 0.8) );

// 正交相机
scene camera = new THREE.PerspectiveCamera(45, 1, 0.1, 2000);
camera.position.x = videoWidth / 2;
camera.position.y = -videoHeight / 2;
camera.position.z = -(videoHeight / 2) / Math.tan(45 / 2)
scene.add(camera); 

// 渲染器
const renderer = new THREE.WebGLRenderer({canvas: document.getElementById( "overlay"),
    alpha: true
});

// 创立 geometry，将 468 集体脸特色点依照肯定的程序 (TRIANGULATION) 组成三角网格，并加载 UV_COORDS
const geometry = new THREE.BufferGeometry()
geometry.setIndex(TRIANGULATION)
geometry.setAttribute('uv', new THREE.Float32BufferAttribute(UV_COORDS.map((item, index) => index % 2 ? item : 1 - item), 2))
geometry.computeVertexNormals()
            
// 创立 material
const textureLoader = new THREE.TextureLoader();
const meshImg = this.meshList[meshIndex].src;// 材质图片地址
textureLoader.load(meshImg,texture=>{
    texture.encoding = THREE.sRGBEncoding
    texture.anisotropy = 16
    const material = new THREE.MeshBasicMaterial({
        map: texture,
        transparent: true,
        color: new THREE.Color(0xffffff),
        reflectivity: 0.5
    });
    const mesh = new THREE.Mesh(geometry, material)
    scene.add(mesh)
})
// 依据 face mesh 实时更新 geometry
updateGeometry(prediction){
    let w = canvasWidth;
    let h = canvasWidth;
    const faceMesh = resolveMesh(prediction.scaledMesh, w, h)
    const positionBuffer = faceMesh.reduce((acc, pos) => acc.concat(pos), [])
    geometry.setAttribute('position', new THREE.Float32BufferAttribute(positionBuffer, 3))
    geometry.attributes.position.needsUpdate = true
}

resolveMesh(faceMesh, vw, vh){

   return faceMesh.map(p => [p[0] - vw / 2, vh / 2 - p[1], -p[2]])

}

// 渲染
render3D(prediction){

    if (prediction) {updateGeometry(prediction)
    }
    renderer.render(scene, threeCamera)
}

复制代码
加载 3D 模型
// 加载 3D 模型
const loader = new GLTFLoader();
const Object3D = new THREE.Object3D();
loader.load(modelUrl, (gltf) => {

const object = gltf.scene
const box = new THREE.Box3().setFromObject(object)
const size = box.getSize(new THREE.Vector3()).length()
const center = box.getCenter(new THREE.Vector3())
object.position.x += (object.position.x - center.x);
object.position.y += (object.position.y - center.y + 1);
object.position.z += (object.position.z - center.z - 15);
Object3D.add(object)
this.scene.add(Object3D)

})

// 计算 Matrix
const position = prediction.midwayBetweenEyes[0]
const scale = this.getScale(prediction.scaledMesh, 234, 454)
const rotation = this.getRotation(prediction.scaledMesh, 10, 50, 280)
object.position.set(…position)
object.scale.setScalar(scale / 20)
object.scale.x *= -1
object.rotation.setFromRotationMatrix(rotation)
object.rotation.y = -object.rotation.y
object.rotateZ(Math.PI)
object.rotateX(-Math.PI * .05)
if (this.morphTarget) {

// flipped
this.morphTarget['leftEye'] && this.morphTarget['leftEye'](1 - prediction.faceRig.eye.r)
this.morphTarget['rightEye'] && this.morphTarget['rightEye'](1 - prediction.faceRig.eye.l)
this.morphTarget['mouth'] && this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)

}