Image analysis and Face Emotion

In this article I will show you a procedure for using tensorflow.js applied to facial emotion recognition. The goal is to provide a JavaScript workflow to use pretrained models useful for this purpose. In this example we will use a very interesting library, face-api.js, which makes it possible to handle face recognition inside a photo or a video frame.

For the purposes of the article and to help with your experiments I leave the complete example and source code here, while below you can check the result directly online. Rest assured no information will be stored — you can allow webcam access, everything runs on your client… and in any case avoid making ugly faces ?

As already mentioned, for this project we will use TensorFlow; for those who don’t know it, TensorFlow is an open source software library developed by Google specialized in machine learning/deep learning. I must say that the power of TensorFlow is even more appreciable thanks to the various released models that allow us to easily create sophisticated “intelligent” systems in a short time. In this specific case we will use a very interesting model: MobileNetV1.
Compared to other similar models, MobileNet performs well in terms of both latency and accuracy. MobileNets, in fact, are a family of mobile-first computer vision models designed to effectively maximize these parameters, because they take into account the limited resources for an application on mobile or web devices.

MobileNets can be used for image classification, detection, and segmentation tasks.

let scoreThreshold = 0.5
var EmotionModel;
var emotion_labels = [“angry”, “disgust”, “fear”, “happy”, “sad”, “surprise”, “neutral”];
var emotion_colors = [“#ff0000”, “#00a800”, “#ff4fc1”, “#ffe100”, “#306eff”, “#ff9d00”, “#7c7c7c”];
var offset_x = 27;
var offset_y = 20;

// START VIDEO
startVideo();

// LOAD FACE-API FOR FACE DETECTION
function loadFaceApi(){
jQuery.getScript(‘/js/FaceEmotion/face-api.js’, function(){
console.log(‘loaded FaceApi’);
setTimeout(function(){
startFaceEmotion();
},4000);
});
}

function startVideo() {
// use MediaDevices API
// docs:
// https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia

var video = document.querySelector(‘video’),canvas;
webcamElement=video;
console.log(“navigator ” + navigator.mediaDevices)
if (navigator.mediaDevices) {
// access the web cam

navigator.mediaDevices.getUserMedia({ video: true })
// permission granted:
.then(function(stream) {
//video.src = window.URL.createObjectURL(stream);
video.srcObject = stream;
loadFaceApi();
})
// permission denied:
.catch(function(error) {
document.body.textContent = ‘Could not access the camera.Error:’ + error.name;
});
}
}

// START LOADING THE MODEL FOR EMOTION RECOGNITION
function startFaceEmotion(){
loadModel(‘/js/FaceEmotion/models/mobilenetv1_models/model.json’);
}

// load emotion model
async function loadModel(path) {
EmotionModel = await tf.loadLayersModel(path)
console.log(EmotionModel);
load_modello_di_riconoscimento();
}

async function load_modello_di_riconoscimento() {
//const Model_url = ‘/js/FaceEmotion/models/tiny_face_detector/tiny_face_detector_model-weights_manifest.json’
const Model_url = ‘/js/FaceEmotion/models/tiny_face_detector/’
await faceapi.loadTinyFaceDetectorModel(Model_url);
console.log(“FaceEmotion modelLoaded”);
var video = document.querySelector(‘video’),canvas;
start_riconoscimento(video);
}

async function start_riconoscimento(videoEl) {

const {
width,
height
} = faceapi.getMediaDimensions(videoEl)

var canvas = canvas || document.createElement(‘canvas’);
canvas.width = width
canvas.height = height

const forwardParams = {
inputSize: parseInt(160),
scoreThreshold
}

const result = await faceapi.detectAllFaces(videoEl, new faceapi.TinyFaceDetectorOptions(forwardParams))
// IF I FIND FACES I PERFORM THE EMOTION CHECK

if (result.length != 0) {

const context = canvas.getContext(‘2d’)
context.drawImage(videoEl, 0, 0, width, height)
let ctx = context;
for (var i = 0; i < result.length; i++) {
ctx.beginPath();
var item = result[i].box;
let s_x = Math.floor(item._x+offset_x);
if (item.y start_riconoscimento(videoEl));
}

async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}

function preprocess(imgData) {
return tf.tidy(() => {
let tensor = tf.browser.fromPixels(imgData).toFloat();
tensor = tensor.resizeBilinear([100, 100])
tensor = tf.cast(tensor, ‘float32’)
const offset = tf.scalar(255.0);
// Normalize the image
const normalized = tensor.div(offset);
//We add a dimension to get a batch shape
const batched = normalized.expandDims(0)
return batched
})
}

function successCallback(stream) {
var videoEl = $(‘#inputVideo’).get(0)
videoEl.srcObject = stream;
}

function errorCallback(error) {
alert(error)
console.log(“navigator.getUserMedia error: “, error);
}

function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}

#emotionContainer{
font-size: 2em;
padding:20px;
text-align: center;

}
#emotion{
color: brown;
}

To help understand the code a bit, here are some notes. The first is: when developing with tensorflow.js you must pay close attention to the versions used. Many of the errors you may get actually stem from an incorrect choice of versions.

Okay, let’s start describing the code a little.

In the first block I do nothing more than create the web container to manage the webcam and import the first libraries necessary for our purpose. We will not dwell on it further, as it is basic HTML.

<html>

    <head>
       <!-- Load TensorFlow.js. This is required to use MobileNet. -->
       <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@1.2.1"/> 
       <!-- Load the MobileNet model. -->
       <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/mobilenet@1.0.0"/> 
       <script
          src="https://code.jquery.com/jquery-3.4.0.min.js"
          integrity="sha256-BJeo0qm959uMBGb65z40ejJYGSgR7REI4+CW1fNKwOg="
          crossorigin="anonymous">
        </script>
    </head>

    <body>
        <div id="emotionContainer">
            <video autoplay="true" width="640" height="480" id="videoElement"></video>
            <p>My mood: <span id ="emotion">Please wait a moment</span></p>
        </div>
    </body>
</html>
<script>
.
.
.

let scoreThreshold = 0.5
var EmotionModel;
var emotion_labels = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"];
var emotion_colors = ["#ff0000", "#00a800", "#ff4fc1", "#ffe100", "#306eff", "#ff9d00", "#7c7c7c"];
var offset_x = 27;
var offset_y = 20;

// START VIDEO

startVideo();

function startVideo() {
    // use MediaDevices API
    // docs:
    // https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
    var video = document.querySelector('video'),canvas;
    webcamElement=video;
    console.log("navigator  " + navigator.mediaDevices)
    if (navigator.mediaDevices) {
        // access the web cam
        navigator.mediaDevices.getUserMedia({ video: true })
            // permission granted:
           .then(function(stream) {
            //video.src = window.URL.createObjectURL(stream);
             video.srcObject = stream;
             loadFaceApi();
            })
            // permission denied:
            .catch(function(error) {
                document.body.textContent = 'Could not access the camera.Error:' + error.name;
            });

    }

}

// LOAD FACE-API FOR FACE DETECTION
 function loadFaceApi(){
   jQuery.getScript('/js/FaceEmotion/face-api.js', function(){
   console.log('loaded FaceApi');
   setTimeout(function(){
     loadTrainedModelEmotion();
   },4000); 
 });
 }

The facial model is loaded by invoking an asynchronous function so, as I said, it is important to manage the loading order well. At the end of the model loading I will start face recognition which we will use to capture emotions.

// START LOADING THE MODEL FOR EMOTION RECOGNITION
async function loadTrainedModelEmotion(){
 path='/js/FaceEmotion/models/mobilenetv1_models/model.json';
 EmotionModel = await tf.loadLayersModel(path)
 console.log(EmotionModel);
 load_modello_di_riconoscimento_facciale().then(function(){
  var video = document.querySelector('video'),canvas;
  start_riconoscimento(video);
 });
}

async function load_modello_di_riconoscimento_facciale() {
 //const Model_url = '/js/FaceEmotion/models/tiny_face_detector/tiny_face_detector_model-weights_manifest.json'
 const Model_url = '/js/FaceEmotion/models/tiny_face_detector/'
 await faceapi.loadTinyFaceDetectorModel(Model_url);
 console.log("FaceEmotion modelLoaded");
}

async function start_riconoscimento(videoEl) {

const {
 width,
 height
 } = faceapi.getMediaDimensions(videoEl)

var canvas = canvas || document.createElement('canvas');
 canvas.width = width
 canvas.height = height

const forwardParams = {
 inputSize: parseInt(160),
 scoreThreshold
 }

const result = await faceapi.detectAllFaces(videoEl, new faceapi.TinyFaceDetectorOptions(forwardParams))
 // IF I FIND FACES I PERFORM THE EMOTION CHECK
 
 if (result.length != 0) {
  const context = canvas.getContext('2d')
  context.drawImage(videoEl, 0, 0, width, height)
  let ctx = context;
  for (var i = 0; i < result.length; i++) {
  ctx.beginPath();
  var item = result[i].box;
  let s_x = Math.floor(item._x+offset_x);
  if (item.y<offset_y){
  var s_y = Math.floor(item._y);
 }
 else{
  var s_y = Math.floor(item._y-offset_y);
 }
 let s_w = Math.floor(item._width-offset_x);
 let s_h = Math.floor(item._height);
 let cT = ctx.getImageData(s_x, s_y, s_w, s_h);
 cT = preprocess(cT);
 z = EmotionModel.predict(cT)
 if(z != "undefined" && z != null)
 {
  let index = z.argMax(1).dataSync()[0]
  let label = emotion_labels[index];
  var emotionEl = document.getElementById('emotion');
  emotionEl.innerHTML = label;
  console.log("result: "+label);
 }

}
 await sleep(500);
 }

async function loadNetWeights(uri) {
 return new Float32Array(await (await fetch(uri)).arrayBuffer())
}

function preprocess(imgData) {
 return tf.tidy(() => {
 let tensor = tf.browser.fromPixels(imgData).toFloat();
 tensor = tensor.resizeBilinear([100, 100])
 tensor = tf.cast(tensor, 'float32')
 const offset = tf.scalar(255.0);
 // Normalize the image 
 const normalized = tensor.div(offset);
 //We add a dimension to get a batch shape 
 const batched = normalized.expandDims(0)
 return batched
 })
}

.
.
.
function sleep(ms) {
 return new Promise(resolve => setTimeout(resolve, ms));
}

.
.
.

</script>
<style>
#emotionContainer{
    font-size: 2em;
    padding:20px;
    text-align: center;
}

#emotion{
    color: brown;
}
</style>