Add a model for facial expression recognition (#100)

README.md

@@ -0,0 +1,40 @@
+
+# Progressive Teacher
+
+Progressive Teacher: [Boosting Facial Expression Recognition by A Semi-Supervised Progressive Teacher](https://scholar.google.com/citations?view_op=view_citation&hl=zh-CN&user=OCwcfAwAAAAJ&citation_for_view=OCwcfAwAAAAJ:u5HHmVD_uO8C)
+
+Note:
+- Progressive Teacher is contributed by [Jing Jiang](https://scholar.google.com/citations?user=OCwcfAwAAAAJ&hl=zh-CN).
+-  [MobileFaceNet](https://link.springer.com/chapter/10.1007/978-3-319-97909-0_46) is used as the backbone and the model is able to classify seven basic facial expressions (angry, disgust, fearful, happy, neutral, sad, surprised).
+- [facial_expression_recognition_mobilefacenet_2022july.onnx](https://github.com/opencv/opencv_zoo/raw/master/models/facial_expression_recognition/facial_expression_recognition_mobilefacenet_2022july.onnx) is implemented thanks to [Chengrui Wang](https://github.com/opencv).
+
+Results of accuracy evaluation on [RAF-DB](http://whdeng.cn/RAF/model1.html).
+
+| Models      | Accuracy | 
+|-------------|----------|
+| Progressive Teacher       | 88.27%  |
+
+
+## Demo
+
+***NOTE***: This demo uses [../face_detection_yunet](../face_detection_yunet) as face detector, which supports 5-landmark detection for now (2021sep).
+
+Run the following command to try the demo:
+```shell
+# recognize the facial expression on images
+python demo.py --input /path/to/image
+```
+
+### Example outputs
+
+Note: Zoom in to to see the recognized facial expression in the top-left corner of each face boxes.
+
+![fer demo](./examples/selfie.jpg)
+
+## License
+
+All files in this directory are licensed under [Apache 2.0 License](./LICENSE).
+
+## Reference
+
+- https://ieeexplore.ieee.org/abstract/document/9629313

demo.py

@@ -0,0 +1,131 @@
+import sys
+import argparse
+import copy
+import datetime
+
+import numpy as np
+import cv2 as cv
+
+from facial_fer_model import FacialExpressionRecog
+
+sys.path.append('../face_detection_yunet')
+from yunet import YuNet
+
+
+def str2bool(v):
+    if v.lower() in ['on', 'yes', 'true', 'y', 't']:
+        return True
+    elif v.lower() in ['off', 'no', 'false', 'n', 'f']:
+        return False
+    else:
+        raise NotImplementedError
+
+
+backends = [cv.dnn.DNN_BACKEND_OPENCV, cv.dnn.DNN_BACKEND_CUDA]
+targets = [cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_CUDA, cv.dnn.DNN_TARGET_CUDA_FP16]
+help_msg_backends = "Choose one of the computation backends: {:d}: OpenCV implementation (default); {:d}: CUDA"
+help_msg_targets = "Chose one of the target computation devices: {:d}: CPU (default); {:d}: CUDA; {:d}: CUDA fp16"
+try:
+    backends += [cv.dnn.DNN_BACKEND_TIMVX]
+    targets += [cv.dnn.DNN_TARGET_NPU]
+    help_msg_backends += "; {:d}: TIMVX"
+    help_msg_targets += "; {:d}: NPU"
+except:
+    print('This version of OpenCV does not support TIM-VX and NPU. Visit https://github.com/opencv/opencv/wiki/TIM-VX-Backend-For-Running-OpenCV-On-NPU for more information.')
+
+parser = argparse.ArgumentParser(description='Facial Expression Recognition')
+parser.add_argument('--input', '-i', type=str, help='Path to the input image. Omit for using default camera.')
+parser.add_argument('--model', '-fm', type=str, default='./facial_expression_recognition_mobilefacenet_2022july.onnx', help='Path to the facial expression recognition model.')
+parser.add_argument('--backend', '-b', type=int, default=backends[0], help=help_msg_backends.format(*backends))
+parser.add_argument('--target', '-t', type=int, default=targets[0], help=help_msg_targets.format(*targets))
+parser.add_argument('--save', '-s', type=str, default=False, help='Set true to save results. This flag is invalid when using camera.')
+parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.')
+args = parser.parse_args()
+
+
+def visualize(image, det_res, fer_res, box_color=(0, 255, 0), text_color=(0, 0, 255)):
+
+    print('%s %3d faces detected.' % (datetime.datetime.now(), len(det_res)))
+
+    output = image.copy()
+    landmark_color = [
+        (255,  0,   0),  # right eye
+        (0,    0, 255),  # left eye
+        (0,  255,   0),  # nose tip
+        (255,  0, 255),  # right mouth corner
+        (0,  255, 255)   # left mouth corner
+    ]
+
+    for ind, (det, fer_type) in enumerate(zip(det_res, fer_res)):
+        bbox = det[0:4].astype(np.int32)
+        fer_type = FacialExpressionRecog.getDesc(fer_type)
+        print("Face %2d: %d %d %d %d %s." % (ind, bbox[0], bbox[1], bbox[0]+bbox[2], bbox[1]+bbox[3], fer_type))
+        cv.rectangle(output, (bbox[0], bbox[1]), (bbox[0]+bbox[2], bbox[1]+bbox[3]), box_color, 2)
+        cv.putText(output, fer_type, (bbox[0], bbox[1]+12), cv.FONT_HERSHEY_DUPLEX, 0.5, text_color)
+        landmarks = det[4:14].astype(np.int32).reshape((5, 2))
+        for idx, landmark in enumerate(landmarks):
+            cv.circle(output, landmark, 2, landmark_color[idx], 2)
+    return output
+
+
+def process(detect_model, fer_model, frame):
+    h, w, _ = frame.shape
+    detect_model.setInputSize([w, h])
+    dets = detect_model.infer(frame)
+
+    if dets is None:
+        return False, None, None
+
+    fer_res = np.zeros(0, dtype=np.int8)
+    for face_points in dets:
+        fer_res = np.concatenate((fer_res, fer_model.infer(frame, face_points[:-1])), axis=0)
+    return True, dets, fer_res
+
+
+if __name__ == '__main__':
+    detect_model = YuNet(modelPath='../face_detection_yunet/face_detection_yunet_2022mar.onnx')
+
+    fer_model = FacialExpressionRecog(modelPath=args.model,
+                                      backendId=args.backend,
+                                      targetId=args.target)
+
+    # If input is an image
+    if args.input is not None:
+        image = cv.imread(args.input)
+
+        # Get detection and fer results
+        status, dets, fer_res = process(detect_model, fer_model, image)
+
+        if status:
+            # Draw results on the input image
+            image = visualize(image, dets, fer_res)
+
+        # Save results
+        if args.save:
+            cv.imwrite('result.jpg', image)
+            print('Results saved to result.jpg\n')
+
+        # Visualize results in a new window
+        if args.vis:
+            cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE)
+            cv.imshow(args.input, image)
+            cv.waitKey(0)
+    else:  # Omit input to call default camera
+        deviceId = 0
+        cap = cv.VideoCapture(deviceId)
+
+        while cv.waitKey(1) < 0:
+            hasFrame, frame = cap.read()
+            if not hasFrame:
+                print('No frames grabbed!')
+                break
+
+            # Get detection and fer results
+            status, dets, fer_res = process(detect_model, fer_model, frame)
+
+            if status:
+                # Draw results on the input image
+                frame = visualize(frame, dets, fer_res)
+
+            # Visualize results in a new window
+            cv.imshow('FER Demo', frame)

facial_fer_model.py

@@ -0,0 +1,178 @@
+# This file is part of OpenCV Zoo project.
+# It is subject to the license terms in the LICENSE file found in the same directory.
+#
+# Copyright (C) 2022, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved.
+# Third party copyrights are property of their respective owners.
+
+import numpy as np
+import cv2 as cv
+
+class FacialExpressionRecog:
+    def __init__(self, modelPath, backendId=0, targetId=0):
+        self._modelPath = modelPath
+        self._backendId = backendId
+        self._targetId = targetId
+
+        self._model = cv.dnn.readNet(self._modelPath)
+        self._model.setPreferableBackend(self._backendId)
+        self._model.setPreferableTarget(self._targetId)
+
+        self._align_model = FaceAlignment()
+
+        self._inputNames = 'data'
+        self._outputNames = ['label']
+        self._inputSize = [112, 112]
+        self._mean = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
+        self._std = np.array([0.5, 0.5, 0.5])[np.newaxis, np.newaxis, :]
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    def setBackend(self, backend_id):
+        self._backendId = backend_id
+        self._model.setPreferableBackend(self._backendId)
+
+    def setTarget(self, target_id):
+        self._targetId = target_id
+        self._model.setPreferableTarget(self._targetId)
+
+    def _preprocess(self, image, bbox):
+        if bbox is not None:
+            image = self._align_model.get_align_image(image, bbox[4:].reshape(-1, 2))
+        image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
+        image = image.astype(np.float32, copy=False) / 255.0
+        image -= self._mean
+        image /= self._std
+        return cv.dnn.blobFromImage(image)
+
+    def infer(self, image, bbox=None):
+        # Preprocess
+        inputBlob = self._preprocess(image, bbox)
+
+        # Forward
+        self._model.setInput(inputBlob, self._inputNames)
+        outputBlob = self._model.forward(self._outputNames)
+
+        # Postprocess
+        results = self._postprocess(outputBlob)
+
+        return results
+
+    def _postprocess(self, outputBlob):
+        result = np.argmax(outputBlob[0], axis=1).astype(np.uint8)
+        return result
+
+    @staticmethod
+    def getDesc(ind):
+        _expression_enum = ["angry", "disgust", "fearful", "happy", "neutral", "sad", "surprised"]
+        return _expression_enum[ind]
+
+
+class FaceAlignment():
+    def __init__(self, reflective=False):
+        self._std_points = np.array([[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366], [41.5493, 92.3655], [70.7299, 92.2041]])
+        self.reflective = reflective
+
+    def __tformfwd(self, trans, uv):
+        uv = np.hstack((uv, np.ones((uv.shape[0], 1))))
+        xy = np.dot(uv, trans)
+        xy = xy[:, 0:-1]
+        return xy
+
+    def __tforminv(self, trans, uv):
+        Tinv = np.linalg.inv(trans)
+        xy = self.__tformfwd(Tinv, uv)
+        return xy
+
+    def __findNonreflectiveSimilarity(self, uv, xy, options=None):
+        options = {"K": 2}
+
+        K = options["K"]
+        M = xy.shape[0]
+        x = xy[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
+        y = xy[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
+        # print '--->x, y:\n', x, y
+
+        tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
+        tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
+        X = np.vstack((tmp1, tmp2))
+        # print '--->X.shape: ', X.shape
+        # print 'X:\n', X
+
+        u = uv[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
+        v = uv[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
+        U = np.vstack((u, v))
+        # print '--->U.shape: ', U.shape
+        # print 'U:\n', U
+
+        # We know that X * r = U
+        if np.linalg.matrix_rank(X) >= 2 * K:
+            r, _, _, _ = np.linalg.lstsq(X, U, rcond=-1)
+            # print(r, X, U, sep="\n")
+            r = np.squeeze(r)
+        else:
+            raise Exception("cp2tform:twoUniquePointsReq")
+
+        sc = r[0]
+        ss = r[1]
+        tx = r[2]
+        ty = r[3]
+
+        Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]])
+        T = np.linalg.inv(Tinv)
+        T[:, 2] = np.array([0, 0, 1])
+
+        return T, Tinv
+
+    def __findSimilarity(self, uv, xy, options=None):
+        options = {"K": 2}
+
+        #    uv = np.array(uv)
+        #    xy = np.array(xy)
+
+        # Solve for trans1
+        trans1, trans1_inv = self.__findNonreflectiveSimilarity(uv, xy, options)
+
+        # manually reflect the xy data across the Y-axis
+        xyR = xy
+        xyR[:, 0] = -1 * xyR[:, 0]
+        # Solve for trans2
+        trans2r, trans2r_inv = self.__findNonreflectiveSimilarity(uv, xyR, options)
+
+        # manually reflect the tform to undo the reflection done on xyR
+        TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]])
+        trans2 = np.dot(trans2r, TreflectY)
+
+        # Figure out if trans1 or trans2 is better
+        xy1 = self.__tformfwd(trans1, uv)
+        norm1 = np.linalg.norm(xy1 - xy)
+        xy2 = self.__tformfwd(trans2, uv)
+        norm2 = np.linalg.norm(xy2 - xy)
+
+        if norm1 <= norm2:
+            return trans1, trans1_inv
+        else:
+            trans2_inv = np.linalg.inv(trans2)
+            return trans2, trans2_inv
+
+    def __get_similarity_transform(self, src_pts, dst_pts):
+        if self.reflective:
+            trans, trans_inv = self.__findSimilarity(src_pts, dst_pts)
+        else:
+            trans, trans_inv = self.__findNonreflectiveSimilarity(src_pts, dst_pts)
+        return trans, trans_inv
+
+    def __cvt_tform_mat_for_cv2(self, trans):
+        cv2_trans = trans[:, 0:2].T
+        return cv2_trans
+
+    def get_similarity_transform_for_cv2(self, src_pts, dst_pts):
+        trans, trans_inv = self.__get_similarity_transform(src_pts, dst_pts)
+        cv2_trans = self.__cvt_tform_mat_for_cv2(trans)
+        return cv2_trans, trans
+
+    def get_align_image(self, image, lm5_points):
+        assert lm5_points is not None
+        tfm, trans = self.get_similarity_transform_for_cv2(lm5_points, self._std_points)
+        return cv.warpAffine(image, tfm, (112, 112))