Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f274625e authored by Ashwini Rao's avatar Ashwini Rao Committed by Steve Kondik
Browse files

Frameworks: Enhance face detection

Add support for facial processing features:
smile detection, gaze detection and blink detection

Change-Id: Ic186ff0658e755a8b5287cffa1dd58d9eafebc9a
parent 46820e10
Loading
Loading
Loading
Loading
+17 −0
Original line number Diff line number Diff line
@@ -1481,6 +1481,23 @@ public class Camera {
         * as a set. Either they are all valid, or none of them are.
         */
        public Point mouth = null;

        /**
         * {@hide}
         */
        public int smileDegree = 0;
        /**
         * {@hide}
         */
        public int smileScore = 0;
        /**
         * {@hide}
         */
        public int blinkDetected = 0;
        /**
         * {@hide}
         */
        public int faceRecognised = 0;
    }

    // Error codes match the enum in include/ui/Camera.h
+125 −6
Original line number Diff line number Diff line
@@ -49,6 +49,25 @@ struct fields_t {
    jmethodID   post_event;
    jmethodID   rect_constructor;
    jmethodID   face_constructor;
    jfieldID    face_id;
    jfieldID    face_leftEye;
    jfieldID    face_rightEye;
    jfieldID    face_mouth;
    jfieldID    face_sm_degree;
    jfieldID    face_sm_score;
    jfieldID    face_blink_detected;
    jfieldID    face_gaze_angle;
    jfieldID    face_updown_dir;
    jfieldID    face_leftright_dir;
    jfieldID    face_roll_dir;
    jfieldID    face_leye_blink;
    jfieldID    face_reye_blink;
    jfieldID    face_left_right_gaze;
    jfieldID    face_top_bottom_gaze;
    jfieldID    face_recognised;
    jfieldID    point_x;
    jfieldID    point_y;
    jmethodID   point_constructor;
};

static fields_t fields;
@@ -82,6 +101,8 @@ private:
    sp<Camera>  mCamera;                // strong reference to native object
    jclass      mFaceClass;  // strong reference to Face class
    jclass      mRectClass;  // strong reference to Rect class
    jclass      mPointClass; // strong reference to Point class
    bool        mIsQcFace;
    Mutex       mLock;

    /*
@@ -132,12 +153,22 @@ JNICameraContext::JNICameraContext(JNIEnv* env, jobject weak_this, jclass clazz,
    mCameraJClass = (jclass)env->NewGlobalRef(clazz);
    mCamera = camera;

    jclass qcfaceClazz = env->FindClass("org/codeaurora/camera/QCFace");
    if (NULL != qcfaceClazz) {
        mFaceClass = (jclass) env->NewGlobalRef(qcfaceClazz);
        mIsQcFace = true;
    } else {
        jclass faceClazz = env->FindClass("android/hardware/Camera$Face");
        mFaceClass = (jclass) env->NewGlobalRef(faceClazz);
        mIsQcFace = false;
    }

    jclass rectClazz = env->FindClass("android/graphics/Rect");
    mRectClass = (jclass) env->NewGlobalRef(rectClazz);

    jclass pointClazz = env->FindClass("android/graphics/Point");
    mPointClass = (jclass) env->NewGlobalRef(pointClazz);

    mManualBufferMode = false;
    mManualCameraCallbackSet = false;
}
@@ -164,6 +195,10 @@ void JNICameraContext::release()
        env->DeleteGlobalRef(mRectClass);
        mRectClass = NULL;
    }
    if (mPointClass != NULL) {
        env->DeleteGlobalRef(mPointClass);
        mPointClass = NULL;
    }
    clearCallbackBuffers_l(env);
    mCamera.clear();
}
@@ -346,12 +381,47 @@ void JNICameraContext::postMetadata(JNIEnv *env, int32_t msgType, camera_frame_m
        env->SetIntField(rect, fields.rect_top, metadata->faces[i].rect[1]);
        env->SetIntField(rect, fields.rect_right, metadata->faces[i].rect[2]);
        env->SetIntField(rect, fields.rect_bottom, metadata->faces[i].rect[3]);

        env->SetObjectField(face, fields.face_rect, rect);
        env->SetIntField(face, fields.face_score, metadata->faces[i].score);

        jobject point1 = env->NewObject(mPointClass, fields.point_constructor);
        env->SetIntField(point1, fields.point_x, metadata->faces[i].left_eye[0]);
        env->SetIntField(point1, fields.point_y, metadata->faces[i].left_eye[1]);
        env->SetObjectField(face, fields.face_leftEye, point1);

        jobject point2 = env->NewObject(mPointClass, fields.point_constructor);
        env->SetIntField(point2, fields.point_x, metadata->faces[i].right_eye[0]);
        env->SetIntField(point2, fields.point_y, metadata->faces[i].right_eye[1]);
        env->SetObjectField(face, fields.face_rightEye, point2);

        jobject point3 = env->NewObject(mPointClass, fields.point_constructor);
        env->SetIntField(point3, fields.point_x, metadata->faces[i].mouth[0]);
        env->SetIntField(point3, fields.point_y, metadata->faces[i].mouth[1]);
        env->SetObjectField(face, fields.face_mouth, point3);

        env->SetIntField(face, fields.face_id, metadata->faces[i].id);

        if (mIsQcFace) {
            env->SetIntField(face, fields.face_sm_degree, metadata->faces[i].smile_degree);
            env->SetIntField(face, fields.face_sm_score, metadata->faces[i].smile_score);
            env->SetIntField(face, fields.face_blink_detected, metadata->faces[i].blink_detected);
            env->SetIntField(face, fields.face_recognised, metadata->faces[i].face_recognised);
            env->SetIntField(face, fields.face_gaze_angle, metadata->faces[i].gaze_angle);
            env->SetIntField(face, fields.face_updown_dir, metadata->faces[i].updown_dir);
            env->SetIntField(face, fields.face_leftright_dir, metadata->faces[i].leftright_dir);
            env->SetIntField(face, fields.face_roll_dir, metadata->faces[i].roll_dir);
            env->SetIntField(face, fields.face_leye_blink, metadata->faces[i].leye_blink);
            env->SetIntField(face, fields.face_reye_blink, metadata->faces[i].reye_blink);
            env->SetIntField(face, fields.face_left_right_gaze, metadata->faces[i].left_right_gaze);
            env->SetIntField(face, fields.face_top_bottom_gaze, metadata->faces[i].top_bottom_gaze);
        }

        env->DeleteLocalRef(face);
        env->DeleteLocalRef(rect);

        env->DeleteLocalRef(point1);
        env->DeleteLocalRef(point2);
        env->DeleteLocalRef(point3);
    }
    env->CallStaticVoidMethod(mCameraJClass, fields.post_event,
            mCameraJObjectWeak, msgType, 0, 0, obj);
@@ -1061,12 +1131,46 @@ int register_android_hardware_Camera(JNIEnv *env)
        { "android/hardware/Camera$CameraInfo", "orientation",   "I", &fields.orientation },
        { "android/hardware/Camera$CameraInfo", "canDisableShutterSound",   "Z",
          &fields.canDisableShutterSound },
        { "android/hardware/Camera$Face", "rect", "Landroid/graphics/Rect;", &fields.face_rect },
        { "android/hardware/Camera$Face", "score", "I", &fields.face_score },
        { "android/graphics/Rect", "left", "I", &fields.rect_left },
        { "android/graphics/Rect", "top", "I", &fields.rect_top },
        { "android/graphics/Rect", "right", "I", &fields.rect_right },
        { "android/graphics/Rect", "bottom", "I", &fields.rect_bottom },
        { "android/graphics/Point", "x", "I", &fields.point_x },
        { "android/graphics/Point", "y", "I", &fields.point_y },
    };

    field facefields_to_find[] = {
        { "android/hardware/Camera$Face", "rect", "Landroid/graphics/Rect;", &fields.face_rect },
        { "android/hardware/Camera$Face", "score", "I", &fields.face_score },
        { "android/hardware/Camera$Face", "id", "I", &fields.face_id },
        { "android/hardware/Camera$Face", "leftEye", "Landroid/graphics/Point;", &fields.face_leftEye },
        { "android/hardware/Camera$Face", "rightEye", "Landroid/graphics/Point;", &fields.face_rightEye },
        { "android/hardware/Camera$Face", "mouth", "Landroid/graphics/Point;", &fields.face_mouth },
        { "android/hardware/Camera$Face", "smileDegree", "I", &fields.face_sm_degree },
        { "android/hardware/Camera$Face", "smileScore", "I", &fields.face_sm_score },
        { "android/hardware/Camera$Face", "blinkDetected", "I", &fields.face_blink_detected },
        { "android/hardware/Camera$Face", "faceRecognised", "I", &fields.face_recognised },
    };

    field qcfacefields_to_find[] = {
        { "org/codeaurora/camera/QCFace", "rect", "Landroid/graphics/Rect;", &fields.face_rect },
        { "org/codeaurora/camera/QCFace", "score", "I", &fields.face_score },
        { "org/codeaurora/camera/QCFace", "id", "I", &fields.face_id },
        { "org/codeaurora/camera/QCFace", "leftEye", "Landroid/graphics/Point;", &fields.face_leftEye },
        { "org/codeaurora/camera/QCFace", "rightEye", "Landroid/graphics/Point;", &fields.face_rightEye },
        { "org/codeaurora/camera/QCFace", "mouth", "Landroid/graphics/Point;", &fields.face_mouth },
        { "org/codeaurora/camera/QCFace", "smileDegree", "I", &fields.face_sm_degree },
        { "org/codeaurora/camera/QCFace", "smileScore", "I", &fields.face_sm_score },
        { "org/codeaurora/camera/QCFace", "blinkDetected", "I", &fields.face_blink_detected },
        { "org/codeaurora/camera/QCFace", "faceRecognized", "I", &fields.face_recognised },
        { "org/codeaurora/camera/QCFace", "gazeAngle", "I", &fields.face_gaze_angle },
        { "org/codeaurora/camera/QCFace", "updownDir", "I", &fields.face_updown_dir },
        { "org/codeaurora/camera/QCFace", "leftrightDir", "I", &fields.face_leftright_dir },
        { "org/codeaurora/camera/QCFace", "rollDir", "I", &fields.face_roll_dir },
        { "org/codeaurora/camera/QCFace", "leyeBlink", "I", &fields.face_leye_blink },
        { "org/codeaurora/camera/QCFace", "reyeBlink", "I", &fields.face_reye_blink },
        { "org/codeaurora/camera/QCFace", "leftrightGaze", "I", &fields.face_left_right_gaze },
        { "org/codeaurora/camera/QCFace", "topbottomGaze", "I", &fields.face_top_bottom_gaze },
    };

    if (find_fields(env, fields_to_find, NELEM(fields_to_find)) < 0)
@@ -1087,13 +1191,28 @@ int register_android_hardware_Camera(JNIEnv *env)
        return -1;
    }

    clazz = env->FindClass("android/graphics/Point");
    fields.point_constructor = env->GetMethodID(clazz, "<init>", "()V");
    if (fields.point_constructor == NULL) {
        ALOGE("Can't find android/graphics/Point.Point()");
        return -1;
    }

    clazz = env->FindClass("org/codeaurora/camera/QCFace");
    if (NULL != clazz) {
        fields.face_constructor = env->GetMethodID(clazz, "<init>", "()V");
        if (find_fields(env, qcfacefields_to_find, NELEM(qcfacefields_to_find)) < 0)
            return -1;
    } else {
    clazz = env->FindClass("android/hardware/Camera$Face");
    fields.face_constructor = env->GetMethodID(clazz, "<init>", "()V");
    if (fields.face_constructor == NULL) {
        ALOGE("Can't find android/hardware/Camera$Face.Face()");
        return -1;
        }

        if (find_fields(env, facefields_to_find, NELEM(fields_to_find)) < 0)
            return -1;
    }
    // Register native functions
    return AndroidRuntime::registerNativeMethods(env, "android/hardware/Camera",
                                              camMethods, NELEM(camMethods));