Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7b7c8f6d authored by Siarhei Vishniakou's avatar Siarhei Vishniakou
Browse files

Add vector of TouchVideoFrame to NotifyMotionArgs 

NotifyMotionArgs is passed to InputListener. We add a vector of
TouchVideoFrame here to NotifyMotionArgs  in order to provide heatmap
information about touch. This heatmap information can later be used to
do further processing of touch. For example, it could be used to
determine whether a given touch stream represents a user intentionally
pressing harder on the screen.

Introduce a new class, TouchVideoFrame, that will represent a single
scan of the touch heatmap for a given touch event.

Test: integration tested by adding a debug log to the case where the
finger area is large. That means, the video frames are being passed
locally to the InputClassfier HAL. Although HAL is not being submitted
together with this change, the current change will make the subsequent
CLs simpler to review.
Bug: 111480215

Change-Id: I4f16c8b2bd726c6bd4bbd5d2b18d2536a2347bda
parent d85333dd

include/input/TouchVideoFrame.h

0 → 100644
+67 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2018 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef _LIBINPUT_TOUCHVIDEOFRAME_H

#define _LIBINPUT_TOUCHVIDEOFRAME_H



#include <stdint.h>

#include <sys/time.h>

#include <vector>



namespace android {



/**

 * Represents data from a single scan of the touchscreen device.

 * Similar in concept to a video frame, but the touch strength is used as

 * the values instead.

 */

class TouchVideoFrame {

public:

    TouchVideoFrame(uint32_t width, uint32_t height, std::vector<int16_t> data,

            const struct timeval& timestamp) :

            mWidth(width), mHeight(height), mData(std::move(data)), mTimestamp(timestamp) {

    }



    /**

     * Width of the frame

     */

    uint32_t getWidth() const { return mWidth; }

    /**

     * Height of the frame

     */

    uint32_t getHeight() const { return mHeight; }

    /**

     * The touch strength data.

     * The array is a 2-D row-major matrix, with dimensions (height, width).

     * Total size of the array should equal getHeight() * getWidth().

     * Data is allowed to be negative.

     */

    const std::vector<int16_t>& getData() const { return mData; }

    /**

     * Time at which the heatmap was taken.

     */

    const struct timeval& getTimestamp() const { return mTimestamp; }



private:

    uint32_t mWidth;

    uint32_t mHeight;

    std::vector<int16_t> mData;

    struct timeval mTimestamp;

};



} // namespace android



#endif // _LIBINPUT_TOUCHVIDEOFRAME_H

services/inputflinger/InputListener.cpp

+6 −3
Original line number Diff line number Diff line
@@ -74,14 +74,16 @@ NotifyMotionArgs::NotifyMotionArgs(uint32_t sequenceNum, nsecs_t eventTime, int3 
        int32_t buttonState, int32_t edgeFlags, uint32_t deviceTimestamp,

        uint32_t pointerCount,

        const PointerProperties* pointerProperties, const PointerCoords* pointerCoords,

        float xPrecision, float yPrecision, nsecs_t downTime) :

        float xPrecision, float yPrecision, nsecs_t downTime,

        const std::vector<TouchVideoFrame>& videoFrames) :

        NotifyArgs(sequenceNum), eventTime(eventTime), deviceId(deviceId), source(source),

        displayId(displayId), policyFlags(policyFlags),

        action(action), actionButton(actionButton),

        flags(flags), metaState(metaState), buttonState(buttonState),

        edgeFlags(edgeFlags), deviceTimestamp(deviceTimestamp),

        pointerCount(pointerCount),

        xPrecision(xPrecision), yPrecision(yPrecision), downTime(downTime) {

        xPrecision(xPrecision), yPrecision(yPrecision), downTime(downTime),

        videoFrames(videoFrames) {

    for (uint32_t i = 0; i < pointerCount; i++) {

        this->pointerProperties[i].copyFrom(pointerProperties[i]);

        this->pointerCoords[i].copyFrom(pointerCoords[i]);
@@ -95,7 +97,8 @@ NotifyMotionArgs::NotifyMotionArgs(const NotifyMotionArgs& other) : 
        metaState(other.metaState), buttonState(other.buttonState),

        edgeFlags(other.edgeFlags),

        deviceTimestamp(other.deviceTimestamp), pointerCount(other.pointerCount),

        xPrecision(other.xPrecision), yPrecision(other.yPrecision), downTime(other.downTime) {

        xPrecision(other.xPrecision), yPrecision(other.yPrecision), downTime(other.downTime),

        videoFrames(other.videoFrames) {

    for (uint32_t i = 0; i < pointerCount; i++) {

        pointerProperties[i].copyFrom(other.pointerProperties[i]);

        pointerCoords[i].copyFrom(other.pointerCoords[i]);

services/inputflinger/InputReader.cpp

+17 −17
Original line number Diff line number Diff line
@@ -2832,7 +2832,7 @@ void CursorInputMapper::sync(nsecs_t when) { 
                        AMOTION_EVENT_ACTION_BUTTON_RELEASE, actionButton, 0,

                        metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,

                        /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                        mXPrecision, mYPrecision, downTime);

                        mXPrecision, mYPrecision, downTime, /* videoFrames */ {});

                getListener()->notifyMotion(&releaseArgs);

            }

        }
@@ -2841,7 +2841,7 @@ void CursorInputMapper::sync(nsecs_t when) { 
                displayId, policyFlags, motionEventAction, 0, 0, metaState, currentButtonState,

                AMOTION_EVENT_EDGE_FLAG_NONE,

                /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                mXPrecision, mYPrecision, downTime);

                mXPrecision, mYPrecision, downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);



        if (buttonsPressed) {
@@ -2853,7 +2853,7 @@ void CursorInputMapper::sync(nsecs_t when) { 
                        mSource, displayId, policyFlags, AMOTION_EVENT_ACTION_BUTTON_PRESS,

                        actionButton, 0, metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,

                        /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                        mXPrecision, mYPrecision, downTime);

                        mXPrecision, mYPrecision, downTime, /* videoFrames */ {});

                getListener()->notifyMotion(&pressArgs);

            }

        }
@@ -2867,7 +2867,7 @@ void CursorInputMapper::sync(nsecs_t when) { 
                    mSource, displayId, policyFlags, AMOTION_EVENT_ACTION_HOVER_MOVE, 0, 0,

                    metaState, currentButtonState, AMOTION_EVENT_EDGE_FLAG_NONE,

                    /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                    mXPrecision, mYPrecision, downTime);

                    mXPrecision, mYPrecision, downTime, /* videoFrames */ {});

            getListener()->notifyMotion(&hoverArgs);

        }
@@ -2881,7 +2881,7 @@ void CursorInputMapper::sync(nsecs_t when) { 
                    AMOTION_EVENT_ACTION_SCROLL, 0, 0, metaState, currentButtonState,

                    AMOTION_EVENT_EDGE_FLAG_NONE,

                    /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                    mXPrecision, mYPrecision, downTime);

                    mXPrecision, mYPrecision, downTime, /* videoFrames */ {});

            getListener()->notifyMotion(&scrollArgs);

        }

    }
@@ -3013,7 +3013,7 @@ void RotaryEncoderInputMapper::sync(nsecs_t when) { 
                AMOTION_EVENT_ACTION_SCROLL, 0, 0, metaState, 0,

                AMOTION_EVENT_EDGE_FLAG_NONE,

                /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                0, 0, 0);

                0, 0, 0, /* videoFrames */ {});

        getListener()->notifyMotion(&scrollArgs);

    }
@@ -5400,7 +5400,7 @@ void TouchInputMapper::dispatchPointerGestures(nsecs_t when, uint32_t policyFlag 
                AMOTION_EVENT_ACTION_HOVER_MOVE, 0, 0,

                metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,

                /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

                0, 0, mPointerGesture.downTime);

                0, 0, mPointerGesture.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6325,7 +6325,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                /* deviceTimestamp */ 0,

                1, &mPointerSimple.lastProperties, &mPointerSimple.lastCoords,

                mOrientedXPrecision, mOrientedYPrecision,

                mPointerSimple.downTime);

                mPointerSimple.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6339,7 +6339,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                /* deviceTimestamp */ 0,

                1, &mPointerSimple.lastProperties, &mPointerSimple.lastCoords,

                mOrientedXPrecision, mOrientedYPrecision,

                mPointerSimple.downTime);

                mPointerSimple.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6355,7 +6355,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                    /* deviceTimestamp */ 0,

                    1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,

                    mOrientedXPrecision, mOrientedYPrecision,

                    mPointerSimple.downTime);

                    mPointerSimple.downTime, /* videoFrames */ {});

            getListener()->notifyMotion(&args);

        }
@@ -6366,7 +6366,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                /* deviceTimestamp */ 0,

                1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,

                mOrientedXPrecision, mOrientedYPrecision,

                mPointerSimple.downTime);

                mPointerSimple.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6382,7 +6382,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                    /* deviceTimestamp */ 0,

                    1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,

                    mOrientedXPrecision, mOrientedYPrecision,

                    mPointerSimple.downTime);

                    mPointerSimple.downTime, /* videoFrames */ {});

            getListener()->notifyMotion(&args);

        }
@@ -6394,7 +6394,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                /* deviceTimestamp */ 0,

                1, &mPointerSimple.currentProperties, &mPointerSimple.currentCoords,

                mOrientedXPrecision, mOrientedYPrecision,

                mPointerSimple.downTime);

                mPointerSimple.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6416,7 +6416,7 @@ void TouchInputMapper::dispatchPointerSimple(nsecs_t when, uint32_t policyFlags, 
                /* deviceTimestamp */ 0,

                1, &mPointerSimple.currentProperties, &pointerCoords,

                mOrientedXPrecision, mOrientedYPrecision,

                mPointerSimple.downTime);

                mPointerSimple.downTime, /* videoFrames */ {});

        getListener()->notifyMotion(&args);

    }
@@ -6478,7 +6478,7 @@ void TouchInputMapper::dispatchMotion(nsecs_t when, uint32_t policyFlags, uint32 
            source, mViewport.displayId, policyFlags,

            action, actionButton, flags, metaState, buttonState, edgeFlags,

            deviceTimestamp, pointerCount, pointerProperties, pointerCoords,

            xPrecision, yPrecision, downTime);

            xPrecision, yPrecision, downTime, /* videoFrames */ {});

    getListener()->notifyMotion(&args);

}
@@ -7403,7 +7403,7 @@ void JoystickInputMapper::sync(nsecs_t when, bool force) { 
            AINPUT_SOURCE_JOYSTICK, ADISPLAY_ID_NONE, policyFlags,

            AMOTION_EVENT_ACTION_MOVE, 0, 0, metaState, buttonState, AMOTION_EVENT_EDGE_FLAG_NONE,

            /* deviceTimestamp */ 0, 1, &pointerProperties, &pointerCoords,

            0, 0, 0);

            0, 0, 0, /* videoFrames */ {});

    getListener()->notifyMotion(&args);

}

services/inputflinger/include/InputListener.h

+6 −1
Original line number Diff line number Diff line
@@ -17,7 +17,10 @@ 
#ifndef _UI_INPUT_LISTENER_H

#define _UI_INPUT_LISTENER_H



#include <vector>



#include <input/Input.h>

#include <input/TouchVideoFrame.h>

#include <utils/RefBase.h>

#include <utils/Vector.h>
@@ -110,6 +113,7 @@ struct NotifyMotionArgs : public NotifyArgs { 
    float xPrecision;

    float yPrecision;

    nsecs_t downTime;

    std::vector<TouchVideoFrame> videoFrames;



    inline NotifyMotionArgs() { }
@@ -119,7 +123,8 @@ struct NotifyMotionArgs : public NotifyArgs { 
            int32_t metaState, int32_t buttonState,

            int32_t edgeFlags, uint32_t deviceTimestamp, uint32_t pointerCount,

            const PointerProperties* pointerProperties, const PointerCoords* pointerCoords,

            float xPrecision, float yPrecision, nsecs_t downTime);

            float xPrecision, float yPrecision, nsecs_t downTime,

            const std::vector<TouchVideoFrame>& videoFrames);



    NotifyMotionArgs(const NotifyMotionArgs& other);