Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8154bbdf authored by Siarhei Vishniakou's avatar Siarhei Vishniakou
Browse files

Rotate TouchVideoFrames 

When a screen orientation change happens, the touch coordinates are
adjusted in InputReader to accomodate this rotation. When the user is
holding the device, the origin (0, 0) is always at the top left of the
screen.
However, currently, the TouchVideoFrames are not being rotated. This
presents a problem, because the touch coordinates cannot be directly
matched to the heatmap, as received in the HAL.

To account for this, we rotate the touch video frame.

Test: atest libinput_tests inputflinger_tests
Bug: 123241238
Change-Id: Iff45c68b1d2b237d2b1657ed76f50bb23ef8468a
parent 6b76bdf9

include/input/TouchVideoFrame.h

+13 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 


#include <stdint.h>

#include <sys/time.h>

#include <ui/DisplayInfo.h>

#include <vector>



namespace android {
@@ -55,11 +56,23 @@ public: 
     */

    const struct timeval& getTimestamp() const;



    /**

     * Rotate the video frame.

     * The rotation value is an enum from ui/DisplayInfo.h

     */

    void rotate(int32_t orientation);



private:

    uint32_t mHeight;

    uint32_t mWidth;

    std::vector<int16_t> mData;

    struct timeval mTimestamp;



    /**

     * Common method for 90 degree and 270 degree rotation

     */

    void rotateQuarterTurn(bool clockwise);

    void rotate180();

};



} // namespace android

libs/input/TouchVideoFrame.cpp

+58 −0
Original line number Diff line number Diff line
@@ -39,4 +39,62 @@ const std::vector<int16_t>& TouchVideoFrame::getData() const { return mData; } 


const struct timeval& TouchVideoFrame::getTimestamp() const { return mTimestamp; }



void TouchVideoFrame::rotate(int32_t orientation) {

    switch (orientation) {

        case DISPLAY_ORIENTATION_90:

            rotateQuarterTurn(true /*clockwise*/);

            break;

        case DISPLAY_ORIENTATION_180:

            rotate180();

            break;

        case DISPLAY_ORIENTATION_270:

            rotateQuarterTurn(false /*clockwise*/);

            break;

    }

}



/**

 * Rotate once clockwise by a quarter turn === rotate 90 degrees

 * Rotate once counterclockwise by a quarter turn === rotate 270 degrees

 * For a clockwise rotation:

 *     An element at position (i, j) is rotated to (j, height - i - 1)

 * For a counterclockwise rotation:

 *     An element at position (i, j) is rotated to (width - j - 1, i)

 */

void TouchVideoFrame::rotateQuarterTurn(bool clockwise) {

    std::vector<int16_t> rotated(mData.size());

    for (size_t i = 0; i < mHeight; i++) {

        for (size_t j = 0; j < mWidth; j++) {

            size_t iRotated, jRotated;

            if (clockwise) {

                iRotated = j;

                jRotated = mHeight - i - 1;

            } else {

                iRotated = mWidth - j - 1;

                jRotated = i;

            }

            size_t indexRotated = iRotated * mHeight + jRotated;

            rotated[indexRotated] = mData[i * mWidth + j];

        }

    }

    mData = std::move(rotated);

    std::swap(mHeight, mWidth);

}



/**

 * An element at position (i, j) is rotated to (height - i - 1, width - j - 1)

 * This is equivalent to moving element [i] to position [height * width - i - 1]

 * Since element at [height * width - i - 1] would move to position [i],

 * we can just swap elements [i] and [height * width - i - 1].

 */

void TouchVideoFrame::rotate180() {

    if (mData.size() == 0) {

        return;

    }

    // Just need to swap elements i and (height * width - 1 - i)

    for (size_t i = 0; i < mData.size() / 2; i++) {

        std::swap(mData[i], mData[mHeight * mWidth - 1 - i]);

    }

}



} // namespace android

libs/input/tests/TouchVideoFrame_test.cpp

+125 −0
Original line number Diff line number Diff line
@@ -67,5 +67,130 @@ TEST(TouchVideoFrame, Equality) { 
    ASSERT_FALSE(frame == changedTimestampFrame);

}



// --- Rotate 90 degrees ---



TEST(TouchVideoFrame, Rotate90_0x0) {

    TouchVideoFrame frame(0, 0, {}, TIMESTAMP);

    TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_90);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate90_1x1) {

    TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);

    TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_90);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate90_2x2) {

    TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);

    TouchVideoFrame frameRotated(2, 2, {3, 1, 4, 2}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_90);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate90_3x2) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameRotated(2, 3, {5, 3, 1, 6, 4, 2}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_90);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate90_3x2_4times) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_90);

    frame.rotate(DISPLAY_ORIENTATION_90);

    frame.rotate(DISPLAY_ORIENTATION_90);

    frame.rotate(DISPLAY_ORIENTATION_90);

    ASSERT_EQ(frame, frameOriginal);

}



// --- Rotate 180 degrees ---



TEST(TouchVideoFrame, Rotate180_0x0) {

    TouchVideoFrame frame(0, 0, {}, TIMESTAMP);

    TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate180_1x1) {

    TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);

    TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate180_2x2) {

    TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);

    TouchVideoFrame frameRotated(2, 2, {4, 3, 2, 1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate180_3x2) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameRotated(3, 2, {6, 5, 4, 3, 2, 1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate180_3x2_2times) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameOriginal);

}



TEST(TouchVideoFrame, Rotate180_3x3) {

    TouchVideoFrame frame(3, 3, {1, 2, 3, 4, 5, 6, 7, 8, 9}, TIMESTAMP);

    TouchVideoFrame frameRotated(3, 3, {9, 8, 7, 6, 5, 4, 3, 2, 1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_180);

    ASSERT_EQ(frame, frameRotated);

}



// --- Rotate 270 degrees ---



TEST(TouchVideoFrame, Rotate270_0x0) {

    TouchVideoFrame frame(0, 0, {}, TIMESTAMP);

    TouchVideoFrame frameRotated(0, 0, {}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_270);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate270_1x1) {

    TouchVideoFrame frame(1, 1, {1}, TIMESTAMP);

    TouchVideoFrame frameRotated(1, 1, {1}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_270);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate270_2x2) {

    TouchVideoFrame frame(2, 2, {1, 2, 3, 4}, TIMESTAMP);

    TouchVideoFrame frameRotated(2, 2, {2, 4, 1, 3}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_270);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate270_3x2) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameRotated(2, 3, {2, 4, 6, 1, 3, 5}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_270);

    ASSERT_EQ(frame, frameRotated);

}



TEST(TouchVideoFrame, Rotate270_3x2_4times) {

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    TouchVideoFrame frameOriginal(3, 2, {1, 2, 3, 4, 5, 6}, TIMESTAMP);

    frame.rotate(DISPLAY_ORIENTATION_270);

    frame.rotate(DISPLAY_ORIENTATION_270);

    frame.rotate(DISPLAY_ORIENTATION_270);

    frame.rotate(DISPLAY_ORIENTATION_270);

    ASSERT_EQ(frame, frameOriginal);

}



} // namespace test

} // namespace android

services/inputflinger/InputReader.cpp

+2 −0
Original line number Diff line number Diff line
@@ -6565,6 +6565,8 @@ void TouchInputMapper::dispatchMotion(nsecs_t when, uint32_t policyFlags, uint32 
    const int32_t displayId = getAssociatedDisplay().value_or(ADISPLAY_ID_NONE);

    const int32_t deviceId = getDeviceId();

    std::vector<TouchVideoFrame> frames = mDevice->getEventHub()->getVideoFrames(deviceId);

    std::for_each(frames.begin(), frames.end(),

            [this](TouchVideoFrame& frame) { frame.rotate(this->mSurfaceOrientation); });

    NotifyMotionArgs args(mContext->getNextSequenceNum(), when, deviceId,

            source, displayId, policyFlags,

            action, actionButton, flags, metaState, buttonState, MotionClassification::NONE,

services/inputflinger/tests/InputReader_test.cpp

+48 −0
Original line number Diff line number Diff line
@@ -6453,4 +6453,52 @@ TEST_F(MultiTouchInputMapperTest, VideoFrames_ReceivedByListener) { 
    ASSERT_EQ(std::vector<TouchVideoFrame>(), motionArgs.videoFrames);

}



TEST_F(MultiTouchInputMapperTest, VideoFrames_AreRotated) {

    MultiTouchInputMapper* mapper = new MultiTouchInputMapper(mDevice);

    prepareAxes(POSITION);

    addConfigurationProperty("touch.deviceType", "touchScreen");

    addMapperAndConfigure(mapper);

    // Unrotated video frame

    TouchVideoFrame frame(3, 2, {1, 2, 3, 4, 5, 6}, {1, 2});

    NotifyMotionArgs motionArgs;



    // Test all 4 orientations

    for (int32_t orientation : {DISPLAY_ORIENTATION_0, DISPLAY_ORIENTATION_90,

             DISPLAY_ORIENTATION_180, DISPLAY_ORIENTATION_270}) {

        SCOPED_TRACE("Orientation " + StringPrintf("%i", orientation));

        clearViewports();

        prepareDisplay(orientation);

        std::vector<TouchVideoFrame> frames{frame};

        mFakeEventHub->setVideoFrames({{mDevice->getId(), frames}});

        processPosition(mapper, 100, 200);

        processSync(mapper);

        ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs));

        frames[0].rotate(orientation);

        ASSERT_EQ(frames, motionArgs.videoFrames);

    }

}



TEST_F(MultiTouchInputMapperTest, VideoFrames_MultipleFramesAreRotated) {

    MultiTouchInputMapper* mapper = new MultiTouchInputMapper(mDevice);

    prepareAxes(POSITION);

    addConfigurationProperty("touch.deviceType", "touchScreen");

    addMapperAndConfigure(mapper);

    // Unrotated video frames. There's no rule that they must all have the same dimensions,

    // so mix these.

    TouchVideoFrame frame1(3, 2, {1, 2, 3, 4, 5, 6}, {1, 2});

    TouchVideoFrame frame2(3, 3, {0, 1, 2, 3, 4, 5, 6, 7, 8}, {1, 3});

    TouchVideoFrame frame3(2, 2, {10, 20, 10, 0}, {1, 4});

    std::vector<TouchVideoFrame> frames{frame1, frame2, frame3};

    NotifyMotionArgs motionArgs;



    prepareDisplay(DISPLAY_ORIENTATION_90);

    mFakeEventHub->setVideoFrames({{mDevice->getId(), frames}});

    processPosition(mapper, 100, 200);

    processSync(mapper);

    ASSERT_NO_FATAL_FAILURE(mFakeListener->assertNotifyMotionWasCalled(&motionArgs));

    std::for_each(frames.begin(), frames.end(),

            [](TouchVideoFrame& frame) { frame.rotate(DISPLAY_ORIENTATION_90); });

    ASSERT_EQ(frames, motionArgs.videoFrames);

}



} // namespace android