Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d65552bb authored by Prabir Pradhan's avatar Prabir Pradhan
Browse files

InputDispatcher: Implement stylus interceptor 

A stylus interceptor is a window that receives all stylus events within
its touchable bounds, while letting all other events be dispatched to
windows behind it. This makes it possible for the framework to create a
stylus interceptor on top of another app to implement handwriting
recognition.

The feature has no effect when the window flag FLAG_NOT_TOUCHABLE is
not set.

The feature has no effect when the window is not a trusted overlay.

Test: atest inputflinger_tests
Change-Id: Id4833840aa3088d21333d3ea08beffbded4debbc
parent 186102db

libs/gui/WindowInfo.cpp

+4 −0
Original line number Original line Diff line number Diff line
@@ -47,6 +47,10 @@ bool WindowInfo::isSpy() const { 
    return inputFeatures.test(Feature::SPY);

    return inputFeatures.test(Feature::SPY);

}

}





bool WindowInfo::interceptsStylus() const {

    return inputFeatures.test(Feature::INTERCEPTS_STYLUS);

}



bool WindowInfo::overlaps(const WindowInfo* other) const {

bool WindowInfo::overlaps(const WindowInfo* other) const {

    return frameLeft < other->frameRight && frameRight > other->frameLeft &&

    return frameLeft < other->frameRight && frameRight > other->frameLeft &&

            frameTop < other->frameBottom && frameBottom > other->frameTop;

            frameTop < other->frameBottom && frameBottom > other->frameTop;

libs/gui/include/gui/WindowInfo.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -138,6 +138,7 @@ struct WindowInfo : public Parcelable { 
        DROP_INPUT = 1u << 3,

        DROP_INPUT = 1u << 3,

        DROP_INPUT_IF_OBSCURED = 1u << 4,

        DROP_INPUT_IF_OBSCURED = 1u << 4,

        SPY = 1u << 5,

        SPY = 1u << 5,

        INTERCEPTS_STYLUS = 1u << 6,

    };

    };





    /* These values are filled in by the WM and passed through SurfaceFlinger

    /* These values are filled in by the WM and passed through SurfaceFlinger
@@ -218,6 +219,8 @@ struct WindowInfo : public Parcelable { 




    bool isSpy() const;

    bool isSpy() const;





    bool interceptsStylus() const;



    bool overlaps(const WindowInfo* other) const;

    bool overlaps(const WindowInfo* other) const;





    bool operator==(const WindowInfo& inputChannel) const;

    bool operator==(const WindowInfo& inputChannel) const;

services/inputflinger/dispatcher/InputDispatcher.cpp

+44 −16
Original line number Original line Diff line number Diff line
@@ -530,12 +530,14 @@ bool isUserActivityEvent(const EventEntry& eventEntry) { 
}

}





// Returns true if the given window can accept pointer events at the given display location.

// Returns true if the given window can accept pointer events at the given display location.

bool windowAcceptsTouchAt(const WindowInfo& windowInfo, int32_t displayId, int32_t x, int32_t y) {

bool windowAcceptsTouchAt(const WindowInfo& windowInfo, int32_t displayId, int32_t x, int32_t y,

                          bool isStylus) {

    if (windowInfo.displayId != displayId || !windowInfo.visible) {

    if (windowInfo.displayId != displayId || !windowInfo.visible) {

        return false;

        return false;

    }

    }

    const auto flags = windowInfo.flags;

    const auto flags = windowInfo.flags;

    if (flags.test(WindowInfo::Flag::NOT_TOUCHABLE)) {

    const bool windowCanInterceptTouch = isStylus && windowInfo.interceptsStylus();

    if (flags.test(WindowInfo::Flag::NOT_TOUCHABLE) && !windowCanInterceptTouch) {

        return false;

        return false;

    }

    }

    const bool isModalWindow = !flags.test(WindowInfo::Flag::NOT_FOCUSABLE) &&

    const bool isModalWindow = !flags.test(WindowInfo::Flag::NOT_FOCUSABLE) &&
@@ -546,6 +548,12 @@ bool windowAcceptsTouchAt(const WindowInfo& windowInfo, int32_t displayId, int32 
    return true;

    return true;

}

}





bool isPointerFromStylus(const MotionEntry& entry, int32_t pointerIndex) {

    return isFromSource(entry.source, AINPUT_SOURCE_STYLUS) &&

            (entry.pointerProperties[pointerIndex].toolType == AMOTION_EVENT_TOOL_TYPE_STYLUS ||

             entry.pointerProperties[pointerIndex].toolType == AMOTION_EVENT_TOOL_TYPE_ERASER);

}



} // namespace

} // namespace





// --- InputDispatcher ---

// --- InputDispatcher ---
@@ -939,8 +947,10 @@ bool InputDispatcher::shouldPruneInboundQueueLocked(const MotionEntry& motionEnt 
                motionEntry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_X));

                motionEntry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_X));

        int32_t y = static_cast<int32_t>(

        int32_t y = static_cast<int32_t>(

                motionEntry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_Y));

                motionEntry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_Y));



        const bool isStylus = isPointerFromStylus(motionEntry, 0 /*pointerIndex*/);

        sp<WindowInfoHandle> touchedWindowHandle =

        sp<WindowInfoHandle> touchedWindowHandle =

                findTouchedWindowAtLocked(displayId, x, y, nullptr);

                findTouchedWindowAtLocked(displayId, x, y, nullptr, isStylus);

        if (touchedWindowHandle != nullptr &&

        if (touchedWindowHandle != nullptr &&

            touchedWindowHandle->getApplicationToken() !=

            touchedWindowHandle->getApplicationToken() !=

                    mAwaitedFocusedApplication->getApplicationToken()) {

                    mAwaitedFocusedApplication->getApplicationToken()) {
@@ -1045,6 +1055,7 @@ void InputDispatcher::addRecentEventLocked(std::shared_ptr<EventEntry> entry) { 




sp<WindowInfoHandle> InputDispatcher::findTouchedWindowAtLocked(int32_t displayId, int32_t x,

sp<WindowInfoHandle> InputDispatcher::findTouchedWindowAtLocked(int32_t displayId, int32_t x,

                                                                int32_t y, TouchState* touchState,

                                                                int32_t y, TouchState* touchState,

                                                                bool isStylus,

                                                                bool addOutsideTargets,

                                                                bool addOutsideTargets,

                                                                bool ignoreDragWindow) {

                                                                bool ignoreDragWindow) {

    if (addOutsideTargets && touchState == nullptr) {

    if (addOutsideTargets && touchState == nullptr) {
@@ -1058,7 +1069,7 @@ sp<WindowInfoHandle> InputDispatcher::findTouchedWindowAtLocked(int32_t displayI 
        }

        }





        const WindowInfo& info = *windowHandle->getInfo();

        const WindowInfo& info = *windowHandle->getInfo();

        if (!info.isSpy() && windowAcceptsTouchAt(info, displayId, x, y)) {

        if (!info.isSpy() && windowAcceptsTouchAt(info, displayId, x, y, isStylus)) {

            return windowHandle;

            return windowHandle;

        }

        }
@@ -1070,16 +1081,15 @@ sp<WindowInfoHandle> InputDispatcher::findTouchedWindowAtLocked(int32_t displayI 
    return nullptr;

    return nullptr;

}

}





std::vector<sp<WindowInfoHandle>> InputDispatcher::findTouchedSpyWindowsAtLocked(int32_t displayId,

std::vector<sp<WindowInfoHandle>> InputDispatcher::findTouchedSpyWindowsAtLocked(

                                                                                 int32_t x,

        int32_t displayId, int32_t x, int32_t y, bool isStylus) const {

                                                                                 int32_t y) const {

    // Traverse windows from front to back and gather the touched spy windows.

    // Traverse windows from front to back and gather the touched spy windows.

    std::vector<sp<WindowInfoHandle>> spyWindows;

    std::vector<sp<WindowInfoHandle>> spyWindows;

    const auto& windowHandles = getWindowHandlesLocked(displayId);

    const auto& windowHandles = getWindowHandlesLocked(displayId);

    for (const sp<WindowInfoHandle>& windowHandle : windowHandles) {

    for (const sp<WindowInfoHandle>& windowHandle : windowHandles) {

        const WindowInfo& info = *windowHandle->getInfo();

        const WindowInfo& info = *windowHandle->getInfo();





        if (!windowAcceptsTouchAt(info, displayId, x, y)) {

        if (!windowAcceptsTouchAt(info, displayId, x, y, isStylus)) {

            continue;

            continue;

        }

        }

        if (!info.isSpy()) {

        if (!info.isSpy()) {
@@ -2062,8 +2072,9 @@ InputEventInjectionResult InputDispatcher::findTouchedWindowTargetsLocked( 
            y = int32_t(entry.pointerCoords[pointerIndex].getAxisValue(AMOTION_EVENT_AXIS_Y));

            y = int32_t(entry.pointerCoords[pointerIndex].getAxisValue(AMOTION_EVENT_AXIS_Y));

        }

        }

        const bool isDown = maskedAction == AMOTION_EVENT_ACTION_DOWN;

        const bool isDown = maskedAction == AMOTION_EVENT_ACTION_DOWN;

        const bool isStylus = isPointerFromStylus(entry, pointerIndex);

        newTouchedWindowHandle = findTouchedWindowAtLocked(displayId, x, y, &tempTouchState,

        newTouchedWindowHandle = findTouchedWindowAtLocked(displayId, x, y, &tempTouchState,

                                                           isDown /*addOutsideTargets*/);

                                                           isStylus, isDown /*addOutsideTargets*/);





        // Handle the case where we did not find a window.

        // Handle the case where we did not find a window.

        if (newTouchedWindowHandle == nullptr) {

        if (newTouchedWindowHandle == nullptr) {
@@ -2100,7 +2111,7 @@ InputEventInjectionResult InputDispatcher::findTouchedWindowTargetsLocked( 
        }

        }





        std::vector<sp<WindowInfoHandle>> newTouchedWindows =

        std::vector<sp<WindowInfoHandle>> newTouchedWindows =

                findTouchedSpyWindowsAtLocked(displayId, x, y);

                findTouchedSpyWindowsAtLocked(displayId, x, y, isStylus);

        if (newTouchedWindowHandle != nullptr) {

        if (newTouchedWindowHandle != nullptr) {

            // Process the foreground window first so that it is the first to receive the event.

            // Process the foreground window first so that it is the first to receive the event.

            newTouchedWindows.insert(newTouchedWindows.begin(), newTouchedWindowHandle);

            newTouchedWindows.insert(newTouchedWindows.begin(), newTouchedWindowHandle);
@@ -2213,9 +2224,11 @@ InputEventInjectionResult InputDispatcher::findTouchedWindowTargetsLocked( 
            const int32_t x = int32_t(entry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_X));

            const int32_t x = int32_t(entry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_X));

            const int32_t y = int32_t(entry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_Y));

            const int32_t y = int32_t(entry.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_Y));





            const bool isStylus = isPointerFromStylus(entry, 0 /*pointerIndex*/);

            sp<WindowInfoHandle> oldTouchedWindowHandle =

            sp<WindowInfoHandle> oldTouchedWindowHandle =

                    tempTouchState.getFirstForegroundWindowHandle();

                    tempTouchState.getFirstForegroundWindowHandle();

            newTouchedWindowHandle = findTouchedWindowAtLocked(displayId, x, y, &tempTouchState);

            newTouchedWindowHandle =

                    findTouchedWindowAtLocked(displayId, x, y, &tempTouchState, isStylus);





            // Drop touch events if requested by input feature

            // Drop touch events if requested by input feature

            if (newTouchedWindowHandle != nullptr &&

            if (newTouchedWindowHandle != nullptr &&
@@ -2477,8 +2490,12 @@ Failed: 
}

}





void InputDispatcher::finishDragAndDrop(int32_t displayId, float x, float y) {

void InputDispatcher::finishDragAndDrop(int32_t displayId, float x, float y) {

    // Prevent stylus interceptor windows from affecting drag and drop behavior for now, until we

    // have an explicit reason to support it.

    constexpr bool isStylus = false;



    const sp<WindowInfoHandle> dropWindow =

    const sp<WindowInfoHandle> dropWindow =

            findTouchedWindowAtLocked(displayId, x, y, nullptr /*touchState*/,

            findTouchedWindowAtLocked(displayId, x, y, nullptr /*touchState*/, isStylus,

                                      false /*addOutsideTargets*/, true /*ignoreDragWindow*/);

                                      false /*addOutsideTargets*/, true /*ignoreDragWindow*/);

    if (dropWindow) {

    if (dropWindow) {

        vec2 local = dropWindow->getInfo()->transform.transform(x, y);

        vec2 local = dropWindow->getInfo()->transform.transform(x, y);
@@ -2511,8 +2528,12 @@ void InputDispatcher::addDragEventLocked(const MotionEntry& entry) { 
            return;

            return;

        }

        }





        // Prevent stylus interceptor windows from affecting drag and drop behavior for now, until

        // we have an explicit reason to support it.

        constexpr bool isStylus = false;



        const sp<WindowInfoHandle> hoverWindowHandle =

        const sp<WindowInfoHandle> hoverWindowHandle =

                findTouchedWindowAtLocked(entry.displayId, x, y, nullptr /*touchState*/,

                findTouchedWindowAtLocked(entry.displayId, x, y, nullptr /*touchState*/, isStylus,

                                          false /*addOutsideTargets*/, true /*ignoreDragWindow*/);

                                          false /*addOutsideTargets*/, true /*ignoreDragWindow*/);

        // enqueue drag exit if needed.

        // enqueue drag exit if needed.

        if (hoverWindowHandle != mDragState->dragHoverWindowHandle &&

        if (hoverWindowHandle != mDragState->dragHoverWindowHandle &&
@@ -4685,15 +4706,22 @@ void InputDispatcher::setInputWindowsLocked( 
        ALOGD("setInputWindows displayId=%" PRId32 " %s", displayId, windowList.c_str());

        ALOGD("setInputWindows displayId=%" PRId32 " %s", displayId, windowList.c_str());

    }

    }





    // Ensure all tokens are null if the window has feature NO_INPUT_CHANNEL

    // Check preconditions for new input windows

    for (const sp<WindowInfoHandle>& window : windowInfoHandles) {

    for (const sp<WindowInfoHandle>& window : windowInfoHandles) {

        const bool noInputWindow =

        const WindowInfo& info = *window->getInfo();

                window->getInfo()->inputFeatures.test(WindowInfo::Feature::NO_INPUT_CHANNEL);



        // Ensure all tokens are null if the window has feature NO_INPUT_CHANNEL

        const bool noInputWindow = info.inputFeatures.test(WindowInfo::Feature::NO_INPUT_CHANNEL);

        if (noInputWindow && window->getToken() != nullptr) {

        if (noInputWindow && window->getToken() != nullptr) {

            ALOGE("%s has feature NO_INPUT_WINDOW, but a non-null token. Clearing",

            ALOGE("%s has feature NO_INPUT_WINDOW, but a non-null token. Clearing",

                  window->getName().c_str());

                  window->getName().c_str());

            window->releaseChannel();

            window->releaseChannel();

        }

        }



        // Ensure all stylus interceptors are trusted overlays

        LOG_ALWAYS_FATAL_IF(info.interceptsStylus() && !info.trustedOverlay,

                            "%s has feature INTERCEPTS_STYLUS, but is not a trusted overlay.",

                            window->getName().c_str());

    }

    }





    // Copy old handles for release if they are no longer present.

    // Copy old handles for release if they are no longer present.

services/inputflinger/dispatcher/InputDispatcher.h

+5 −9
Original line number Original line Diff line number Diff line
@@ -234,16 +234,12 @@ private: 
    // to transfer focus to a new application.

    // to transfer focus to a new application.

    std::shared_ptr<EventEntry> mNextUnblockedEvent GUARDED_BY(mLock);

    std::shared_ptr<EventEntry> mNextUnblockedEvent GUARDED_BY(mLock);





    sp<android::gui::WindowInfoHandle> findTouchedWindowAtLocked(int32_t displayId, int32_t x,

    sp<android::gui::WindowInfoHandle> findTouchedWindowAtLocked(

                                                                 int32_t y, TouchState* touchState,

            int32_t displayId, int32_t x, int32_t y, TouchState* touchState, bool isStylus = false,

                                                                 bool addOutsideTargets = false,

            bool addOutsideTargets = false, bool ignoreDragWindow = false) REQUIRES(mLock);

                                                                 bool ignoreDragWindow = false)

            REQUIRES(mLock);





    std::vector<sp<android::gui::WindowInfoHandle>> findTouchedSpyWindowsAtLocked(int32_t displayId,

    std::vector<sp<android::gui::WindowInfoHandle>> findTouchedSpyWindowsAtLocked(

                                                                                  int32_t x,

            int32_t displayId, int32_t x, int32_t y, bool isStylus) const REQUIRES(mLock);

                                                                                  int32_t y) const

            REQUIRES(mLock);





    sp<Connection> getConnectionLocked(const sp<IBinder>& inputConnectionToken) const

    sp<Connection> getConnectionLocked(const sp<IBinder>& inputConnectionToken) const

            REQUIRES(mLock);

            REQUIRES(mLock);

services/inputflinger/tests/InputDispatcher_test.cpp

+97 −1
Original line number Original line Diff line number Diff line
@@ -1003,6 +1003,7 @@ public: 
        mInfo.ownerPid = INJECTOR_PID;

        mInfo.ownerPid = INJECTOR_PID;

        mInfo.ownerUid = INJECTOR_UID;

        mInfo.ownerUid = INJECTOR_UID;

        mInfo.displayId = displayId;

        mInfo.displayId = displayId;

        mInfo.trustedOverlay = false;

    }

    }





    sp<FakeWindowHandle> clone(

    sp<FakeWindowHandle> clone(
@@ -1054,7 +1055,9 @@ public: 




    void setFlags(Flags<WindowInfo::Flag> flags) { mInfo.flags = flags; }

    void setFlags(Flags<WindowInfo::Flag> flags) { mInfo.flags = flags; }





    void setInputFeatures(WindowInfo::Feature features) { mInfo.inputFeatures = features; }

    void setInputFeatures(Flags<WindowInfo::Feature> features) { mInfo.inputFeatures = features; }



    void setTrustedOverlay(bool trustedOverlay) { mInfo.trustedOverlay = trustedOverlay; }





    void setWindowTransform(float dsdx, float dtdx, float dtdy, float dsdy) {

    void setWindowTransform(float dsdx, float dtdx, float dtdy, float dsdy) {

        mInfo.transform.set(dsdx, dtdx, dtdy, dsdy);

        mInfo.transform.set(dsdx, dtdx, dtdy, dsdy);
@@ -6600,4 +6603,97 @@ TEST_F(InputDispatcherSpyWindowTest, ReceivesSecondPointerAsDown) { 
    spyRight->consumeMotionDown();

    spyRight->consumeMotionDown();

}

}





class InputDispatcherStylusInterceptorTest : public InputDispatcherTest {

public:

    std::pair<sp<FakeWindowHandle>, sp<FakeWindowHandle>> setupStylusOverlayScenario() {

        std::shared_ptr<FakeApplicationHandle> overlayApplication =

                std::make_shared<FakeApplicationHandle>();

        sp<FakeWindowHandle> overlay =

                new FakeWindowHandle(overlayApplication, mDispatcher, "Stylus interceptor window",

                                     ADISPLAY_ID_DEFAULT);

        overlay->setFocusable(false);

        overlay->setOwnerInfo(111, 111);

        overlay->setFlags(WindowInfo::Flag::NOT_TOUCHABLE | WindowInfo::Flag::SPLIT_TOUCH);

        overlay->setInputFeatures(WindowInfo::Feature::INTERCEPTS_STYLUS);

        overlay->setTrustedOverlay(true);



        std::shared_ptr<FakeApplicationHandle> application =

                std::make_shared<FakeApplicationHandle>();

        sp<FakeWindowHandle> window =

                new FakeWindowHandle(application, mDispatcher, "Application window",

                                     ADISPLAY_ID_DEFAULT);

        window->setFocusable(true);

        window->setOwnerInfo(222, 222);

        window->setFlags(WindowInfo::Flag::SPLIT_TOUCH);



        mDispatcher->setFocusedApplication(ADISPLAY_ID_DEFAULT, application);

        mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {overlay, window}}});

        setFocusedWindow(window);

        window->consumeFocusEvent(true /*hasFocus*/, true /*inTouchMode*/);

        return {std::move(overlay), std::move(window)};

    }



    void sendFingerEvent(int32_t action) {

        NotifyMotionArgs motionArgs =

                generateMotionArgs(action, AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_STYLUS,

                                   ADISPLAY_ID_DEFAULT, {PointF{20, 20}});

        mDispatcher->notifyMotion(&motionArgs);

    }



    void sendStylusEvent(int32_t action) {

        NotifyMotionArgs motionArgs =

                generateMotionArgs(action, AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_STYLUS,

                                   ADISPLAY_ID_DEFAULT, {PointF{30, 40}});

        motionArgs.pointerProperties[0].toolType = AMOTION_EVENT_TOOL_TYPE_STYLUS;

        mDispatcher->notifyMotion(&motionArgs);

    }

};



TEST_F(InputDispatcherStylusInterceptorTest, UntrustedOverlay_AbortsDispatcher) {

    auto [overlay, window] = setupStylusOverlayScenario();

    overlay->setTrustedOverlay(false);

    // Configuring an untrusted overlay as a stylus interceptor should cause Dispatcher to abort.

    ASSERT_DEATH(mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {overlay, window}}}),

                 ".* not a trusted overlay");

}



TEST_F(InputDispatcherStylusInterceptorTest, ConsmesOnlyStylusEvents) {

    auto [overlay, window] = setupStylusOverlayScenario();

    mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {overlay, window}}});



    sendStylusEvent(AMOTION_EVENT_ACTION_DOWN);

    overlay->consumeMotionDown();

    sendStylusEvent(AMOTION_EVENT_ACTION_UP);

    overlay->consumeMotionUp();



    sendFingerEvent(AMOTION_EVENT_ACTION_DOWN);

    window->consumeMotionDown();

    sendFingerEvent(AMOTION_EVENT_ACTION_UP);

    window->consumeMotionUp();



    overlay->assertNoEvents();

    window->assertNoEvents();

}



TEST_F(InputDispatcherStylusInterceptorTest, SpyWindowStylusInterceptor) {

    auto [overlay, window] = setupStylusOverlayScenario();

    overlay->setInputFeatures(overlay->getInfo()->inputFeatures | WindowInfo::Feature::SPY);

    mDispatcher->setInputWindows({{ADISPLAY_ID_DEFAULT, {overlay, window}}});



    sendStylusEvent(AMOTION_EVENT_ACTION_DOWN);

    overlay->consumeMotionDown();

    window->consumeMotionDown();

    sendStylusEvent(AMOTION_EVENT_ACTION_UP);

    overlay->consumeMotionUp();

    window->consumeMotionUp();



    sendFingerEvent(AMOTION_EVENT_ACTION_DOWN);

    window->consumeMotionDown();

    sendFingerEvent(AMOTION_EVENT_ACTION_UP);

    window->consumeMotionUp();



    overlay->assertNoEvents();

    window->assertNoEvents();

}



} // namespace android::inputdispatcher
 
} // namespace android::inputdispatcher