Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a5ba9f1f authored by Arpit Singh's avatar Arpit Singh
Browse files

Enable cursor to transition across multiple displays 

This CL enables cursor to move between displays. It uses a fake
topology that assumes all available displays are connected in the
following order:
default-display (top-edge) -> next-display (right-edge)
                           -> next-display (right-edge) ...

Test: presubmit
Bug: 367659738
Bug: 367660694
Flag: com.android.input.flags.connected_displays_cursor
Change-Id: Iff2b9eea52714fec00eadb64b0014df6e7c65916
parent f9854d27

services/inputflinger/PointerChoreographer.cpp

+177 −38
Original line number Original line Diff line number Diff line
@@ -103,6 +103,9 @@ std::unordered_set<ui::LogicalDisplayId> getPrivacySensitiveDisplaysFromWindowIn 




// --- PointerChoreographer ---

// --- PointerChoreographer ---





const bool PointerChoreographer::IS_TOPOLOGY_AWARE =

        com::android::input::flags::connected_displays_cursor();



PointerChoreographer::PointerChoreographer(InputListenerInterface& inputListener,

PointerChoreographer::PointerChoreographer(InputListenerInterface& inputListener,

                                           PointerChoreographerPolicyInterface& policy)

                                           PointerChoreographerPolicyInterface& policy)

      : PointerChoreographer(

      : PointerChoreographer(
@@ -204,12 +207,16 @@ void PointerChoreographer::fadeMouseCursorOnKeyPress(const android::NotifyKeyArg 
}

}





NotifyMotionArgs PointerChoreographer::processMotion(const NotifyMotionArgs& args) {

NotifyMotionArgs PointerChoreographer::processMotion(const NotifyMotionArgs& args) {

    NotifyMotionArgs newArgs(args);

    PointerDisplayChange pointerDisplayChange;

    { // acquire lock

        std::scoped_lock _l(mLock);

        std::scoped_lock _l(mLock);



        if (isFromMouse(args)) {

        if (isFromMouse(args)) {

        return processMouseEventLocked(args);

            newArgs = processMouseEventLocked(args);

            pointerDisplayChange = calculatePointerDisplayChangeToNotify();

        } else if (isFromTouchpad(args)) {

        } else if (isFromTouchpad(args)) {

        return processTouchpadEventLocked(args);

            newArgs = processTouchpadEventLocked(args);

            pointerDisplayChange = calculatePointerDisplayChangeToNotify();

        } else if (isFromDrawingTablet(args)) {

        } else if (isFromDrawingTablet(args)) {

            processDrawingTabletEventLocked(args);

            processDrawingTabletEventLocked(args);

        } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) {

        } else if (mStylusPointerIconEnabled && isStylusHoverEvent(args)) {
@@ -217,7 +224,13 @@ NotifyMotionArgs PointerChoreographer::processMotion(const NotifyMotionArgs& arg 
        } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) {

        } else if (isFromSource(args.source, AINPUT_SOURCE_TOUCHSCREEN)) {

            processTouchscreenAndStylusEventLocked(args);

            processTouchscreenAndStylusEventLocked(args);

        }

        }

    return args;

    } // release lock



    if (pointerDisplayChange) {

        // pointer display may have changed if mouse crossed display boundary

        notifyPointerDisplayChange(pointerDisplayChange, mPolicy);

    }

    return newArgs;

}

}





NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotionArgs& args) {

NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotionArgs& args) {
@@ -242,16 +255,10 @@ NotifyMotionArgs PointerChoreographer::processMouseEventLocked(const NotifyMotio 
        pc.setPosition(args.xCursorPosition, args.yCursorPosition);

        pc.setPosition(args.xCursorPosition, args.yCursorPosition);

    } else {

    } else {

        // This is a relative mouse, so move the cursor by the specified amount.

        // This is a relative mouse, so move the cursor by the specified amount.

        const float deltaX = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);

        processPointerDeviceMotionEventLocked(/*byref*/ newArgs, /*byref*/ pc);

        const float deltaY = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);

        pc.move(deltaX, deltaY);

        const auto [x, y] = pc.getPosition();

        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);

        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);

        newArgs.xCursorPosition = x;

        newArgs.yCursorPosition = y;

    }

    }

    if (canUnfadeOnDisplay(displayId)) {

    // Note displayId may have changed if the cursor moved to a different display

    if (canUnfadeOnDisplay(newArgs.displayId)) {

        pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);

        pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);

    }

    }

    return newArgs;

    return newArgs;
@@ -265,24 +272,9 @@ NotifyMotionArgs PointerChoreographer::processTouchpadEventLocked(const NotifyMo 
    newArgs.displayId = displayId;

    newArgs.displayId = displayId;

    if (args.getPointerCount() == 1 && args.classification == MotionClassification::NONE) {

    if (args.getPointerCount() == 1 && args.classification == MotionClassification::NONE) {

        // This is a movement of the mouse pointer.

        // This is a movement of the mouse pointer.

        const float deltaX = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);

        processPointerDeviceMotionEventLocked(/*byref*/ newArgs, /*byref*/ pc);

        const float deltaY = args.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);

        pc.move(deltaX, deltaY);

        if (canUnfadeOnDisplay(displayId)) {

            pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);

        }



        const auto [x, y] = pc.getPosition();

        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);

        newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);

        newArgs.xCursorPosition = x;

        newArgs.yCursorPosition = y;

    } else {

    } else {

        // This is a trackpad gesture with fake finger(s) that should not move the mouse pointer.

        // This is a trackpad gesture with fake finger(s) that should not move the mouse pointer.

        if (canUnfadeOnDisplay(displayId)) {

            pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);

        }



        const auto [x, y] = pc.getPosition();

        const auto [x, y] = pc.getPosition();

        for (uint32_t i = 0; i < newArgs.getPointerCount(); i++) {

        for (uint32_t i = 0; i < newArgs.getPointerCount(); i++) {

            newArgs.pointerCoords[i].setAxisValue(AMOTION_EVENT_AXIS_X,

            newArgs.pointerCoords[i].setAxisValue(AMOTION_EVENT_AXIS_X,
@@ -293,9 +285,61 @@ NotifyMotionArgs PointerChoreographer::processTouchpadEventLocked(const NotifyMo 
        newArgs.xCursorPosition = x;

        newArgs.xCursorPosition = x;

        newArgs.yCursorPosition = y;

        newArgs.yCursorPosition = y;

    }

    }



    // Note displayId may have changed if the cursor moved to a different display

    if (canUnfadeOnDisplay(newArgs.displayId)) {

        pc.unfade(PointerControllerInterface::Transition::IMMEDIATE);

    }

    return newArgs;

    return newArgs;

}

}





void PointerChoreographer::processPointerDeviceMotionEventLocked(NotifyMotionArgs& newArgs,

                                                                 PointerControllerInterface& pc) {

    const float deltaX = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_X);

    const float deltaY = newArgs.pointerCoords[0].getAxisValue(AMOTION_EVENT_AXIS_RELATIVE_Y);

    vec2 unconsumedDelta = pc.move(deltaX, deltaY);

    if (IS_TOPOLOGY_AWARE && (std::abs(unconsumedDelta.x) > 0 || std::abs(unconsumedDelta.y) > 0)) {

        handleUnconsumedDeltaLocked(pc, unconsumedDelta);

        // pointer may have moved to a different viewport

        newArgs.displayId = pc.getDisplayId();

    }

    const auto [x, y] = pc.getPosition();

    newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_X, x);

    newArgs.pointerCoords[0].setAxisValue(AMOTION_EVENT_AXIS_Y, y);

    newArgs.xCursorPosition = x;

    newArgs.yCursorPosition = y;

}



void PointerChoreographer::handleUnconsumedDeltaLocked(PointerControllerInterface& pc,

                                                       const vec2& unconsumedDelta) {

    const ui::LogicalDisplayId sourceDisplayId = pc.getDisplayId();

    const auto& sourceViewport = *findViewportByIdLocked(sourceDisplayId);

    std::optional<AdjacentDisplay> destinationDisplay =

            findDestinationDisplayLocked(sourceViewport, unconsumedDelta);

    if (!destinationDisplay) {

        // no adjacent display

        return;

    }



    const DisplayViewport* destinationViewport =

            findViewportByIdLocked(destinationDisplay->displayId);

    if (destinationViewport == nullptr) {

        // Topology is likely out of sync with viewport info, wait for them to be updated

        LOG(WARNING) << "Cannot find viewport for adjacent display "

                     << destinationDisplay->displayId << "of source display " << sourceDisplayId;

        return;

    }



    mDefaultMouseDisplayId = destinationDisplay->displayId;

    auto pcNode = mMousePointersByDisplay.extract(sourceDisplayId);

    pcNode.key() = destinationDisplay->displayId;

    mMousePointersByDisplay.insert(std::move(pcNode));



    // This will place cursor at the center of the target display for now

    // TODO (b/367660694) place the cursor at the appropriate position in destination display

    pc.setDisplayViewport(*destinationViewport);

}



void PointerChoreographer::processDrawingTabletEventLocked(const android::NotifyMotionArgs& args) {

void PointerChoreographer::processDrawingTabletEventLocked(const android::NotifyMotionArgs& args) {

    if (args.displayId == ui::LogicalDisplayId::INVALID) {

    if (args.displayId == ui::LogicalDisplayId::INVALID) {

        return;

        return;
@@ -441,7 +485,8 @@ void PointerChoreographer::processDeviceReset(const NotifyDeviceResetArgs& args) 
}

}





void PointerChoreographer::onControllerAddedOrRemovedLocked() {

void PointerChoreographer::onControllerAddedOrRemovedLocked() {

    if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows()) {

    if (!com::android::input::flags::hide_pointer_indicators_for_secure_windows() &&

        !IS_TOPOLOGY_AWARE) {

        return;

        return;

    }

    }

    bool requireListener = !mTouchPointersByDevice.empty() || !mMousePointersByDisplay.empty() ||

    bool requireListener = !mTouchPointersByDevice.empty() || !mMousePointersByDisplay.empty() ||
@@ -674,6 +719,10 @@ PointerChoreographer::calculatePointerDisplayChangeToNotify() { 
}

}





void PointerChoreographer::setDefaultMouseDisplayId(ui::LogicalDisplayId displayId) {

void PointerChoreographer::setDefaultMouseDisplayId(ui::LogicalDisplayId displayId) {

    if (IS_TOPOLOGY_AWARE) {

        // default display will be set based on the topology

        return;

    }

    PointerDisplayChange pointerDisplayChange;

    PointerDisplayChange pointerDisplayChange;





    { // acquire lock

    { // acquire lock
@@ -887,6 +936,7 @@ void PointerChoreographer::PointerChoreographerDisplayInfoListener::onWindowInfo 
        mPrivacySensitiveDisplays = std::move(newPrivacySensitiveDisplays);

        mPrivacySensitiveDisplays = std::move(newPrivacySensitiveDisplays);

        mPointerChoreographer->onPrivacySensitiveDisplaysChanged(mPrivacySensitiveDisplays);

        mPointerChoreographer->onPrivacySensitiveDisplaysChanged(mPrivacySensitiveDisplays);

    }

    }

    mPointerChoreographer->populateFakeDisplayTopology(windowInfosUpdate.displayInfos);

}

}





void PointerChoreographer::PointerChoreographerDisplayInfoListener::setInitialDisplayInfos(

void PointerChoreographer::PointerChoreographerDisplayInfoListener::setInitialDisplayInfos(
@@ -907,4 +957,93 @@ void PointerChoreographer::PointerChoreographerDisplayInfoListener:: 
    mPointerChoreographer = nullptr;

    mPointerChoreographer = nullptr;

}

}





void PointerChoreographer::populateFakeDisplayTopology(

        const std::vector<gui::DisplayInfo>& displayInfos) {

    if (!IS_TOPOLOGY_AWARE) {

        return;

    }

    std::scoped_lock _lock(mLock);



    if (displayInfos.size() == mTopology.size()) {

        bool displaysChanged = false;

        for (const auto& displayInfo : displayInfos) {

            if (mTopology.find(displayInfo.displayId) == mTopology.end()) {

                displaysChanged = true;

                break;

            }

        }



        if (!displaysChanged) {

            return;

        }

    }



    // create a fake topology assuming following order

    // default-display (top-edge) -> next-display (right-edge) -> next-display (right-edge) ...

    // ┌─────────┬─────────┐

    // │ next    │ next 2  │ ...

    // ├─────────┼─────────┘

    // │ default │

    // └─────────┘

    mTopology.clear();



    // treat default display as base, in real topology it should be the primary-display

    ui::LogicalDisplayId previousDisplay = ui::LogicalDisplayId::DEFAULT;

    for (const auto& displayInfo : displayInfos) {

        if (displayInfo.displayId == ui::LogicalDisplayId::DEFAULT) {

            continue;

        }

        if (previousDisplay == ui::LogicalDisplayId::DEFAULT) {

            mTopology[previousDisplay].push_back({displayInfo.displayId, DisplayPosition::TOP, 0});

            mTopology[displayInfo.displayId].push_back(

                    {previousDisplay, DisplayPosition::BOTTOM, 0});

        } else {

            mTopology[previousDisplay].push_back(

                    {displayInfo.displayId, DisplayPosition::RIGHT, 0});

            mTopology[displayInfo.displayId].push_back({previousDisplay, DisplayPosition::LEFT, 0});

        }

        previousDisplay = displayInfo.displayId;

    }



    // update default pointer display. In real topology it should be the primary-display

    if (mTopology.find(mDefaultMouseDisplayId) == mTopology.end()) {

        mDefaultMouseDisplayId = ui::LogicalDisplayId::DEFAULT;

    }

}



std::optional<PointerChoreographer::AdjacentDisplay>

PointerChoreographer::findDestinationDisplayLocked(const DisplayViewport& sourceViewport,

                                                   const vec2& unconsumedDelta) const {

    DisplayPosition sourceBoundary;

    if (unconsumedDelta.x > 0) {

        sourceBoundary = DisplayPosition::RIGHT;

    } else if (unconsumedDelta.x < 0) {

        sourceBoundary = DisplayPosition::LEFT;

    } else if (unconsumedDelta.y > 0) {

        sourceBoundary = DisplayPosition::BOTTOM;

    } else {

        sourceBoundary = DisplayPosition::TOP;

    }



    // Choreographer works in un-rotate coordinate space so we need to rotate boundary by viewport

    // orientation to find the rotated boundary

    constexpr int MOD = ftl::to_underlying(ui::Rotation::ftl_last) + 1;

    sourceBoundary = static_cast<DisplayPosition>(

            (ftl::to_underlying(sourceBoundary) + ftl::to_underlying(sourceViewport.orientation)) %

            MOD);



    if (mTopology.find(sourceViewport.displayId) == mTopology.end()) {

        // Topology is likely out of sync with viewport info, wait for them to be updated

        LOG(WARNING) << "Source display missing from topology " << sourceViewport.displayId;

        return std::nullopt;

    }



    for (const auto& adjacentDisplay : mTopology.at(sourceViewport.displayId)) {

        if (adjacentDisplay.position == sourceBoundary) {

            return adjacentDisplay;

        }

    }

    return std::nullopt;

}



} // namespace android
 
} // namespace android

services/inputflinger/PointerChoreographer.h

+29 −0
Original line number Original line Diff line number Diff line
@@ -137,6 +137,8 @@ private: 
    void processTouchscreenAndStylusEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);

    void processTouchscreenAndStylusEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);

    void processStylusHoverEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);

    void processStylusHoverEventLocked(const NotifyMotionArgs& args) REQUIRES(mLock);

    void processDeviceReset(const NotifyDeviceResetArgs& args);

    void processDeviceReset(const NotifyDeviceResetArgs& args);

    void processPointerDeviceMotionEventLocked(NotifyMotionArgs& newArgs,

                                               PointerControllerInterface& pc) REQUIRES(mLock);

    void onControllerAddedOrRemovedLocked() REQUIRES(mLock);

    void onControllerAddedOrRemovedLocked() REQUIRES(mLock);

    void onPrivacySensitiveDisplaysChangedLocked(

    void onPrivacySensitiveDisplaysChangedLocked(

            const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays)

            const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays)
@@ -144,6 +146,32 @@ private: 
    void onPrivacySensitiveDisplaysChanged(

    void onPrivacySensitiveDisplaysChanged(

            const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays);

            const std::unordered_set<ui::LogicalDisplayId>& privacySensitiveDisplays);





    void handleUnconsumedDeltaLocked(PointerControllerInterface& pc, const vec2& unconsumedDelta)

            REQUIRES(mLock);



    // TODO(b/362719483) remove these when real topology is available

    enum class DisplayPosition : int32_t {

        RIGHT = 0,

        TOP = 1,

        LEFT = 2,

        BOTTOM = 3,

        ftl_last = BOTTOM,

    };



    struct AdjacentDisplay {

        ui::LogicalDisplayId displayId;

        DisplayPosition position;

        float offsetPx;

    };

    void populateFakeDisplayTopology(const std::vector<gui::DisplayInfo>& displayInfos);



    std::optional<AdjacentDisplay> findDestinationDisplayLocked(

            const DisplayViewport& sourceViewport, const vec2& unconsumedDelta) const

            REQUIRES(mLock);



    std::unordered_map<ui::LogicalDisplayId, std::vector<AdjacentDisplay>> mTopology

            GUARDED_BY(mLock);



    /* This listener keeps tracks of visible privacy sensitive displays and updates the

    /* This listener keeps tracks of visible privacy sensitive displays and updates the

     * choreographer if there are any changes.

     * choreographer if there are any changes.

     *

     *
@@ -211,6 +239,7 @@ protected: 
                                  const WindowListenerUnregisterConsumer& unregisterListener);

                                  const WindowListenerUnregisterConsumer& unregisterListener);





private:

private:

    const static bool IS_TOPOLOGY_AWARE;

    const WindowListenerRegisterConsumer mRegisterListener;

    const WindowListenerRegisterConsumer mRegisterListener;

    const WindowListenerUnregisterConsumer mUnregisterListener;

    const WindowListenerUnregisterConsumer mUnregisterListener;

};

};

services/inputflinger/include/PointerControllerInterface.h

+6 −2
Original line number Original line Diff line number Diff line
@@ -72,8 +72,12 @@ public: 
    /* Dumps the state of the pointer controller. */

    /* Dumps the state of the pointer controller. */

    virtual std::string dump() = 0;

    virtual std::string dump() = 0;





    /* Move the pointer. */

    /* Move the pointer and return unconsumed delta if the pointer has crossed the current

    virtual void move(float deltaX, float deltaY) = 0;

     * viewport bounds .

     *

     * Return value may be used to move pointer to corresponding adjacent display, if it exists in

     * the display-topology */

    [[nodiscard]] virtual vec2 move(float deltaX, float deltaY) = 0;





    /* Sets the absolute location of the pointer. */

    /* Sets the absolute location of the pointer. */

    virtual void setPosition(float x, float y) = 0;

    virtual void setPosition(float x, float y) = 0;

services/inputflinger/tests/FakePointerController.cpp

+4 −2
Original line number Original line Diff line number Diff line
@@ -148,8 +148,8 @@ bool FakePointerController::isPointerShown() { 
    return mIsPointerShown;

    return mIsPointerShown;

}

}





void FakePointerController::move(float deltaX, float deltaY) {

vec2 FakePointerController::move(float deltaX, float deltaY) {

    if (!mEnabled) return;

    if (!mEnabled) return {};





    mX += deltaX;

    mX += deltaX;

    if (mX < mMinX) mX = mMinX;

    if (mX < mMinX) mX = mMinX;
@@ -157,6 +157,8 @@ void FakePointerController::move(float deltaX, float deltaY) { 
    mY += deltaY;

    mY += deltaY;

    if (mY < mMinY) mY = mMinY;

    if (mY < mMinY) mY = mMinY;

    if (mY > mMaxY) mY = mMaxY;

    if (mY > mMaxY) mY = mMaxY;



    return {};

}

}





void FakePointerController::fade(Transition) {

void FakePointerController::fade(Transition) {

services/inputflinger/tests/FakePointerController.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -65,7 +65,7 @@ public: 




private:

private:

    std::string dump() override { return ""; }

    std::string dump() override { return ""; }

    void move(float deltaX, float deltaY) override;

    vec2 move(float deltaX, float deltaY) override;

    void unfade(Transition) override;

    void unfade(Transition) override;

    void setPresentation(Presentation) override {}

    void setPresentation(Presentation) override {}

    void setSpots(const PointerCoords*, const uint32_t*, BitSet32 spotIdBits,

    void setSpots(const PointerCoords*, const uint32_t*, BitSet32 spotIdBits,