Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 43a899fc authored by Tony Mak's avatar Tony Mak
Browse files

Populate person and reference time, uses more than the last message in NAS

Test: atest SmartActionsHelperTest.java

BUG: 120809869

Change-Id: I6143c977a096135ae254d4106f02ee6d8140a37b
parent 82fa8d94
Loading
Loading
Loading
Loading
+52 −30
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ package android.ext.services.notification;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.app.Notification;
import android.app.Person;
import android.app.RemoteAction;
import android.content.Context;
import android.os.Bundle;
@@ -31,8 +32,14 @@ import android.view.textclassifier.TextClassificationManager;
import android.view.textclassifier.TextClassifier;
import android.view.textclassifier.TextLinks;

import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.List;
import java.util.stream.Collectors;

@@ -50,6 +57,8 @@ public class SmartActionsHelper {
    private static final int MAX_ACTIONS_PER_LINK = 1;
    private static final int MAX_SMART_ACTIONS = 3;
    private static final int MAX_SUGGESTED_REPLIES = 3;
    // TODO: Make this configurable.
    private static final int MAX_MESSAGES_TO_EXTRACT = 5;

    private static final ConversationActions.TypeConfig TYPE_CONFIG =
            new ConversationActions.TypeConfig.Builder().setIncludedTypes(
@@ -64,9 +73,6 @@ public class SmartActionsHelper {

    /**
     * Adds action adjustments based on the notification contents.
     *
     * TODO: Once we have a API in {@link TextClassificationManager} to predict smart actions
     * from notification text / message, we can replace most of the code here by consuming that API.
     */
    @NonNull
    ArrayList<Notification.Action> suggestActions(@Nullable Context context,
@@ -84,9 +90,13 @@ public class SmartActionsHelper {
        if (tcm == null) {
            return EMPTY_ACTION_LIST;
        }
        List<ConversationActions.Message> messages = extractMessages(entry.getNotification());
        if (messages.isEmpty()) {
            return EMPTY_ACTION_LIST;
        }
        // TODO: Move to TextClassifier.suggestConversationActions once it is ready.
        return suggestActionsFromText(
                tcm,
                getMostSalientActionText(entry.getNotification()), MAX_SMART_ACTIONS);
                tcm, messages.get(messages.size() - 1).getText(), MAX_SMART_ACTIONS);
    }

    ArrayList<CharSequence> suggestReplies(@Nullable Context context,
@@ -104,16 +114,12 @@ public class SmartActionsHelper {
        if (tcm == null) {
            return EMPTY_REPLY_LIST;
        }
        CharSequence text = getMostSalientActionText(entry.getNotification());
        // TODO: Populate the actual Person object from the message.
        ConversationActions.Message message =
                new ConversationActions.Message.Builder(
                        ConversationActions.Message.PERSON_USER_REMOTE)
                        .setText(text)
                        .build();

        List<ConversationActions.Message> messages = extractMessages(entry.getNotification());
        if (messages.isEmpty()) {
            return EMPTY_REPLY_LIST;
        }
        ConversationActions.Request request =
                new ConversationActions.Request.Builder(Collections.singletonList(message))
                new ConversationActions.Request.Builder(messages)
                        .setMaxSuggestions(MAX_SUGGESTED_REPLIES)
                        .setHints(HINTS)
                        .setTypeConfig(TYPE_CONFIG)
@@ -142,10 +148,6 @@ public class SmartActionsHelper {
        if (!Process.myUserHandle().equals(entry.getSbn().getUser())) {
            return false;
        }
        if (notification.actions != null
                && notification.actions.length >= Notification.MAX_ACTION_BUTTONS) {
            return false;
        }
        if ((notification.flags & FLAG_MASK_INELGIBILE_FOR_ACTIONS) != 0) {
            return false;
        }
@@ -178,21 +180,41 @@ public class SmartActionsHelper {

    /** Returns the text most salient for action extraction in a notification. */
    @Nullable
    private CharSequence getMostSalientActionText(@NonNull Notification notification) {
        /* If it's messaging style, use the most recent message. */
        // TODO: Use the last few X messages instead and take the Person object into consideration.
    private List<ConversationActions.Message> extractMessages(@NonNull Notification notification) {
        Parcelable[] messages = notification.extras.getParcelableArray(Notification.EXTRA_MESSAGES);
        if (messages != null && messages.length != 0) {
            Bundle lastMessage = (Bundle) messages[messages.length - 1];
            CharSequence lastMessageText =
                    lastMessage.getCharSequence(Notification.MessagingStyle.Message.KEY_TEXT);
            if (!TextUtils.isEmpty(lastMessageText)) {
                return lastMessageText;
        if (messages == null || messages.length == 0) {
            return Arrays.asList(new ConversationActions.Message.Builder(
                    ConversationActions.Message.PERSON_USER_REMOTE)
                    .setText(notification.extras.getCharSequence(Notification.EXTRA_TEXT))
                    .build());
        }
        Person localUser = notification.extras.getParcelable(Notification.EXTRA_MESSAGING_PERSON);
        Deque<ConversationActions.Message> extractMessages = new ArrayDeque<>();
        for (int i = messages.length - 1; i >= 0; i--) {
            Notification.MessagingStyle.Message message =
                    Notification.MessagingStyle.Message.getMessageFromBundle((Bundle) messages[i]);
            if (message == null) {
                continue;
            }
            Person senderPerson = message.getSenderPerson();
            // Skip encoding once the sender is missing as it is important to distinguish
            // local user and remote user when generating replies.
            if (senderPerson == null) {
                break;
            }
            Person author = localUser != null && localUser.equals(senderPerson)
                    ? ConversationActions.Message.PERSON_USER_LOCAL : senderPerson;
            extractMessages.push(new ConversationActions.Message.Builder(author)
                    .setText(message.getText())
                    .setReferenceTime(
                            ZonedDateTime.ofInstant(Instant.ofEpochMilli(message.getTimestamp()),
                                    ZoneOffset.systemDefault()))
                    .build());
            if (extractMessages.size() >= MAX_MESSAGES_TO_EXTRACT) {
                break;
            }
        }

        // Fall back to using the normal text.
        return notification.extras.getCharSequence(Notification.EXTRA_TEXT);
        return new ArrayList<>(extractMessages);
    }

    /** Returns a list of actions to act on entities in a given piece of text. */
+236 −0
Original line number Diff line number Diff line
/**
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package android.ext.services.notification;

import static com.google.common.truth.Truth.assertAbout;
import static com.google.common.truth.Truth.assertThat;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import android.annotation.NonNull;
import android.app.Notification;
import android.app.Person;
import android.content.Context;
import android.os.Process;
import android.service.notification.StatusBarNotification;
import android.support.test.InstrumentationRegistry;
import android.support.test.runner.AndroidJUnit4;
import android.view.textclassifier.ConversationActions;
import android.view.textclassifier.TextClassificationManager;
import android.view.textclassifier.TextClassifier;

import com.google.common.truth.FailureStrategy;
import com.google.common.truth.Subject;
import com.google.common.truth.SubjectFactory;

import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;

import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;

import javax.annotation.Nullable;

@RunWith(AndroidJUnit4.class)
public class SmartActionHelperTest {

    private SmartActionsHelper mSmartActionsHelper = new SmartActionsHelper();
    private Context mContext;
    @Mock private TextClassifier mTextClassifier;
    @Mock private NotificationEntry mNotificationEntry;
    @Mock private StatusBarNotification mStatusBarNotification;
    private Notification.Builder mNotificationBuilder;
    private AssistantSettings mSettings;

    @Before
    public void setup() {
        MockitoAnnotations.initMocks(this);
        mContext = InstrumentationRegistry.getTargetContext();

        mContext.getSystemService(TextClassificationManager.class)
                .setTextClassifier(mTextClassifier);
        when(mTextClassifier.suggestConversationActions(any(ConversationActions.Request.class)))
                .thenReturn(new ConversationActions(Collections.emptyList()));

        when(mNotificationEntry.getSbn()).thenReturn(mStatusBarNotification);
        // The notification is eligible to have smart suggestions.
        when(mNotificationEntry.hasInlineReply()).thenReturn(true);
        when(mNotificationEntry.isMessaging()).thenReturn(true);
        when(mStatusBarNotification.getPackageName()).thenReturn("random.app");
        when(mStatusBarNotification.getUser()).thenReturn(Process.myUserHandle());
        mNotificationBuilder = new Notification.Builder(mContext, "channel");
        mSettings = AssistantSettings.createForTesting(
                null, null, Process.myUserHandle().getIdentifier(), null);
        mSettings.mGenerateActions = true;
        mSettings.mGenerateReplies = true;
    }

    @Test
    public void testSuggestReplies_notMessagingApp() {
        when(mNotificationEntry.isMessaging()).thenReturn(false);
        ArrayList<CharSequence> textReplies =
                mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings);
        assertThat(textReplies).isEmpty();
    }

    @Test
    public void testSuggestReplies_noInlineReply() {
        when(mNotificationEntry.hasInlineReply()).thenReturn(false);
        ArrayList<CharSequence> textReplies =
                mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings);
        assertThat(textReplies).isEmpty();
    }

    @Test
    public void testSuggestReplies_nonMessageStyle() {
        Notification notification = mNotificationBuilder.setContentText("Where are you?").build();
        when(mNotificationEntry.getNotification()).thenReturn(notification);

        List<ConversationActions.Message> messages = getMessagesInRequest();
        assertThat(messages).hasSize(1);
        MessageSubject.assertThat(messages.get(0)).hasText("Where are you?");
    }

    @Test
    public void testSuggestReplies_messageStyle() {
        Person me = new Person.Builder().setName("Me").build();
        Person userA = new Person.Builder().setName("A").build();
        Person userB = new Person.Builder().setName("B").build();
        Notification.MessagingStyle style =
                new Notification.MessagingStyle(me)
                        .addMessage("firstMessage", 1000, (Person) null)
                        .addMessage("secondMessage", 2000, me)
                        .addMessage("thirdMessage", 3000, userA)
                        .addMessage("fourthMessage", 4000, userB);
        Notification notification =
                mNotificationBuilder
                        .setContentText("You have three new messages")
                        .setStyle(style)
                        .build();
        when(mNotificationEntry.getNotification()).thenReturn(notification);

        List<ConversationActions.Message> messages = getMessagesInRequest();
        assertThat(messages).hasSize(3);

        ConversationActions.Message secondMessage = messages.get(0);
        MessageSubject.assertThat(secondMessage).hasText("secondMessage");
        MessageSubject.assertThat(secondMessage)
                .hasPerson(ConversationActions.Message.PERSON_USER_LOCAL);
        MessageSubject.assertThat(secondMessage)
                .hasReferenceTime(createZonedDateTimeFromMsUtc(2000));

        ConversationActions.Message thirdMessage = messages.get(1);
        MessageSubject.assertThat(thirdMessage).hasText("thirdMessage");
        MessageSubject.assertThat(thirdMessage).hasPerson(userA);
        MessageSubject.assertThat(thirdMessage)
                .hasReferenceTime(createZonedDateTimeFromMsUtc(3000));

        ConversationActions.Message fourthMessage = messages.get(2);
        MessageSubject.assertThat(fourthMessage).hasText("fourthMessage");
        MessageSubject.assertThat(fourthMessage).hasPerson(userB);
        MessageSubject.assertThat(fourthMessage)
                .hasReferenceTime(createZonedDateTimeFromMsUtc(4000));
    }

    @Test
    public void testSuggestReplies_messageStyle_noPerson() {
        Person me = new Person.Builder().setName("Me").build();
        Notification.MessagingStyle style =
                new Notification.MessagingStyle(me).addMessage("message", 1000, (Person) null);
        Notification notification =
                mNotificationBuilder
                        .setContentText("You have one new message")
                        .setStyle(style)
                        .build();
        when(mNotificationEntry.getNotification()).thenReturn(notification);

        mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings);

        verify(mTextClassifier, never())
                .suggestConversationActions(any(ConversationActions.Request.class));
    }

    private ZonedDateTime createZonedDateTimeFromMsUtc(long msUtc) {
        return ZonedDateTime.ofInstant(Instant.ofEpochMilli(msUtc), ZoneOffset.systemDefault());
    }

    private List<ConversationActions.Message> getMessagesInRequest() {
        mSmartActionsHelper.suggestReplies(mContext, mNotificationEntry, mSettings);

        ArgumentCaptor<ConversationActions.Request> argumentCaptor =
                ArgumentCaptor.forClass(ConversationActions.Request.class);
        verify(mTextClassifier).suggestConversationActions(argumentCaptor.capture());
        ConversationActions.Request request = argumentCaptor.getValue();
        return request.getConversation();
    }

    private static final class MessageSubject
            extends Subject<MessageSubject, ConversationActions.Message> {

        private static final SubjectFactory<MessageSubject, ConversationActions.Message> FACTORY =
                new SubjectFactory<MessageSubject, ConversationActions.Message>() {
                    @Override
                    public MessageSubject getSubject(
                            @NonNull FailureStrategy failureStrategy,
                            @NonNull ConversationActions.Message subject) {
                        return new MessageSubject(failureStrategy, subject);
                    }
                };

        private MessageSubject(
                FailureStrategy failureStrategy, @Nullable ConversationActions.Message subject) {
            super(failureStrategy, subject);
        }

        private void hasText(String text) {
            if (!Objects.equals(text, getSubject().getText().toString())) {
                failWithBadResults("has text", text, "has", getSubject().getText());
            }
        }

        private void hasPerson(Person person) {
            if (!Objects.equals(person, getSubject().getAuthor())) {
                failWithBadResults("has author", person, "has", getSubject().getAuthor());
            }
        }

        private void hasReferenceTime(ZonedDateTime referenceTime) {
            if (!Objects.equals(referenceTime, getSubject().getReferenceTime())) {
                failWithBadResults(
                        "has reference time",
                        referenceTime,
                        "has",
                        getSubject().getReferenceTime());
            }
        }

        private static MessageSubject assertThat(ConversationActions.Message message) {
            return assertAbout(FACTORY).that(message);
        }
    }
}