Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 11fa37eb authored by Matthew Fritze's avatar Matthew Fritze Committed by Matt Fritze
Browse files

Add Settings Search Regression test

Take a snapshot of the currently available search results, and
verify that search results aren't accidentally removed.

We use 4 items to identify a search result:
- Title
- Data Key
- Slice Uri

We use Title & Key to identify the search result, since they should not change.
The Slice Uri is used to make sure we don't regress on Slice Availability.

Test: runtest --path packages/apps/Settings/tests/unit/src/com/android/settings/search/SettingsSearchResultRegressionTest.java
Change-Id: I22498229bbcd1e90c9e0a026af9df4367a98190a
parent 10995c9b
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@
    <uses-permission android:name="android.permission.BLUETOOTH_ADMIN" />
    <uses-permission android:name="android.permission.READ_LOGS" />
    <uses-permission android:name="android.permission.MOUNT_UNMOUNT_FILESYSTEMS" />
    <uses-permission android:name="android.permission.READ_SEARCH_INDEXABLES"/>

    <instrumentation
            android:name="android.support.test.runner.AndroidJUnitRunner"
+623 −0

File added.

Preview size limit exceeded, changes collapsed.

+81 −0
Original line number Diff line number Diff line
package uitests.src.com.android.settings.search;

import android.text.TextUtils;
import java.util.Objects;


/**
 * Data class for {@link com.android.settings.search.SettingsSearchResultRegressionTest}
 */
public class SearchData {
  public final String title;
  public final String key;

  public String getTitle() {
    return title;
  }

  public String getKey() {
    return key;
  }
  public static final String DELIM = ";";

  public static SearchData from(String searchDataString) {
    String[] split = searchDataString.trim().split(DELIM, -1);

    if (split.length != 2) {
      throw new IllegalArgumentException("Arg is invalid: " + searchDataString);
    }

    return new SearchData.Builder()
        .setTitle(split[0])
        .setKey(split[1])
        .build();
  }

  @Override
  public String toString() {
    return title + DELIM + key;
  }

  @Override
  public boolean equals(Object obj) {
    if (!(obj instanceof SearchData)) {
      return false;
    }

    SearchData other = (SearchData) obj;
    return TextUtils.equals(this.title, other.title)
        && TextUtils.equals(this.key, other.key);
  }

  @Override
  public int hashCode() {
    return Objects.hash(title, key);
  }

  private SearchData(
      SearchData.Builder builder) {
    this.title = builder.title;
    this.key = builder.key;
  }

  public static class Builder {
    protected String title = "";
    protected String key = "";

    public SearchData build() {
      return new SearchData(this);
    }

    public SearchData.Builder setTitle(String title) {
      this.title = title;
      return this;
    }

    public SearchData.Builder setKey(String key) {
      this.key = key;
      return this;
    }
  }
}
+210 −0
Original line number Diff line number Diff line

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.android.settings.search;

import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;

import android.content.ContentResolver;
import android.content.Context;
import android.database.Cursor;
import android.net.Uri;

import android.platform.test.annotations.Presubmit;
import android.support.test.InstrumentationRegistry;
import android.support.test.filters.SmallTest;
import android.support.test.runner.AndroidJUnit4;
import android.text.TextUtils;

import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import uitests.src.com.android.settings.search.SearchData;

@SmallTest
@RunWith(AndroidJUnit4.class)
public class SettingsSearchResultRegressionTest {

  private Context mContext;

  public interface IndexColumns {
    String DATA_TITLE = "data_title";
    String DATA_KEY_REF = "data_key_reference";
  }

  private static final String ERROR_RESULTS_MISSING =
      "\nSettings search results missing. \n"
          + "If the changes are intentional, we want to update the master-list.\n";

  private static final String ERROR_NEW_RESULTS =
      "\nNew settings search results have been found.\nIf the changes are intentional, we want to"
      + "prevent the new results from regressing.\n";

  private static final String ERROR_RERUN_TEST =
      "Please re-run the test \"generate_search_result_list\" by removing the '@Ignore' annotation above 'generate_search_result_list' test, and run: \n"
      + "$ runtest --path packages/apps/Settings/tests/uitests/src/com/android/settings/search/SettingsSearchResultRegressionTest.java \n"
      + "and copy the output into 'packages/apps/Settings/tests/uitests/assets/search_result_list'\n";


  @Before
  public void setUp() {
    mContext = InstrumentationRegistry.getContext();
  }

  /**
   * Tests that the set of search results does not regress.
   * <p>
   *     The data set used here (/tests/unit/assets/search_results_list) needs to be updated
   *     every once in a while so that we can check newly added results.
   * </p>
   */
  @Test
  @Presubmit
  public void searchResultsDoNotRegress() {
    final ContentResolver resolver = mContext.getContentResolver();
    final Uri uri = getTestProviderUri();
    final Cursor cursor = resolver.query(uri, null, null, null, null);

    if (cursor == null) {
      // Assume Settings Intelligence is wrong.
      return;
    }

    final Set<SearchData> availableSearchResults = getSearchDataFromCursor(cursor);
    final Set<SearchData> registeredSearchResults = getRegisteredResults();

    // Seed with results that we expect
    final Set<SearchData> missingSearchResults = new HashSet<>(registeredSearchResults);
    // Seed with results that are available
    final Set<SearchData> newSearchResults = new HashSet<>(availableSearchResults);

    // Remove all available results, leaving results that have been removed.
    missingSearchResults.removeAll(availableSearchResults);
    // Remove all results we expect, leaving results that have not yet been registered.
    newSearchResults.removeAll(registeredSearchResults);

    assertWithMessage(ERROR_RESULTS_MISSING + ERROR_RERUN_TEST)
        .that(missingSearchResults).isEmpty();
    assertWithMessage(ERROR_NEW_RESULTS + ERROR_RERUN_TEST).that(newSearchResults).isEmpty();
  }

  // TODO (b/113907111) add a test to catch duplicate title search results.

  /**
   * Test to generate a new list of search results. Uncomment the Test annotation and run the
   * test to generate the list.
   */
  @Ignore
  @Test
  public void generate_search_result_list() {
    final ContentResolver resolver = mContext.getContentResolver();
    final Uri uri = getTestProviderUri();
    final Cursor cursor = resolver.query(uri, null, null, null, null);
    final List<SearchData> availableSearchResults =
        new ArrayList<>(getSearchDataFromCursor(cursor));

    Collections.sort(availableSearchResults, Comparator.comparing(SearchData::getTitle)
        .thenComparing(SearchData::getKey));

    assertThat(generateListFromSearchData(availableSearchResults)).isNull();
  }

  private Set<SearchData> getSearchDataFromCursor(Cursor cursor) {
    final Set<SearchData> searchData = new HashSet<>();

    final int titleIndex = cursor.getColumnIndex(
        IndexColumns.DATA_TITLE);
    final int keyIndex = cursor.getColumnIndex(
        IndexColumns.DATA_KEY_REF);

    while (cursor.moveToNext()) {
      String title = cursor.getString(titleIndex);
      String key = cursor.getString(keyIndex);

      if (TextUtils.isEmpty(title)) {
        title = "";
      }

      if (TextUtils.isEmpty(key)) {
        key = "";
      }

      searchData.add(new SearchData.Builder()
          .setTitle(title)
          .setKey(key)
          .build());
    }

    return searchData;
  }

  /**
   * Utility method to generate the list of search results that this class uses to validate
   * results.
   */
  private String generateListFromSearchData(List<SearchData> searchData) {
    StringBuilder builder = new StringBuilder();
    for (SearchData searchResult : searchData) {
      builder.append(searchResult.title)
          .append(
              SearchData.DELIM)
          .append(searchResult.key)
          .append("\n");
    }
    return builder.toString();
  }

  private Uri getTestProviderUri() {
    return new Uri.Builder()
        .scheme(ContentResolver.SCHEME_CONTENT)
        .authority("com.google.android.settings.intelligence.modules.search.regression")
        .build();
  }

  private Set<SearchData> getRegisteredResults() {
    final String filename = "search_results_list";
    final Set<SearchData> registeredResults = new HashSet<>();

    try {
      final InputStream in = mContext.getAssets().open(filename);
      BufferedReader reader = new BufferedReader(new InputStreamReader(in));
      String line;
      while ((line = reader.readLine()) != null) {
        registeredResults.add(
            SearchData.from(line));
      }
    } catch (Exception e) {
      throw new IllegalArgumentException("Error initializing registered result list "
          + filename, e);
    }

    return registeredResults;
  }
}
 No newline at end of file