cli-test: a tool for testing command-line programs.

../.clang-format-2

 No newline at end of file

cc_binary {

    name: "cli-test",

    host_supported: true,

    srcs: ["cli-test.cpp"],

    cflags: ["-Wall", "-Werror"],

    shared_libs: ["libbase"],

}

# cli-test

## What?

`cli-test` makes integration testing of command-line tools easier.

## Goals

* Readable syntax. Common cases should be concise, and pretty much anyone

  should be able to read tests even if they've never seen this tool before.

* Minimal issues with quoting. The toybox tests -- being shell scripts --

  quickly become a nightmare of quoting. Using a non ad hoc format (such as

  JSON) would have introduced similar but different quoting issues. A custom

  format, while annoying, side-steps this.

* Sensible defaults. We expect your exit status to be 0 unless you say

  otherwise. We expect nothing on stderr unless you say otherwise. And so on.

* Convention over configuration. Related to sensible defaults, we don't let you

  configure things that aren't absolutely necessary. So you can't keep your test

  data anywhere except in the `files/` subdirectory of the directory containing

  your test, for example.

## Non Goals

* Portability. Just being able to run on Linux (host and device) is sufficient

  for our needs. macOS is probably easy enough if we ever need it, but Windows

  probably doesn't make sense.

## Syntax

Any all-whitespace line, or line starting with `#` is ignored.

A test looks like this:

```

name: unzip -l

command: unzip -l $FILES/example.zip d1/d2/x.txt

after: [ ! -f d1/d2/x.txt ]

expected-stdout:

	Archive:  $FILES/example.zip

	  Length      Date    Time    Name

	---------  ---------- -----   ----

	     1024  2017-06-04 08:45   d1/d2/x.txt

	---------                     -------

	     1024                     1 file

---

```

The `name:` line names the test, and is only for human consumption.

The `command:` line is the command to be run. Additional commands can be

supplied as zero or more `before:` lines (run before `command:`) and zero or

more `after:` lines (run after `command:`). These are useful for both

setup/teardown but also for testing post conditions (as in the example above).

Any `command:`, `before:`, or `after:` line is expected to exit with status 0.

Anything else is considered a test failure.

The `expected-stdout:` line is followed by zero or more tab-prefixed lines that

are otherwise the exact output expected from the command. (There's magic behind

the scenes to rewrite the test files directory to `$FILES` because otherwise any

path in the output would depend on the temporary directory used to run the test.)

There is currently no `expected-stderr:` line. Standard error is implicitly

expected to be empty, and any output will cause a test failure. (The support is

there, but not wired up because we haven't needed it yet.)

The fields can appear in any order, but every test must contain at least a

`name:` line and a `command:` line.

## Output

The output is intended to resemble gtest.

## Future Directions

* It's often useful to be able to *match* against stdout/stderr/a file rather

  than give exact expected output. We might want to add explicit support for

  this. In the meantime, it's possible to use an `after:` with `grep -q` if

  you redirect in your `command:`.

* In addition to using a `before:` (which will fail a test), it can be useful

  to be able to specify tests that would cause us to *skip* a test. An example

  would be "am I running as root?".

* It might be useful to be able to make exit status assertions other than 0?

* There's currently no way (other than the `files/` directory) to share repeated

  setup between tests.

/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

#include <errno.h>

#include <getopt.h>

#include <inttypes.h>

#include <libgen.h>

#include <stdarg.h>

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <sys/wait.h>

#include <time.h>

#include <unistd.h>

#include <string>

#include <vector>

#include <android-base/chrono_utils.h>

#include <android-base/file.h>

#include <android-base/stringprintf.h>

#include <android-base/strings.h>

#include <android-base/test_utils.h>

// Example:

// name: unzip -n

// before: mkdir -p d1/d2

// before: echo b > d1/d2/a.txt

// command: unzip -q -n $FILES/zip/example.zip d1/d2/a.txt && cat d1/d2/a.txt

// expected-stdout:

// 	b

struct Test {

  std::string test_filename;

  std::string name;

  std::string command;

  std::vector<std::string> befores;

  std::vector<std::string> afters;

  std::string expected_stdout;

  std::string expected_stderr;

  int exit_status = 0;

};

static const char* g_progname;

static bool g_verbose;

static const char* g_file;

static size_t g_line;

enum Color { kRed, kGreen };

static void Print(Color c, const char* lhs, const char* fmt, ...) {

  va_list ap;

  va_start(ap, fmt);

  if (isatty(0)) printf("%s", (c == kRed) ? "\e[31m" : "\e[32m");

  printf("%s%s", lhs, isatty(0) ? "\e[0m" : "");

  vfprintf(stdout, fmt, ap);

  putchar('\n');

  va_end(ap);

}

static void Die(int error, const char* fmt, ...) {

  va_list ap;

  va_start(ap, fmt);

  fprintf(stderr, "%s: ", g_progname);

  vfprintf(stderr, fmt, ap);

  if (error != 0) fprintf(stderr, ": %s", strerror(error));

  fprintf(stderr, "\n");

  va_end(ap);

  _exit(1);

}

static void V(const char* fmt, ...) {

  if (!g_verbose) return;

  va_list ap;

  va_start(ap, fmt);

  fprintf(stderr, "           - ");

  vfprintf(stderr, fmt, ap);

  fprintf(stderr, "\n");

  va_end(ap);

}

static void SetField(const char* what, std::string* field, std::string_view value) {

  if (!field->empty()) {

    Die(0, "%s:%zu: %s already set to '%s'", g_file, g_line, what, field->c_str());

  }

  field->assign(value);

}

// Similar to ConsumePrefix, but also trims, so "key:value" and "key: value"

// are equivalent.

static bool Match(std::string* s, const std::string& prefix) {

  if (!android::base::StartsWith(*s, prefix)) return false;

  s->assign(android::base::Trim(s->substr(prefix.length())));

  return true;

}

static void CollectTests(std::vector<Test>* tests, const char* test_filename) {

  std::string absolute_test_filename;

  if (!android::base::Realpath(test_filename, &absolute_test_filename)) {

    Die(errno, "realpath '%s'", test_filename);

  }

  std::string content;

  if (!android::base::ReadFileToString(test_filename, &content)) {

    Die(errno, "couldn't read '%s'", test_filename);

  }

  size_t count = 0;

  g_file = test_filename;

  g_line = 0;

  auto lines = android::base::Split(content, "\n");

  std::unique_ptr<Test> test(new Test);

  while (g_line < lines.size()) {

    auto line = lines[g_line++];

    if (line.empty() || line[0] == '#') continue;

    if (line[0] == '-') {

      if (test->name.empty() || test->command.empty()) {

        Die(0, "%s:%zu: each test requires both a name and a command", g_file, g_line);

      }

      test->test_filename = absolute_test_filename;

      tests->push_back(*test.release());

      test.reset(new Test);

      ++count;

    } else if (Match(&line, "name:")) {

      SetField("name", &test->name, line);

    } else if (Match(&line, "command:")) {

      SetField("command", &test->command, line);

    } else if (Match(&line, "before:")) {

      test->befores.push_back(line);

    } else if (Match(&line, "after:")) {

      test->afters.push_back(line);

    } else if (Match(&line, "expected-stdout:")) {

      // Collect tab-indented lines.

      std::string text;

      while (g_line < lines.size() && !lines[g_line].empty() && lines[g_line][0] == '\t') {

        text += lines[g_line++].substr(1) + "\n";

      }

      SetField("expected stdout", &test->expected_stdout, text);

    } else {

      Die(0, "%s:%zu: syntax error: \"%s\"", g_file, g_line, line.c_str());

    }

  }

  if (count == 0) Die(0, "no tests found in '%s'", g_file);

}

static const char* Plural(size_t n) {

  return (n == 1) ? "" : "s";

}

static std::string ExitStatusToString(int status) {

  if (WIFSIGNALED(status)) {

    return android::base::StringPrintf("was killed by signal %d (%s)", WTERMSIG(status),

                                       strsignal(WTERMSIG(status)));

  }

  if (WIFSTOPPED(status)) {

    return android::base::StringPrintf("was stopped by signal %d (%s)", WSTOPSIG(status),

                                       strsignal(WSTOPSIG(status)));

  }

  return android::base::StringPrintf("exited with status %d", WEXITSTATUS(status));

}

static bool RunCommands(const char* what, const std::vector<std::string>& commands) {

  bool result = true;

  for (auto& command : commands) {

    V("running %s \"%s\"", what, command.c_str());

    int exit_status = system(command.c_str());

    if (exit_status != 0) {

      result = false;

      fprintf(stderr, "Command (%s) \"%s\" %s\n", what, command.c_str(),

              ExitStatusToString(exit_status).c_str());

    }

  }

  return result;

}

static bool CheckOutput(const char* what, std::string actual_output,

                        const std::string& expected_output, const std::string& FILES) {

  // Rewrite the output to reverse any expansion of $FILES.

  actual_output = android::base::StringReplace(actual_output, FILES, "$FILES", true);

  bool result = (actual_output == expected_output);

  if (!result) {

    fprintf(stderr, "Incorrect %s.\nExpected:\n%s\nActual:\n%s\n", what, expected_output.c_str(),

            actual_output.c_str());

  }

  return result;

}

static int RunTests(const std::vector<Test>& tests) {

  std::vector<std::string> failures;

  Print(kGreen, "[==========]", " Running %zu tests.", tests.size());

  android::base::Timer total_timer;

  for (const auto& test : tests) {

    bool failed = false;

    Print(kGreen, "[ RUN      ]", " %s", test.name.c_str());

    android::base::Timer test_timer;

    // Set $FILES for this test.

    std::string FILES = android::base::Dirname(test.test_filename) + "/files";

    V("setenv(\"FILES\", \"%s\")", FILES.c_str());

    setenv("FILES", FILES.c_str(), 1);

    // Make a safe space to run the test.

    TemporaryDir td;

    V("chdir(\"%s\")", td.path);

    if (chdir(td.path)) Die(errno, "chdir(\"%s\")", td.path);

    // Perform any setup specified for this test.

    if (!RunCommands("before", test.befores)) failed = true;

    if (!failed) {

      V("running command \"%s\"", test.command.c_str());

      CapturedStdout test_stdout;

      CapturedStderr test_stderr;

      int exit_status = system(test.command.c_str());

      test_stdout.Stop();

      test_stderr.Stop();

      V("exit status %d", exit_status);

      if (exit_status != test.exit_status) {

        failed = true;

        fprintf(stderr, "Incorrect exit status: expected %d but %s\n", test.exit_status,

                ExitStatusToString(exit_status).c_str());

      }

      if (!CheckOutput("stdout", test_stdout.str(), test.expected_stdout, FILES)) failed = true;

      if (!CheckOutput("stderr", test_stderr.str(), test.expected_stderr, FILES)) failed = true;

      if (!RunCommands("after", test.afters)) failed = true;

    }

    std::stringstream duration;

    duration << test_timer;

    if (failed) {

      failures.push_back(test.name);

      Print(kRed, "[  FAILED  ]", " %s (%s)", test.name.c_str(), duration.str().c_str());

    } else {

      Print(kGreen, "[       OK ]", " %s (%s)", test.name.c_str(), duration.str().c_str());

    }

  }

  // Summarize the whole run and explicitly list all the failures.

  std::stringstream duration;

  duration << total_timer;

  Print(kGreen, "[==========]", " %zu tests ran. (%s total)", tests.size(), duration.str().c_str());

  size_t fail_count = failures.size();

  size_t pass_count = tests.size() - fail_count;

  Print(kGreen, "[  PASSED  ]", " %zu test%s.", pass_count, Plural(pass_count));

  if (!failures.empty()) {

    Print(kRed, "[  FAILED  ]", " %zu test%s.", fail_count, Plural(fail_count));

    for (auto& failure : failures) {

      Print(kRed, "[  FAILED  ]", " %s", failure.c_str());

    }

  }

  return (fail_count == 0) ? 0 : 1;

}

static void ShowHelp(bool full) {

  fprintf(full ? stdout : stderr, "usage: %s [-v] FILE...\n", g_progname);

  if (!full) exit(EXIT_FAILURE);

  printf(

      "\n"

      "Run tests.\n"

      "\n"

      "-v\tVerbose (show workings)\n");

  exit(EXIT_SUCCESS);

}

int main(int argc, char* argv[]) {

  g_progname = basename(argv[0]);

  static const struct option opts[] = {

      {"help", no_argument, 0, 'h'},

      {"verbose", no_argument, 0, 'v'},

      {},

  };

  int opt;

  while ((opt = getopt_long(argc, argv, "hv", opts, nullptr)) != -1) {

    switch (opt) {

      case 'h':

        ShowHelp(true);

        break;

      case 'v':

        g_verbose = true;

        break;

      default:

        ShowHelp(false);

        break;

    }

  }

  argv += optind;

  if (!*argv) Die(0, "no test files provided");

  std::vector<Test> tests;

  for (; *argv; ++argv) CollectTests(&tests, *argv);

  return RunTests(tests);

}

    host_supported: true,

    corpus: ["testdata/*"],

}

sh_test {

    name: "ziptool-tests",

    src: "run-ziptool-tests-on-android.sh",

    filename: "run-ziptool-tests-on-android.sh",

    test_suites: ["general-tests"],

    host_supported: true,

    device_supported: false,

    test_config: "ziptool-tests.xml",

    data: ["cli-tests/**/*"],

    target_required: ["cli-test", "ziptool"],

}