Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6aa551ec authored by Peter Qiu's avatar Peter Qiu Committed by ChromeOS Commit Bot
Browse files

crash-reporter: add support for device coredump 

Copy the device coredump file to /var/spool/crash directory when
udev event "devcoredump" is detected. Also include the last 50 lines
of /var/log/messages and /var/log/net.log in the coredump file for
debugging purpose. Only perform the collection if feedback is allowed
(metrics are enabled) or the device is running a developer image.

By default, crash-reporter will not upload device coredumps to the crash
server. The user can enable device coredumps upload via a to-be-added
crosh command, which should only be done at the direction of CrOS engineers.

BUG=chromium:464872
TEST=USE="asan clang" FEATURES=test emerge-$BOARD crash-reporter
Manual Test:
1. Trigger wifi firmware error on a Clapper running developer image by
   running following command:
   "echo 1 > /sys/kernel/debug/iwlwifi/0000\:01\:00.0/iwlmvm/fw_restart"
2. Verify there is a device coredump file in "/var/spool/crash/"
   "devcoredump_iwlwifi*.devcore", as well as the additional log file
   ".log" and the meta file ".meta".

Change-Id: Ic4cf67d4b5715a6f422505f409276d1261b7d51f
Reviewed-on: https://chromium-review.googlesource.com/257091


Reviewed-by: default avatarZeping Qiu <zqiu@chromium.org>
Commit-Queue: Zeping Qiu <zqiu@chromium.org>
Tested-by: default avatarZeping Qiu <zqiu@chromium.org>
Reviewed-by: default avatarMike Frysinger <vapier@chromium.org>
parent 74580891

crash_reporter/99-crash-reporter.rules

+1 −0
Original line number Diff line number Diff line
@@ -3,3 +3,4 @@ ACTION=="change", SUBSYSTEM=="drm", KERNEL=="card0", ENV{ERROR}=="1", RUN+="/sbi 
ACTION=="change", SUBSYSTEM=="i2c", DRIVER=="cyapa", ENV{ERROR}=="1", RUN+="/sbin/crash_reporter --udev=SUBSYSTEM=i2c-cyapa:ACTION=change"

# For detecting Atmel trackpad/touchscreen issue. Passing into crash_reporter SUBSYSTEM=i2c-atmel_mxt_ts since crash_reporter does not handle DRIVER string.

ACTION=="change", SUBSYSTEM=="i2c", DRIVER=="atmel_mxt_ts", ENV{ERROR}=="1", RUN+="/sbin/crash_reporter --udev=SUBSYSTEM=i2c-atmel_mxt_ts:ACTION=change"

ACTION=="add", SUBSYSTEM=="devcoredump", RUN+="/sbin/crash_reporter --udev=SUBSYSTEM=devcoredump:ACTION=add:KERNEL_NUMBER=%n"

crash_reporter/crash_reporter_logs.conf

+9 −0
Original line number Diff line number Diff line
@@ -74,6 +74,15 @@ crash_reporter-udev-collection---TouchNoise=cat /var/log/touch_noise.log 
# Periodically collect touch event log for debugging (crosbug.com/p/17244)

crash_reporter-udev-collection---TouchEvent=cat /var/log/touch_event.log



# Collect the last 50 lines of /var/log/messages and /var/log/net.log for

# intel wifi driver (iwlwifi) for debugging purpose.

crash_reporter-udev-collection-devcoredump-iwlwifi=\

  echo "===/var/log/messages==="; \

  tail -n 50 /var/log/messages; \

  echo "===/var/log/net.log==="; \

  tail -n 50 /var/log/net.log; \

  echo EOF



# Dump the last 50 lines of the last two powerd log files -- if the job has

# already restarted, we want to see the end of the previous instance's logs.

powerd=\

crash_reporter/crash_sender

+18 −1
Original line number Diff line number Diff line
@@ -69,6 +69,10 @@ RUN_FILE="/var/run/crash_sender.pid" 
# Maximum time to sleep between sends.

SECONDS_SEND_SPREAD=${SECONDS_SEND_SPREAD:-600}



# Set this to 1 to allow uploading of device coredumps.

DEVCOREDUMP_UPLOAD_FLAG_FILE=\

"/var/lib/crash_reporter/device_coredump_upload_allowed"



# The syslog tag for all logging we emit.

TAG="$(basename $0)[$$]"
@@ -156,6 +160,12 @@ is_developer_mode() { 
  crossystem "devsw_boot?1"  # exit status will be accurate

}



# Return 0 if the uploading of device coredumps is allowed.

is_device_coredump_upload_allowed() {

  [ -f "${DEVCOREDUMP_UPLOAD_FLAG_FILE}" ] && return 0

  return 1

}



# Generate a uniform random number in 0..max-1.

generate_uniform_random() {

  local max=$1
@@ -508,7 +518,8 @@ send_crashes() { 
    local kind=$(get_kind "${meta_path}")

    if [ "${kind}" != "minidump" ] && \

       [ "${kind}" != "kcrash" ] && \

       [ "${kind}" != "log" ]; then

       [ "${kind}" != "log" ] &&

       [ "${kind}" != "devcore" ]; then

      lecho "Unknown report kind ${kind}.  Removing report."

      remove_report "${meta_path}"

      continue
@@ -527,6 +538,12 @@ send_crashes() { 
      continue

    fi



    # Ignore device coredump if device coredump uploading is not allowed.

    if [ "${kind}" = "devcore" ] && ! is_device_coredump_upload_allowed; then

      lecho "Ignoring device coredump. Device coredump upload not allowed."

      continue

    fi



    if ! is_mock && ! is_official_image; then

      lecho "Not an official OS version.  Removing crash."

      remove_report "${meta_path}"

crash_reporter/udev_collector.cc

+157 −19
Original line number Diff line number Diff line
@@ -8,26 +8,41 @@ 
#include <utility>

#include <vector>



#include <base/files/file_enumerator.h>

#include <base/files/file_util.h>

#include <base/logging.h>

#include <base/strings/string_number_conversions.h>

#include <base/strings/string_split.h>

#include <base/strings/string_util.h>

#include <base/strings/stringprintf.h>

#include <chromeos/process.h>



static const char kCollectUdevSignature[] = "crash_reporter-udev-collection";

static const char kGzipPath[] = "/bin/gzip";

static const char kUdevExecName[] = "udev";

static const char kUdevSignatureKey[] = "sig";



using base::FilePath;



UdevCollector::UdevCollector() {}

namespace {



const char kCollectUdevSignature[] = "crash_reporter-udev-collection";

const char kGzipPath[] = "/bin/gzip";

const char kUdevExecName[] = "udev";

const char kUdevSignatureKey[] = "sig";

const char kUdevSubsystemDevCoredump[] = "devcoredump";

const char kDefaultDevCoredumpDirectory[] = "/sys/class/devcoredump";

const char kDevCoredumpFilePrefixFormat[] = "devcoredump_%s";



}  // namespace



UdevCollector::UdevCollector()

    : dev_coredump_directory_(kDefaultDevCoredumpDirectory) {}



UdevCollector::~UdevCollector() {}



bool UdevCollector::HandleCrash(const std::string &udev_event) {

  if (!is_feedback_allowed_function_()) {

    LOG(ERROR) << "No consent given to collect crash info.";

  if (IsDeveloperImage()) {

    LOG(INFO) << "developer image - collect udev crash info.";

  } else if (is_feedback_allowed_function_()) {

    LOG(INFO) << "Consent given - collect udev crash info.";

  } else {

    LOG(INFO) << "Ignoring - Non-developer image and no consent given.";

    return false;

  }
@@ -43,24 +58,43 @@ bool UdevCollector::HandleCrash(const std::string &udev_event) { 
    udev_event_map[iter->first] = iter->second;

  }



  // Make sure the crash directory exists, or create it if it doesn't.

  FilePath crash_directory;

  if (!GetCreatedCrashDirectoryByEuid(0, &crash_directory, nullptr)) {

    LOG(ERROR) << "Could not get crash directory.";

    return false;

  }



  if (udev_event_map["SUBSYSTEM"] == kUdevSubsystemDevCoredump) {

    int instance_number;

    if (!base::StringToInt(udev_event_map["KERNEL_NUMBER"], &instance_number)) {

      LOG(ERROR) << "Invalid kernel number: "

                 << udev_event_map["KERNEL_NUMBER"];

      return false;

    }

    return ProcessDevCoredump(crash_directory, instance_number);

  }



  return ProcessUdevCrashLogs(crash_directory,

                              udev_event_map["ACTION"],

                              udev_event_map["KERNEL"],

                              udev_event_map["SUBSYSTEM"]);

}



bool UdevCollector::ProcessUdevCrashLogs(const FilePath& crash_directory,

                                         const std::string& action,

                                         const std::string& kernel,

                                         const std::string& subsystem) {

  // Construct the basename string for crash_reporter_logs.conf:

  //   "crash_reporter-udev-collection-[action]-[name]-[subsystem]"

  // If a udev field is not provided, "" is used in its place, e.g.:

  //   "crash_reporter-udev-collection-[action]--[subsystem]"

  // Hence, "" is used as a wildcard name string.

  // TODO(sque, crosbug.com/32238): Implement wildcard checking.

  std::string basename = udev_event_map["ACTION"] + "-" +

                         udev_event_map["KERNEL"] + "-" +

                         udev_event_map["SUBSYSTEM"];

  std::string basename = action + "-" + kernel + "-" + subsystem;

  std::string udev_log_name = std::string(kCollectUdevSignature) + '-' +

                              basename;



  // Make sure the crash directory exists, or create it if it doesn't.

  FilePath crash_directory;

  if (!GetCreatedCrashDirectoryByEuid(0, &crash_directory, nullptr)) {

    LOG(ERROR) << "Could not get crash directory.";

    return false;

  }

  // Create the destination path.

  std::string log_file_name =

      FormatDumpBasename(basename, time(nullptr), 0);
@@ -85,10 +119,114 @@ bool UdevCollector::HandleCrash(const std::string &udev_event) { 
  else

    crash_path = crash_path_zipped;



  std::string exec_name = std::string(kUdevExecName) + "-" +

      udev_event_map["SUBSYSTEM"];

  std::string exec_name = std::string(kUdevExecName) + "-" + subsystem;

  AddCrashMetaData(kUdevSignatureKey, udev_log_name);

  WriteCrashMetaData(GetCrashPath(crash_directory, log_file_name, "meta"),

                     exec_name, crash_path.value());

  return true;

}



bool UdevCollector::ProcessDevCoredump(const FilePath& crash_directory,

                                       int instance_number) {

  FilePath coredump_path =

      FilePath(base::StringPrintf("%s/devcd%d/data",

                                  dev_coredump_directory_.c_str(),

                                  instance_number));

  if (!base::PathExists(coredump_path)) {

    LOG(ERROR) << "Device coredump file " << coredump_path.value()

               << " does not exist";

    return false;

  }



  // Add coredump file to the crash directory.

  if (!AppendDevCoredump(crash_directory, coredump_path, instance_number)) {

    ClearDevCoredump(coredump_path);

    return false;

  }



  // Clear the coredump data to allow generation of future device coredumps

  // without having to wait for the 5-minutes timeout.

  return ClearDevCoredump(coredump_path);

}



bool UdevCollector::AppendDevCoredump(const FilePath& crash_directory,

                                      const FilePath& coredump_path,

                                      int instance_number) {

  // Retrieve the driver name of the failing device.

  std::string driver_name = GetFailingDeviceDriverName(instance_number);

  if (driver_name.empty()) {

    LOG(ERROR) << "Failed to obtain driver name for instance: "

               << instance_number;

    return false;

  }



  std::string coredump_prefix =

      base::StringPrintf(kDevCoredumpFilePrefixFormat, driver_name.c_str());



  std::string dump_basename = FormatDumpBasename(coredump_prefix,

                                                 time(nullptr),

                                                 instance_number);

  FilePath core_path = GetCrashPath(crash_directory, dump_basename, "devcore");

  FilePath log_path = GetCrashPath(crash_directory, dump_basename, "log");

  FilePath meta_path = GetCrashPath(crash_directory, dump_basename, "meta");



  // Collect coredump data.

  if (!base::CopyFile(coredump_path, core_path)) {

    LOG(ERROR) << "Failed to copy device coredumpm file from "

               << coredump_path.value() << " to " << core_path.value();

    return false;

  }



  // Collect additional logs if one is specified in the config file.

  std::string udev_log_name = std::string(kCollectUdevSignature) + '-' +

      kUdevSubsystemDevCoredump + '-' + driver_name;

  bool result = GetLogContents(log_config_path_, udev_log_name, log_path);

  if (result) {

    AddCrashMetaUploadFile("logs", log_path.value());

  }



  WriteCrashMetaData(meta_path, coredump_prefix, core_path.value());



  return true;

}



bool UdevCollector::ClearDevCoredump(const FilePath& coredump_path) {

  if (!base::WriteFile(coredump_path, "0", 1)) {

    LOG(ERROR) << "Failed to delete the coredump data file "

               << coredump_path.value();

    return false;

  }

  return true;

}



std::string UdevCollector::GetFailingDeviceDriverName(int instance_number) {

  FilePath failing_uevent_path =

      FilePath(base::StringPrintf("%s/devcd%d/failing_device/uevent",

                                  dev_coredump_directory_.c_str(),

                                  instance_number));

  if (!base::PathExists(failing_uevent_path)) {

    LOG(ERROR) << "Failing uevent path " << failing_uevent_path.value()

               << " does not exist";

    return "";

  }



  std::string uevent_content;

  if (!base::ReadFileToString(failing_uevent_path, &uevent_content)) {

    LOG(ERROR) << "Failed to read uevent file " << failing_uevent_path.value();

    return "";

  }



  // Parse uevent file contents as key-value pairs.

  std::vector<std::pair<std::string, std::string>> uevent_keyval;

  base::SplitStringIntoKeyValuePairs(uevent_content, '=', '\n', &uevent_keyval);

  std::vector<std::pair<std::string, std::string>>::const_iterator iter;

  for (iter = uevent_keyval.begin();

       iter != uevent_keyval.end();

       ++iter) {

    if (iter->first == "DRIVER") {

      return iter->second;

    }

  }



  return "";

}

crash_reporter/udev_collector.h

+24 −0
Original line number Diff line number Diff line
@@ -26,9 +26,33 @@ class UdevCollector : public CrashCollector { 
  // could be omitted, in which case it would be treated as a wildcard (*).

  bool HandleCrash(const std::string& udev_event);



 protected:

  std::string dev_coredump_directory_;



 private:

  friend class UdevCollectorTest;



  // Process udev crash logs, collecting log files according to the config

  // file (crash_reporter_logs.conf).

  bool ProcessUdevCrashLogs(const base::FilePath& crash_directory,

                            const std::string& action,

                            const std::string& kernel,

                            const std::string& subsystem);

  // Process device coredump, collecting device coredump file.

  // |instance_number| is the kernel number of the virtual device for the device

  // coredump instance.

  bool ProcessDevCoredump(const base::FilePath& crash_directory,

                          int instance_number);

  // Copy device coredump file to crash directory, and perform necessary

  // coredump file management.

  bool AppendDevCoredump(const base::FilePath& crash_directory,

                         const base::FilePath& coredump_path,

                         int instance_number);

  // Clear the device coredump file by performing a dummy write to it.

  bool ClearDevCoredump(const base::FilePath& coredump_path);

  // Return the driver name of the device that generates the coredump.

  std::string GetFailingDeviceDriverName(int instance_number);



  // Mutator for unit testing.

  void set_log_config_path(const std::string& path) {

    log_config_path_ = base::FilePath(path);