Merge "Faster bugreports (1/n)" (8fa63e04) · Commits · e / os / android_frameworks_native-old

cmds/dumpstate/Android.bp

+3 −0

Original line number	Diff line number	Diff line
		@@ -105,6 +105,7 @@ cc_binary {
		name: "dumpstate",
		defaults: ["dumpstate_defaults"],
		srcs: [
		"DumpPool.cpp",
		"dumpstate.cpp",
		"main.cpp",
		],
		@@ -132,6 +133,7 @@ cc_test {
		name: "dumpstate_test",
		defaults: ["dumpstate_defaults"],
		srcs: [
		"DumpPool.cpp",
		"dumpstate.cpp",
		"tests/dumpstate_test.cpp",
		],
		@@ -148,6 +150,7 @@ cc_test {
		name: "dumpstate_smoke_test",
		defaults: ["dumpstate_defaults"],
		srcs: [
		"DumpPool.cpp",
		"dumpstate.cpp",
		"tests/dumpstate_smoke_test.cpp",
		],

cmds/dumpstate/DumpPool.cpp

0 → 100644

+171 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "dumpstate"

		#include "DumpPool.h"

		#include <array>
		#include <thread>

		#include <log/log.h>

		#include "dumpstate.h"
		#include "DumpstateInternal.h"
		#include "DumpstateUtil.h"

		namespace android {
		namespace os {
		namespace dumpstate {

		const std::string DumpPool::PREFIX_TMPFILE_NAME = "dump-tmp.";

		DumpPool::DumpPool(const std::string& tmp_root) : tmp_root_(tmp_root), shutdown_(false) {
		assert(!tmp_root.empty());
		deleteTempFiles(tmp_root_);
		}

		DumpPool::~DumpPool() {
		shutdown();
		}

		void DumpPool::start(int thread_counts) {
		assert(thread_counts > 0);
		assert(threads_.empty());
		if (thread_counts > MAX_THREAD_COUNT) {
		thread_counts = MAX_THREAD_COUNT;
		}
		MYLOGI("Start thread pool:%d", thread_counts);
		shutdown_ = false;
		for (int i = 0; i < thread_counts; i++) {
		threads_.emplace_back(std::thread([=]() {
		setThreadName(pthread_self(), i + 1);
		loop();
		}));
		}
		}

		void DumpPool::shutdown() {
		std::unique_lock lock(lock_);
		if (shutdown_ \|\| threads_.empty()) {
		return;
		}
		while (!tasks_.empty()) tasks_.pop();
		futures_map_.clear();

		shutdown_ = true;
		condition_variable_.notify_all();
		lock.unlock();

		for (auto& thread : threads_) {
		thread.join();
		}
		threads_.clear();
		deleteTempFiles(tmp_root_);
		MYLOGI("shutdown thread pool");
		}

		void DumpPool::waitForTask(const std::string& task_name, const std::string& title,
		int out_fd) {
		DurationReporter duration_reporter("Wait for " + task_name, true);
		auto iterator = futures_map_.find(task_name);
		if (iterator == futures_map_.end()) {
		MYLOGW("Task %s does not exist", task_name.c_str());
		return;
		}
		Future future = iterator->second;
		futures_map_.erase(iterator);

		std::string result = future.get();
		if (result.empty()) {
		return;
		}
		DumpFileToFd(out_fd, title, result);
		if (unlink(result.c_str())) {
		MYLOGE("Failed to unlink (%s): %s\n", result.c_str(), strerror(errno));
		}
		}

		std::unique_ptr<DumpPool::TmpFile> DumpPool::createTempFile() {
		auto tmp_file_ptr = std::make_unique<TmpFile>();
		std::string file_name_format = "%s/" + PREFIX_TMPFILE_NAME + "XXXXXX";
		snprintf(tmp_file_ptr->path, sizeof(tmp_file_ptr->path), file_name_format.c_str(),
		tmp_root_.c_str());
		tmp_file_ptr->fd.reset(TEMP_FAILURE_RETRY(
		mkostemp(tmp_file_ptr->path, O_CLOEXEC)));
		if (tmp_file_ptr->fd.get() == -1) {
		MYLOGE("open(%s, %s)\n", tmp_file_ptr->path, strerror(errno));
		tmp_file_ptr = nullptr;
		return tmp_file_ptr;
		}
		return tmp_file_ptr;
		}

		void DumpPool::deleteTempFiles(const std::string& folder) {
		std::unique_ptr<DIR, decltype(&closedir)> dir_ptr(opendir(folder.c_str()),
		&closedir);
		if (!dir_ptr) {
		MYLOGE("Failed to opendir (%s): %s\n", folder.c_str(), strerror(errno));
		return;
		}
		int dir_fd = dirfd(dir_ptr.get());
		if (dir_fd < 0) {
		MYLOGE("Failed to get fd of dir (%s): %s\n", folder.c_str(),
		strerror(errno));
		return;
		}

		struct dirent* de;
		while ((de = readdir(dir_ptr.get()))) {
		if (de->d_type != DT_REG) {
		continue;
		}
		std::string file_name(de->d_name);
		if (file_name.find(PREFIX_TMPFILE_NAME) != 0) {
		continue;
		}
		if (unlinkat(dir_fd, file_name.c_str(), 0)) {
		MYLOGE("Failed to unlink (%s): %s\n", file_name.c_str(),
		strerror(errno));
		}
		}
		}

		void DumpPool::setThreadName(const pthread_t thread, int id) {
		std::array<char, 15> name;
		snprintf(name.data(), name.size(), "dumpstate_%d", id);
		pthread_setname_np(thread, name.data());
		}

		void DumpPool::loop() {
		std::unique_lock lock(lock_);
		while (!shutdown_) {
		if (tasks_.empty()) {
		condition_variable_.wait(lock);
		continue;
		} else {
		std::packaged_task<std::string()> task = std::move(tasks_.front());
		tasks_.pop();
		lock.unlock();
		std::invoke(task);
		lock.lock();
		}
		}
		}

		} // namespace dumpstate
		} // namespace os
		} // namespace android

cmds/dumpstate/DumpPool.h

0 → 100644

+161 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef FRAMEWORK_NATIVE_CMD_DUMPPOOL_H_
		#define FRAMEWORK_NATIVE_CMD_DUMPPOOL_H_

		#include <future>
		#include <map>
		#include <queue>
		#include <string>

		#include <android-base/file.h>
		#include <android-base/macros.h>

		namespace android {
		namespace os {
		namespace dumpstate {

		/*
		* A thread pool with the fixed number of threads to execute multiple dump tasks
		* simultaneously for the dumpstate. The dump task is a callable function
		* included a file descriptor as a parameter, and the task could dump results to
		* that fd. For example:
		*
		* void DumpXXXX(int out_fd) {
		* dprintf(out_fd, "Dump result to out_fd ...");
		* }
		* ...
		* DumpPool pool(tmp_root);
		* pool.enqueueTask("TaskName", &DumpXXXX, std::placeholders::_1);
		* ...
		* pool.waitForTask("TaskName");
		*
		* DumpXXXX is a callable function included a out_fd parameter. Using the
		* enqueueTask method in DumpPool to enqueue the task to the pool. The
		* std::placeholders::_1 is placeholder for DumpPool to pass a fd argument.
		*/
		class DumpPool {
		public:
		/*
		* Creates a thread pool.
		*
		* \|tmp_root\| A path to a temporary folder for threads to create temporary
		* files.
		*/
		explicit DumpPool(const std::string& tmp_root);
		~DumpPool();

		/*
		* Starts the threads in the pool.
		*
		* \|thread_counts\| the number of threads to start.
		*/
		void start(int thread_counts = MAX_THREAD_COUNT);

		/*
		* Requests to shutdown the pool and waits until all threads exit the loop.
		*/
		void shutdown();

		/*
		* Adds a task with a task name into the queue of the thread pool.
		*
		* \|task_name\| The name of the task.
		* \|f\| Callable function to execute the task. This function must
		* include a parameter of file descriptor to output dump result.
		* \|args\| A list of arguments.
		*/
		template<class F, class... Args> void enqueueTask(const std::string& task_name,
		F&& f, Args&&... args) {
		auto func = std::bind(std::forward<F>(f), std::forward<Args>(args)...);
		futures_map_[task_name] = post(func);
		if (threads_.empty()) {
		start();
		}
		}

		/*
		* Waits until the task is finished. Dumps the task results to the STDOUT_FILENO.
		*/
		void waitForTask(const std::string& task_name) {
		waitForTask(task_name, "", STDOUT_FILENO);
		}

		/*
		* Waits until the task is finished. Dumps the task results to the specified
		* out_fd.
		*
		* \|task_name\| The name of the task.
		* \|title\| Dump title string to the out_fd, an empty string for nothing.
		* \|out_fd\| The target file to dump the result from the task.
		*/
		void waitForTask(const std::string& task_name, const std::string& title, int out_fd);

		static const std::string PREFIX_TMPFILE_NAME;

		private:
		using Task = std::packaged_task<std::string()>;
		using Future = std::shared_future<std::string>;

		template<class T> Future post(T dump_func) {
		Task packaged_task([=]() {
		std::unique_ptr<TmpFile> tmp_file_ptr = createTempFile();
		if (!tmp_file_ptr) {
		return std::string("");
		}
		std::invoke(dump_func, tmp_file_ptr->fd.get());
		fsync(tmp_file_ptr->fd.get());
		return std::string(tmp_file_ptr->path);
		});
		std::unique_lock lock(lock_);
		auto future = packaged_task.get_future().share();
		tasks_.push(std::move(packaged_task));
		condition_variable_.notify_one();
		return future;
		}

		typedef struct {
		android::base::unique_fd fd;
		char path[1024];
		} TmpFile;

		std::unique_ptr<TmpFile> createTempFile();
		void deleteTempFiles(const std::string& folder);
		void setThreadName(const pthread_t thread, int id);
		void loop();

		private:
		static const int MAX_THREAD_COUNT = 4;

		/* A path to a temporary folder for threads to create temporary files. */
		std::string tmp_root_;
		bool shutdown_;
		std::mutex lock_; // A lock for the tasks_.
		std::condition_variable condition_variable_;

		std::vector<std::thread> threads_;
		std::queue<Task> tasks_;
		std::map<std::string, Future> futures_map_;

		DISALLOW_COPY_AND_ASSIGN(DumpPool);
		};

		} // namespace dumpstate
		} // namespace os
		} // namespace android

		#endif //FRAMEWORK_NATIVE_CMD_DUMPPOOL_H_

cmds/dumpstate/tests/dumpstate_test.cpp

+76 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@
		#include "DumpstateService.h"
		#include "android/os/BnDumpstate.h"
		#include "dumpstate.h"
		#include "DumpPool.h"

		#include <gmock/gmock.h>
		#include <gtest/gtest.h>
		@@ -46,6 +47,7 @@ namespace dumpstate {

		using ::android::hardware::dumpstate::V1_1::DumpstateMode;
		using ::testing::EndsWith;
		using ::testing::Eq;
		using ::testing::HasSubstr;
		using ::testing::IsEmpty;
		using ::testing::IsNull;
		@@ -1618,6 +1620,80 @@ TEST_F(DumpstateUtilTest, DumpFileOnDryRun) {
		EXPECT_THAT(out, EndsWith("skipped on dry run\n"));
		}

		class DumpPoolTest : public DumpstateBaseTest {
		public:
		void SetUp() {
		DumpstateBaseTest::SetUp();
		CreateOutputFile();
		}

		void CreateOutputFile() {
		out_path_ = kTestDataPath + "out.txt";
		out_fd_.reset(TEMP_FAILURE_RETRY(open(out_path_.c_str(),
		O_WRONLY \| O_CREAT \| O_TRUNC \| O_CLOEXEC \| O_NOFOLLOW,
		S_IRUSR \| S_IWUSR \| S_IRGRP \| S_IROTH)));
		ASSERT_GE(out_fd_.get(), 0) << "could not create FD for path "
		<< out_path_;
		}

		int getTempFileCounts(const std::string& folder) {
		int count = 0;
		std::unique_ptr<DIR, decltype(&closedir)> dir_ptr(opendir(folder.c_str()),
		&closedir);
		if (!dir_ptr) {
		return -1;
		}
		int dir_fd = dirfd(dir_ptr.get());
		if (dir_fd < 0) {
		return -1;
		}

		struct dirent* de;
		while ((de = readdir(dir_ptr.get()))) {
		if (de->d_type != DT_REG) {
		continue;
		}
		std::string file_name(de->d_name);
		if (file_name.find(DumpPool::PREFIX_TMPFILE_NAME) != 0) {
		continue;
		}
		count++;
		}
		return count;
		}

		android::base::unique_fd out_fd_;
		std::string out_path_;
		};

		TEST_F(DumpPoolTest, EnqueueTask) {
		DumpPool pool(kTestDataPath);
		auto dump_func_1 = [](int out_fd) {
		dprintf(out_fd, "A");
		};
		auto dump_func_2 = [](int out_fd) {
		dprintf(out_fd, "B");
		sleep(1);
		};
		auto dump_func_3 = [](int out_fd) {
		dprintf(out_fd, "C");
		};
		pool.enqueueTask(/* task_name = */"1", dump_func_1, std::placeholders::_1);
		pool.enqueueTask(/* task_name = */"2", dump_func_2, std::placeholders::_1);
		pool.enqueueTask(/* task_name = */"3", dump_func_3, std::placeholders::_1);

		pool.waitForTask("1", "", out_fd_.get());
		pool.waitForTask("2", "", out_fd_.get());
		pool.waitForTask("3", "", out_fd_.get());

		std::string result;
		ReadFileToString(out_path_, &result);
		EXPECT_THAT(result, StrEq("A\nB\nC\n"));
		EXPECT_THAT(getTempFileCounts(kTestDataPath), Eq(0));
		pool.shutdown();
		}


		} // namespace dumpstate
		} // namespace os
		} // namespace android