Add code coverage report support to Bluetooth (337fbdec) · Commits · e / os / android_packages_modules_Bluetooth

system/build/Android.bp

+35 −0

Original line number	Diff line number	Diff line
		@@ -63,3 +63,38 @@ fluoride_defaults {
		],
		cpp_std: "c++17",
		}

		// Enables code coverage for a set of source files. Must be combined with
		// "clang_coverage_bin" in order to work. See //test/gen_coverage.py for more information
		// on generating code coverage.
		cc_defaults {
		name: "clang_file_coverage",
		target: {
		linux: {
		enabled: true,
		clang_cflags: [
		"-fprofile-instr-generate",
		"-fcoverage-mapping",
		],
		},
		},
		}

		// Enabled code coverage on a binary. These flags allow libraries that were
		// compiled with "clang_file_coverage" to be properly linked together in
		// order to create a binary that will create a profraw file when ran. Note
		// these flags themselves don't enable code coverage for the source files
		// compiled in the binary. See //test/gen_coverage.py for more information
		// on generating code coverage.
		cc_defaults {
		name: "clang_coverage_bin",
		target: {
		linux: {
		enabled: true,
		ldflags: [
		"-fprofile-instr-generate",
		"-fcoverage-mapping",
		],
		},
		},
		}

system/profile/avrcp/Android.bp

+8 −2

Original line number	Diff line number	Diff line
		cc_library_static {
		name: "avrcp-target-service",
		defaults: ["fluoride_defaults"],
		defaults: [
		"fluoride_defaults",
		"clang_file_coverage"
		],
		host_supported: true,
		include_dirs: [
		"packages/modules/Bluetooth/system",
		@@ -26,7 +29,10 @@ cc_library_static {
		cc_test {
		name: "net_test_avrcp",
		test_suites: ["general-tests"],
		defaults: ["fluoride_defaults"],
		defaults: [
		"fluoride_defaults",
		"clang_coverage_bin",
		],
		host_supported: true,
		include_dirs: [
		"packages/modules/Bluetooth/system",

system/profile/sdp/Android.bp

+8 −2

Original line number	Diff line number	Diff line
		cc_library_static {
		name: "sdp_service",
		defaults: ["fluoride_defaults"],
		defaults: [
		"fluoride_defaults",
		"clang_file_coverage"
		],
		host_supported: true,
		include_dirs: [
		"packages/modules/Bluetooth/system/",
		@@ -17,7 +20,10 @@ cc_library_static {
		cc_test {
		name: "bluetooth_test_sdp",
		test_suites: ["general-tests"],
		defaults: ["fluoride_defaults"],
		defaults: [
		"fluoride_defaults",
		"clang_coverage_bin",
		],
		host_supported: true,
		include_dirs: [
		"packages/modules/Bluetooth/system/",

system/test/gen_coverage.py

0 → 100755

+396 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		#
		# Copyright 2018, The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import argparse
		import datetime
		import logging
		import json
		import os
		import shutil
		import subprocess
		import sys
		import webbrowser

		from run_host_unit_tests import *

		"""
		This script is used to generate code coverage results host supported libraries.
		The script by default will generate an html report that summarizes the coverage
		results of the specified tests. The results can also be browsed to provide a
		report of which lines have been traveled upon execution of the binary.

		NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as
		having been executed 0 times, thus reducing overall coverage.

		The steps in order to add coverage support to a new library and its
		corrisponding host test are as follows.

		1. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the
		source library(s) you want statistics for.
		NOTE: Forgoing this default will cause no coverage data to be generated for
		the source files in the library.

		2. Add "clang_coverage_bin" as a default to the host supported test binary that
		excercises the libraries that you covered in step 1.
		NOTE: Forgetting to add this will cause there to be NO coverage data
		generated when the binary is run.

		3. Add the host test binary name and the files/directories you want coverage
		statistics for to the COVERAGE_TESTS variable defined below. You may add
		individual filenames or a directory to be tested.
		NOTE: Avoid using a / at the beginning of a covered_files entry as this
		breaks how the coverage generator resolves filenames.

		TODO: Support generating XML data and printing results to standard out.
		"""

		COVERAGE_TESTS = [
		{
		"test_name": "net_test_avrcp",
		"covered_files": [
		"packages/modules/Bluetooth/system/profile/avrcp",
		],
		}, {
		"test_name": "bluetooth_test_sdp",
		"covered_files": [
		"packages/modules/Bluetooth/system/profile/sdp",
		],
		},
		]

		WORKING_DIR = '/tmp/coverage'
		SOONG_UI_BASH = 'build/soong/soong_ui.bash'
		LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r328903/bin'
		LLVM_MERGE = LLVM_DIR + '/llvm-profdata'
		LLVM_COV = LLVM_DIR + '/llvm-cov'

		def write_root_html_head(f):
		# Write the header part of the root html file. This was pulled from the
		# page source of one of the generated html files.
		f.write("<!doctype html><html><head>" \
		"<meta name='viewport' content='width=device-width,initial-scale=1'><met" \
		"a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \
		"s'></head><body><h2>Coverage Report</h2><h4>Created: " +
		str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) +
		"</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \
		"ge.html#interpreting-reports'>here</a> for information about interpreti" \
		"ng this report.</p><div class='centered'><table><tr><td class='column-e" \
		"ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \
		"</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \
		"='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \
		"gion Coverage</td></tr>"
		)


		def write_root_html_column(f, covered, count):
		percent = covered * 100.0 / count
		value = "%.2f%% (%d/%d) " % (percent, covered, count)
		color = 'column-entry-yellow'
		if percent == 100:
		color = 'column-entry-green'
		if percent < 80.0:
		color = 'column-entry-red'
		f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>")


		def write_root_html_rows(f, tests):
		totals = {
		"functions":{
		"covered": 0,
		"count": 0
		},
		"instantiations":{
		"covered": 0,
		"count": 0
		},
		"lines":{
		"covered": 0,
		"count": 0
		},
		"regions":{
		"covered": 0,
		"count": 0
		}
		}

		# Write the tests with their coverage summaries.
		for test in tests:
		test_name = test['test_name']
		covered_files = test['covered_files']
		json_results = generate_coverage_json(test)
		test_totals = json_results['data'][0]['totals']

		f.write("<tr class='light-row'><td><pre><a href=\'" +
		os.path.join(test_name, "index.html") + "\'>" + test_name +
		"</a></pre></td>")
		for field_name in ['functions', 'instantiations', 'lines', 'regions']:
		field = test_totals[field_name]
		totals[field_name]['covered'] += field['covered']
		totals[field_name]['count'] += field['count']
		write_root_html_column(f, field['covered'], field['count'])
		f.write("</tr>");

		#Write the totals row.
		f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>")
		for field_name in ['functions', 'instantiations', 'lines', 'regions']:
		field = totals[field_name]
		write_root_html_column(f, field['covered'], field['count'])
		f.write("</tr>");


		def write_root_html_tail(f):
		# Pulled from the generated html coverage report.
		f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \
		"/h5></body></html>")


		def generate_root_html(tests):
		# Copy the css file from one of the coverage reports.
		source_file = os.path.join(os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css")
		dest_file = os.path.join(WORKING_DIR, "style.css")
		shutil.copy2(source_file, dest_file)

		# Write the root index.html file that sumarizes all the tests.
		f = open(os.path.join(WORKING_DIR, "index.html"), "w")
		write_root_html_head(f)
		write_root_html_rows(f, tests)
		write_root_html_tail(f)


		def get_profraw_for_test(test_name):
		test_root = get_native_test_root_or_die()
		test_cmd = os.path.join(os.path.join(test_root, test_name), test_name)
		if not os.path.isfile(test_cmd):
		logging.error('The test ' + test_name + ' does not exist, please compile first')
		sys.exit(1)

		profraw_file_name = test_name + ".profraw"
		profraw_path = os.path.join(WORKING_DIR, os.path.join(test_name, profraw_file_name))
		llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\""

		test_cmd = llvm_env_var + " " + test_cmd
		logging.info('Generating profraw data for ' + test_name)
		logging.debug('cmd: ' + test_cmd)
		if subprocess.call(test_cmd, shell=True) != 0:
		logging.error('Test ' + test_name + ' failed. Please fix the test before generating coverage.')
		sys.exit(1)

		if not os.path.isfile(profraw_path):
		logging.error('Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?')
		sys.exit(1)

		return profraw_file_name


		def merge_profraw_data(test_name):
		cmd = []
		cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge "))

		test_working_dir = os.path.join(WORKING_DIR, test_name);
		cmd.append(os.path.join(test_working_dir, test_name + ".profraw"))
		profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

		cmd.append('-o ' + profdata_file)
		logging.info('Combining profraw files into profdata for ' + test_name)
		logging.debug('cmd: ' + " ".join(cmd))
		if subprocess.call(" ".join(cmd), shell=True) != 0:
		logging.error('Failed to merge profraw files for ' + test_name)
		sys.exit(1)


		def generate_coverage_html(test):
		COVERAGE_ROOT = '/proc/self/cwd'

		test_name = test['test_name']
		file_list = test['covered_files']

		test_working_dir = os.path.join(WORKING_DIR, test_name)
		test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

		cmd = [
		os.path.join(get_android_root_or_die(), LLVM_COV),
		"show",
		"-format=html",
		"-summary-only",
		"-show-line-counts-or-regions",
		"-show-instantiation-summary",
		"-instr-profile=" + test_profdata_file,
		"-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" +
		get_android_root_or_die() + "\"",
		"-output-dir=" + test_working_dir
		]

		# We have to have one object file not as an argument otherwise we can't specify source files.
		test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
		cmd.append(test_cmd)

		# Filter out the specific files we want coverage for
		for filename in file_list:
		cmd.append(os.path.join(get_android_root_or_die(), filename))

		logging.info('Generating coverage report for ' + test['test_name'])
		logging.debug('cmd: ' + " ".join(cmd))
		if subprocess.call(" ".join(cmd), shell=True) != 0:
		logging.error('Failed to generate coverage for ' + test['test_name'])
		sys.exit(1)


		def generate_coverage_json(test):
		COVERAGE_ROOT = '/proc/self/cwd'
		test_name = test['test_name']
		file_list = test['covered_files']

		test_working_dir = os.path.join(WORKING_DIR, test_name)
		test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

		cmd = [
		os.path.join(get_android_root_or_die(), LLVM_COV),
		"export",
		"-summary-only",
		"-show-region-summary",
		"-instr-profile=" + test_profdata_file,
		"-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"",
		]

		test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
		cmd.append(test_cmd)

		# Filter out the specific files we want coverage for
		for filename in file_list:
		cmd.append(os.path.join(get_android_root_or_die(), filename))

		logging.info('Generating coverage json for ' + test['test_name'])
		logging.debug('cmd: ' + " ".join(cmd))

		json_str = subprocess.check_output(" ".join(cmd), shell=True)
		return json.loads(json_str)


		def write_json_summary(test):
		test_name = test['test_name']
		test_working_dir = os.path.join(WORKING_DIR, test_name)
		test_json_summary_file = os.path.join(test_working_dir, test_name + '.json')
		logging.debug('Writing json summary file: ' + test_json_summary_file)
		json_file = open(test_json_summary_file, 'w')
		json.dump(generate_coverage_json(test), json_file)
		json_file.close()


		def list_tests():
		for test in COVERAGE_TESTS:
		print "Test Name: " + test['test_name']
		print "Covered Files: "
		for covered_file in test['covered_files']:
		print " " + covered_file
		print


		def main():
		parser = argparse.ArgumentParser(description='Generate code coverage for enabled tests.')
		parser.add_argument(
		'-l', '--list-tests',
		action='store_true',
		dest='list_tests',
		help='List all the available tests to be run as well as covered files.')
		parser.add_argument(
		'-a', '--all',
		action='store_true',
		help='Runs all available tests and prints their outputs. If no tests ' \
		'are specified via the -t option all tests will be run.')
		parser.add_argument(
		'-t', '--test',
		dest='tests',
		action='append',
		type=str,
		metavar='TESTNAME',
		default=[],
		help='Specifies a test to be run. Multiple tests can be specified by ' \
		'using this option multiple times. ' \
		'Example: \"gen_coverage.py -t test1 -t test2\"')
		parser.add_argument(
		'-o', '--output',
		type=str,
		metavar='DIRECTORY',
		default='/tmp/coverage',
		help='Specifies the directory to store all files. The directory will be ' \
		'created if it does not exist. Default is \"/tmp/coverage\"')
		parser.add_argument(
		'-s', '--skip-html',
		dest='skip_html',
		action='store_true',
		help='Skip opening up the results of the coverage report in a browser.')
		parser.add_argument(
		'-j', '--json-file',
		dest='json_file',
		action='store_true',
		help='Write out summary results to json file in test directory.')

		logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(levelname)s %(message)s')
		logging.addLevelName(logging.DEBUG, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG))
		logging.addLevelName(logging.INFO, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO))
		logging.addLevelName(logging.WARNING, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING))
		logging.addLevelName(logging.ERROR, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR))

		args = parser.parse_args()
		logging.debug("Args: " + str(args))

		# Set the working directory
		global WORKING_DIR
		WORKING_DIR = os.path.abspath(args.output)
		logging.debug("Working Dir: " + WORKING_DIR)

		# Print out the list of tests then exit
		if args.list_tests:
		list_tests()
		sys.exit(0)

		# Check to see if a test was specified and if so only generate coverage for
		# that test.
		if len(args.tests) == 0:
		args.all = True

		tests_to_run = []
		for test in COVERAGE_TESTS:
		if args.all or test['test_name'] in args.tests:
		tests_to_run.append(test)
		if test['test_name'] in args.tests:
		args.tests.remove(test['test_name'])

		# Error if a test was specified but doesn't exist.
		if len(args.tests) != 0:
		for test_name in args.tests:
		logging.error('\"' + test_name + '\" was not found in the list of available tests.')
		sys.exit(1)

		# Generate the info for the tests
		for test in tests_to_run:
		logging.info('Getting coverage for ' + test['test_name'])
		get_profraw_for_test(test['test_name'])
		merge_profraw_data(test['test_name'])
		if args.json_file:
		write_json_summary(test)
		generate_coverage_html(test)

		# Generate the root index.html page that sumarizes all of the coverage reports.
		generate_root_html(tests_to_run)

		# Open the results in a browser.
		if not args.skip_html:
		webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html'))


		if __name__ == '__main__':
		main()

system/test/run_host_unit_tests.py

+14 −0

Original line number	Diff line number	Diff line
		@@ -200,6 +200,20 @@ def main():
		print 'TEST FAILLED: ' + HOST_TESTS[index]
		sys.exit(0)
		print 'TEST PASSED ' + str(len(test_results)) + ' tests were run'

		dist_dir = get_android_dist_dir_or_die()
		log_output_path = os.path.join(dist_dir, 'gtest/coverage')
		cmd_path = os.path.join(get_android_root_or_die(), 'packages/modules/Bluetooth/system/test')
		print cmd_path
		cmd = [
		os.path.join(cmd_path, 'gen_coverage.py'),
		'--skip-html',
		'--json-file',
		'-o',
		log_output_path,
		]
		subprocess.call(cmd)

		sys.exit(0)