Merge "pdl: Implement c++ backend generator" am: b315d83f7a (b43bfffc) · Commits · e / os / android_packages_modules_Bluetooth

tools/pdl/Android.bp

+123 −0

Original line number	Original line	Diff line number	Diff line
	@@ -318,6 +318,15 @@ genrule_defaults {
	],		],
	}		}

			// Defaults for PDL python backend generation.
			genrule_defaults {
			name: "pdl_cxx_generator_defaults",
			tools: [
			":pdl",
			":pdl_cxx_generator",
			],
			}

	// Generate the python parser+serializer backend for the		// Generate the python parser+serializer backend for the
	// little endian test file located at tests/canonical/le_test_file.pdl.		// little endian test file located at tests/canonical/le_test_file.pdl.
	genrule {		genrule {
	@@ -440,3 +449,117 @@ rust_test_host {
	],		],
	test_suites: ["general-tests"],		test_suites: ["general-tests"],
	}		}

			// Generate the C++ parser+serializer backend for the
			// little endian test file located at tests/canonical/le_test_file.pdl.
			genrule {
			name: "pdl_cxx_canonical_le_src_gen",
			defaults: ["pdl_cxx_generator_defaults"],
			cmd: "set -o pipefail;" +
			" $(location :pdl) $(in) \|" +
			" $(location :pdl_cxx_generator)" +
			" --namespace le_test" +
			" --output $(out)",
			srcs: [
			"tests/canonical/le_test_file.pdl",
			],
			out: [
			"canonical_le_test_file.h",
			],
			}

			// Generate the C++ parser+serializer backend tests for the
			// little endian test file located at tests/canonical/le_test_file.pdl.
			genrule {
			name: "pdl_cxx_canonical_le_test_gen",
			cmd: "set -o pipefail;" +
			" inputs=( $(in) ) &&" +
			" $(location :pdl) $${inputs[0]} \|" +
			" $(location :pdl_cxx_unittest_generator)" +
			" --output $(out)" +
			" --test-vectors $${inputs[1]}" +
			" --include-header $$(basename $${inputs[2]})" +
			" --using-namespace le_test" +
			" --namespace le_test" +
			" --parser-test-suite LeParserTest" +
			" --serializer-test-suite LeSerializerTest",
			tools: [
			":pdl",
			":pdl_cxx_unittest_generator",
			],
			srcs: [
			"tests/canonical/le_test_file.pdl",
			"tests/canonical/le_test_vectors.json",
			":pdl_cxx_canonical_le_src_gen",
			],
			out: [
			"canonical_le_test.cc",
			],
			}

			// Generate the C++ parser+serializer backend for the
			// big endian test file.
			genrule {
			name: "pdl_cxx_canonical_be_src_gen",
			defaults: ["pdl_cxx_generator_defaults"],
			cmd: "set -o pipefail;" +
			" $(location :pdl) $(in) \|" +
			" $(location :pdl_cxx_generator)" +
			" --namespace be_test" +
			" --output $(out)",
			srcs: [
			":pdl_be_test_file",
			],
			out: [
			"canonical_be_test_file.h",
			],
			}

			// Generate the C++ parser+serializer backend tests for the
			// big endian test file.
			genrule {
			name: "pdl_cxx_canonical_be_test_gen",
			cmd: "set -o pipefail;" +
			" inputs=( $(in) ) &&" +
			" $(location :pdl) $${inputs[0]} \|" +
			" $(location :pdl_cxx_unittest_generator)" +
			" --output $(out)" +
			" --test-vectors $${inputs[1]}" +
			" --include-header $$(basename $${inputs[2]})" +
			" --using-namespace be_test" +
			" --namespace be_test" +
			" --parser-test-suite BeParserTest" +
			" --serializer-test-suite BeSerializerTest",
			tools: [
			":pdl",
			":pdl_cxx_unittest_generator",
			],
			srcs: [
			":pdl_be_test_file",
			"tests/canonical/be_test_vectors.json",
			":pdl_cxx_canonical_be_src_gen",
			],
			out: [
			"canonical_be_test.cc",
			],
			}

			// Test the generated C++ parser+serializer against
			// pre-generated binary inputs.
			cc_test_host {
			name: "pdl_cxx_generator_test",
			local_include_dirs: [
			"scripts",
			],
			generated_headers: [
			"pdl_cxx_canonical_le_src_gen",
			"pdl_cxx_canonical_be_src_gen",
			],
			generated_sources: [
			"pdl_cxx_canonical_le_test_gen",
			"pdl_cxx_canonical_be_test_gen",
			],
			static_libs: [
			"libgtest",
			],
			}

tools/pdl/scripts/Android.bp

+25 −0

Original line number	Original line	Diff line number	Diff line
	@@ -21,5 +21,30 @@ python_binary_host {
	"generate_python_backend.py",		"generate_python_backend.py",
	"pdl/ast.py",		"pdl/ast.py",
	"pdl/core.py",		"pdl/core.py",
			"pdl/utils.py",
			],
			}

			// C++ generator.
			python_binary_host {
			name: "pdl_cxx_generator",
			main: "generate_cxx_backend.py",
			srcs: [
			"generate_cxx_backend.py",
			"pdl/ast.py",
			"pdl/core.py",
			"pdl/utils.py",
			],
			}

			// C++ test generator.
			python_binary_host {
			name: "pdl_cxx_unittest_generator",
			main: "generate_cxx_backend_tests.py",
			srcs: [
			"generate_cxx_backend_tests.py",
			"pdl/ast.py",
			"pdl/core.py",
			"pdl/utils.py",
	],		],
	}		}

tools/pdl/scripts/generate_cxx_backend.py

0 → 100755

+1378 −0

File added.

Preview size limit exceeded, changes collapsed.

tools/pdl/scripts/generate_cxx_backend_tests.py

0 → 100755

+305 −0

Original line number	Original line	Diff line number	Diff line
			#!/usr/bin/env python3

			import argparse
			from dataclasses import dataclass, field
			import json
			from pathlib import Path
			import sys
			from textwrap import dedent
			from typing import List, Tuple, Union, Optional

			from pdl import ast, core
			from pdl.utils import indent, to_pascal_case


			def get_cxx_scalar_type(width: int) -> str:
			"""Return the cxx scalar type to be used to back a PDL type."""
			for n in [8, 16, 32, 64]:
			if width <= n:
			return f'uint{n}_t'
			# PDL type does not fit on non-extended scalar types.
			assert False


			def generate_packet_parser_test(parser_test_suite: str, packet: ast.PacketDeclaration, tests: List[object]) -> str:
			"""Generate the implementation of unit tests for the selected packet."""

			def parse_packet(packet: ast.PacketDeclaration) -> str:
			parent = parse_packet(packet.parent) if packet.parent else "input"
			return f"{packet.id}View::Create({parent})"

			def input_bytes(input: str) -> List[str]:
			input = bytes.fromhex(input)
			input_bytes = []
			for i in range(0, len(input), 16):
			input_bytes.append(' '.join(f'0x{b:x},' for b in input[i:i + 16]))
			return input_bytes

			def get_field(decl: ast.Declaration, var: str, id: str) -> str:
			if isinstance(decl, ast.StructDeclaration):
			return f"{var}.{id}_"
			else:
			return f"{var}.Get{to_pascal_case(id)}()"

			def check_members(decl: ast.Declaration, var: str, expected: object) -> List[str]:
			checks = []
			for (id, value) in expected.items():
			field = core.get_packet_field(decl, id)
			sanitized_var = var.replace('[', '_').replace(']', '')
			field_var = f'{sanitized_var}_{id}'

			if isinstance(field, ast.ScalarField):
			checks.append(f"ASSERT_EQ({get_field(decl, var, id)}, {value});")

			elif (isinstance(field, ast.TypedefField) and
			isinstance(field.type, (ast.EnumDeclaration, ast.CustomFieldDeclaration, ast.ChecksumDeclaration))):
			checks.append(f"ASSERT_EQ({get_field(decl, var, id)}, {field.type_id}({value}));")

			elif isinstance(field, ast.TypedefField):
			checks.append(f"{field.type_id} const& {field_var} = {get_field(decl, var, id)};")
			checks.extend(check_members(field.type, field_var, value))

			elif isinstance(field, (ast.PayloadField, ast.BodyField)):
			checks.append(f"std::vector<uint8_t> expected_{field_var} {{")
			for i in range(0, len(value), 16):
			checks.append(' ' + ' '.join([f"0x{v:x}," for v in value[i:i + 16]]))
			checks.append("};")
			checks.append(f"ASSERT_EQ({get_field(decl, var, id)}, expected_{field_var});")

			elif isinstance(field, ast.ArrayField) and field.width:
			checks.append(f"std::vector<{get_cxx_scalar_type(field.width)}> expected_{field_var} {{")
			step = int(16 * 8 / field.width)
			for i in range(0, len(value), step):
			checks.append(' ' + ' '.join([f"0x{v:x}," for v in value[i:i + step]]))
			checks.append("};")
			checks.append(f"ASSERT_EQ({get_field(decl, var, id)}, expected_{field_var});")

			elif (isinstance(field, ast.ArrayField) and isinstance(field.type, ast.EnumDeclaration)):
			checks.append(f"std::vector<{field.type_id}> expected_{field_var} {{")
			for v in value:
			checks.append(f" {field.type_id}({v}),")
			checks.append("};")
			checks.append(f"ASSERT_EQ({get_field(decl, var, id)}, expected_{field_var});")

			elif isinstance(field, ast.ArrayField):
			checks.append(f"std::vector<{field.type_id}> {field_var} = {get_field(decl, var, id)};")
			checks.append(f"ASSERT_EQ({field_var}.size(), {len(value)});")
			for (n, value) in enumerate(value):
			checks.extend(check_members(field.type, f"{field_var}[{n}]", value))

			else:
			pass

			return checks

			generated_tests = []
			for (test_nr, test) in enumerate(tests):
			child_packet_id = test.get('packet', packet.id)
			child_packet = packet.file.packet_scope[child_packet_id]

			generated_tests.append(
			dedent("""\

			TEST_F({parser_test_suite}, {packet_id}_Case{test_nr}) {{
			pdl::packet::slice input(std::shared_ptr<std::vector<uint8_t>>(new std::vector<uint8_t> {{
			{input_bytes}
			}}));
			{child_packet_id}View packet = {parse_packet};
			ASSERT_TRUE(packet.IsValid());
			{checks}
			}}
			""").format(parser_test_suite=parser_test_suite,
			packet_id=packet.id,
			child_packet_id=child_packet_id,
			test_nr=test_nr,
			input_bytes=indent(input_bytes(test['packed']), 2),
			parse_packet=parse_packet(child_packet),
			checks=indent(check_members(packet, 'packet', test['unpacked']), 1)))

			return ''.join(generated_tests)


			def generate_packet_serializer_test(serializer_test_suite: str, packet: ast.PacketDeclaration,
			tests: List[object]) -> str:
			"""Generate the implementation of unit tests for the selected packet."""

			def build_packet(decl: ast.Declaration, var: str, initializer: object) -> (str, List[str]):
			fields = core.get_unconstrained_parent_fields(decl) + decl.fields
			declarations = []
			parameters = []
			for field in fields:
			sanitized_var = var.replace('[', '_').replace(']', '')
			field_id = getattr(field, 'id', None)
			field_var = f'{sanitized_var}_{field_id}'
			value = initializer['payload'] if isinstance(field, (ast.PayloadField,
			ast.BodyField)) else initializer.get(field_id, None)

			if isinstance(field, ast.ScalarField):
			parameters.append(f"{value}")

			elif isinstance(field, ast.TypedefField) and isinstance(field.type, ast.EnumDeclaration):
			parameters.append(f"{field.type_id}({value})")

			elif isinstance(field, ast.TypedefField):
			(element, intermediate_declarations) = build_packet(field.type, field_var, value)
			declarations.extend(intermediate_declarations)
			parameters.append(element)

			elif isinstance(field, (ast.PayloadField, ast.BodyField)):
			declarations.append(f"std::vector<uint8_t> {field_var} {{")
			for i in range(0, len(value), 16):
			declarations.append(' ' + ' '.join([f"0x{v:x}," for v in value[i:i + 16]]))
			declarations.append("};")
			parameters.append(f"std::move({field_var})")

			elif isinstance(field, ast.ArrayField) and field.width:
			declarations.append(f"std::vector<{get_cxx_scalar_type(field.width)}> {field_var} {{")
			step = int(16 * 8 / field.width)
			for i in range(0, len(value), step):
			declarations.append(' ' + ' '.join([f"0x{v:x}," for v in value[i:i + step]]))
			declarations.append("};")
			parameters.append(f"std::move({field_var})")

			elif isinstance(field, ast.ArrayField) and isinstance(field.type, ast.EnumDeclaration):
			declarations.append(f"std::vector<{field.type_id}> {field_var} {{")
			for v in value:
			declarations.append(f" {field.type_id}({v}),")
			declarations.append("};")
			parameters.append(f"std::move({field_var})")

			elif isinstance(field, ast.ArrayField):
			elements = []
			for (n, value) in enumerate(value):
			(element, intermediate_declarations) = build_packet(field.type, f'{field_var}_{n}', value)
			elements.append(element)
			declarations.extend(intermediate_declarations)
			declarations.append(f"std::vector<{field.type_id}> {field_var} {{")
			for element in elements:
			declarations.append(f" {element},")
			declarations.append("};")
			parameters.append(f"std::move({field_var})")

			else:
			pass

			constructor_name = f'{decl.id}Builder' if isinstance(decl, ast.PacketDeclaration) else decl.id
			return (f"{constructor_name}({', '.join(parameters)})", declarations)

			def output_bytes(output: str) -> List[str]:
			output = bytes.fromhex(output)
			output_bytes = []
			for i in range(0, len(output), 16):
			output_bytes.append(' '.join(f'0x{b:x},' for b in output[i:i + 16]))
			return output_bytes

			generated_tests = []
			for (test_nr, test) in enumerate(tests):
			child_packet_id = test.get('packet', packet.id)
			child_packet = packet.file.packet_scope[child_packet_id]

			(built_packet, intermediate_declarations) = build_packet(child_packet, 'packet', test['unpacked'])
			generated_tests.append(
			dedent("""\

			TEST_F({serializer_test_suite}, {packet_id}_Case{test_nr}) {{
			std::vector<uint8_t> expected_output {{
			{output_bytes}
			}};
			{intermediate_declarations}
			{child_packet_id}Builder packet = {built_packet};
			ASSERT_EQ(packet.pdl::packet::Builder::Serialize(), expected_output);
			}}
			""").format(serializer_test_suite=serializer_test_suite,
			packet_id=packet.id,
			child_packet_id=child_packet_id,
			test_nr=test_nr,
			output_bytes=indent(output_bytes(test['packed']), 2),
			built_packet=built_packet,
			intermediate_declarations=indent(intermediate_declarations, 1)))

			return ''.join(generated_tests)


			def run(input: argparse.FileType, output: argparse.FileType, test_vectors: argparse.FileType, include_header: List[str],
			using_namespace: List[str], namespace: str, parser_test_suite: str, serializer_test_suite: str):

			file = ast.File.from_json(json.load(input))
			tests = json.load(test_vectors)
			core.desugar(file)

			include_header = '\n'.join([f'#include <{header}>' for header in include_header])
			using_namespace = '\n'.join([f'using namespace {namespace};' for namespace in using_namespace])

			skipped_tests = [
			'Packet_Checksum_Field_FromStart',
			'Packet_Checksum_Field_FromEnd',
			'Struct_Checksum_Field_FromStart',
			'Struct_Checksum_Field_FromEnd',
			'PartialParent5',
			'PartialParent12',
			]

			output.write(
			dedent("""\
			// File generated from {input_name} and {test_vectors_name}, with the command:
			// {input_command}
			// /!\\ Do not edit by hand

			#include <cstdint>
			#include <string>
			#include <gtest/gtest.h>
			#include <packet_runtime.h>

			{include_header}
			{using_namespace}

			namespace {namespace} {{

			class {parser_test_suite} : public testing::Test {{}};
			class {serializer_test_suite} : public testing::Test {{}};
			""").format(parser_test_suite=parser_test_suite,
			serializer_test_suite=serializer_test_suite,
			input_name=input.name,
			input_command=' '.join(sys.argv),
			test_vectors_name=test_vectors.name,
			include_header=include_header,
			using_namespace=using_namespace,
			namespace=namespace))

			for decl in file.declarations:
			if decl.id in skipped_tests:
			continue

			if isinstance(decl, ast.PacketDeclaration):
			matching_tests = [test['tests'] for test in tests if test['packet'] == decl.id]
			matching_tests = [test for test_list in matching_tests for test in test_list]
			if matching_tests:
			output.write(generate_packet_parser_test(parser_test_suite, decl, matching_tests))
			output.write(generate_packet_serializer_test(serializer_test_suite, decl, matching_tests))

			output.write(f"}} // namespace {namespace}\n")


			def main() -> int:
			"""Generate cxx PDL backend."""
			parser = argparse.ArgumentParser(description=__doc__)
			parser.add_argument('--input', type=argparse.FileType('r'), default=sys.stdin, help='Input PDL-JSON source')
			parser.add_argument('--output', type=argparse.FileType('w'), default=sys.stdout, help='Output C++ file')
			parser.add_argument('--test-vectors', type=argparse.FileType('r'), required=True, help='Input PDL test file')
			parser.add_argument('--namespace', type=str, default='pdl', help='Namespace of the generated file')
			parser.add_argument('--parser-test-suite', type=str, default='ParserTest', help='Name of the parser test suite')
			parser.add_argument('--serializer-test-suite',
			type=str,
			default='SerializerTest',
			help='Name of the serializer test suite')
			parser.add_argument('--include-header', type=str, default=[], action='append', help='Added include directives')
			parser.add_argument('--using-namespace',
			type=str,
			default=[],
			action='append',
			help='Added using namespace statements')
			return run(**vars(parser.parse_args()))


			if __name__ == '__main__':
			sys.exit(main())

tools/pdl/scripts/generate_python_backend.py

+1 −14

Original line number	Original line	Diff line number	Diff line
	@@ -9,20 +9,7 @@ from textwrap import dedent
	from typing import List, Tuple, Union, Optional		from typing import List, Tuple, Union, Optional

	from pdl import ast, core		from pdl import ast, core
			from pdl.utils import indent

	def indent(lines: List[str], depth: int) -> str:
	"""Indent a code block to the selected depth.
	The first line is intentionally not indented so that
	the caller may use it as:

	'''
	def generated():
	{codeblock}
	'''
	"""
	sep = '\n' + (' ' * (depth * 4))
	return sep.join(lines)


	def mask(width: int) -> str:		def mask(width: int) -> str: