Merge "pdl: Expand the list of tests for the python generator" am: f5eb27f29f... (7767a2c2) · Commits · e / os / android_packages_modules_Bluetooth

tools/pdl/Android.bp

+33 −6

Original line number	Diff line number	Diff line
		@@ -76,10 +76,10 @@ rust_test_host {
		],
		}

		// Generate the python parser+serializer backend for the test file
		// located at tests/examples/complex.pdl.
		// Generate the python parser+serializer backend for the
		// little endian test file located at tests/canonical/le_test_file.pdl.
		genrule {
		name: "pdl_python_generator_test_gen",
		name: "pdl_python_generator_le_test_gen",
		cmd: "$(location :pdl) $(in) \|" +
		" $(location scripts/generate_python_backend.py)" +
		" --output $(out) --custom-type-location tests.custom_types",
		@@ -91,10 +91,32 @@ genrule {
		"tests/custom_types.py",
		],
		srcs: [
		"tests/examples/complex.pdl",
		"tests/canonical/le_test_file.pdl",
		],
		out: [
		"pdl_test.py",
		"le_pdl_test.py",
		],
		}

		// Generate the python parser+serializer backend for the
		// big endian test file located at tests/canonical/be_test_file.pdl.
		genrule {
		name: "pdl_python_generator_be_test_gen",
		cmd: "$(location :pdl) $(in) \|" +
		" $(location scripts/generate_python_backend.py)" +
		" --output $(out) --custom-type-location tests.custom_types",
		tools: [ ":pdl" ],
		tool_files: [
		"scripts/generate_python_backend.py",
		"scripts/pdl/core.py",
		"scripts/pdl/ast.py",
		"tests/custom_types.py",
		],
		srcs: [
		"tests/canonical/be_test_file.pdl",
		],
		out: [
		"be_pdl_test.py",
		],
		}

		@@ -106,7 +128,12 @@ python_test_host {
		srcs: [
		"tests/python_generator_test.py",
		"tests/custom_types.py",
		":pdl_python_generator_test_gen",
		":pdl_python_generator_le_test_gen",
		":pdl_python_generator_be_test_gen",
		],
		data: [
		"tests/canonical/le_test_vectors.json",
		"tests/canonical/be_test_vectors.json",
		],
		test_options: {
		unit_test: true,

tools/pdl/scripts/generate_python_backend.py

+19 −3

Original line number	Diff line number	Diff line
		@@ -350,12 +350,30 @@ class FieldParser:
		"""Parse body and payload fields."""

		size = core.get_payload_field_size(field)
		offset_from_end = core.get_field_offset_from_end(field)
		self.consume_span_()

		# The payload or body has a known size.
		# Consume the payload and update the span in case
		# fields are placed after the payload.
		if size:
		self.check_size_(f'{field.id}_size')
		self.append_(f"payload = span[:{field.id}_size]")
		else:
		self.append_(f"span = span[{field.id}_size:]")
		# The payload or body is the last field of a packet,
		# consume the remaining span.
		elif offset_from_end == 0:
		self.append_(f"payload = span")
		self.append_(f"span = bytes([])")
		# The payload or body is followed by fields of static size.
		# Consume the span that is not reserved for the following fields.
		elif offset_from_end is not None:
		if (offset_from_end % 8) != 0:
		raise Exception('Payload field offset from end of packet is not a multiple of 8')
		offset_from_end = int(offset_from_end / 8)
		self.check_size_(f'{offset_from_end}')
		self.append_(f"payload = span[:-{offset_from_end}]")
		self.append_(f"span = span[-{offset_from_end}:]")
		self.append_(f"fields['payload'] = payload")

		def parse_checksum_field_(self, field: ast.ChecksumField):
		@@ -616,9 +634,7 @@ def generate_checksum_declaration_check(decl: ast.ChecksumDeclaration) -> str:


		def run(input: argparse.FileType, output: argparse.FileType, custom_type_location: Optional[str]):
		# with open(input) as pdl_json:
		file = ast.File.from_json(json.load(input))

		core.desugar(file)

		custom_types = []

tools/pdl/scripts/pdl/core.py

+35 −2

Original line number	Diff line number	Diff line
		@@ -9,11 +9,15 @@ def desugar_field_(field: Field, constraints: Dict[str, Constraint]) -> List[Fie

		if isinstance(field, ScalarField) and field.id in constraints:
		value = constraints[field.id].value
		return [FixedField(kind='fixed_field', loc=field.loc, width=field.width, value=value)]
		fixed = FixedField(kind='fixed_field', loc=field.loc, width=field.width, value=value)
		fixed.parent = field.parent
		return [fixed]

		elif isinstance(field, TypedefField) and field.id in constraints:
		tag_id = constraints[field.id].tag_id
		return [FixedField(kind='fixed_field', loc=field.loc, enum_id=field.type_id, tag_id=tag_id)]
		fixed = FixedField(kind='fixed_field', loc=field.loc, enum_id=field.type_id, tag_id=tag_id)
		fixed.parent = field.parent
		return [fixed]

		elif isinstance(field, GroupField):
		group = field.parent.file.group_scope[field.group_id]
		@@ -169,6 +173,35 @@ def get_array_element_size(field: ArrayField) -> Optional[int]:
		return field.width or get_declaration_size(field.type)


		def get_field_offset_from_start(field: Field) -> Optional[int]:
		"""Return the field bit offset from the start of the parent packet, if it
		can be statically computed. If the offset is variable None is returned
		instead."""
		offset = 0
		field_index = field.parent.fields.index(field)
		for f in field.parent.fields[:field_index]:
		size = get_field_size(f)
		if size is None:
		return None

		offset += size
		return offset


		def get_field_offset_from_end(field: Field) -> Optional[int]:
		"""Return the field bit offset from the end of the parent packet, if it
		can be statically computed. If the offset is variable None is returned
		instead. The selected field size is not counted towards the offset."""
		offset = 0
		field_index = field.parent.fields.index(field)
		for f in field.parent.fields[field_index + 1:]:
		size = get_field_size(f)
		if size is None:
		return None
		offset += size
		return offset


		def is_bit_field(field: Field) -> bool:
		"""Identify fields that can have bit granularity.
		These include: ScalarField, FixedField, TypedefField with enum type,

tools/pdl/tests/canonical/be_test_file.pdl

0 → 100644

+634 −0

Original line number	Diff line number	Diff line
		big_endian_packets

		// Preliminary definitions

		custom_field SizedCustomField : 8 "SizedCustomField"
		custom_field UnsizedCustomField "UnsizedCustomField"
		checksum Checksum : 8 "Checksum"

		enum Enum7 : 7 {
		A = 1,
		B = 2,
		}

		enum Enum16 : 16 {
		A = 0xaabb,
		B = 0xccdd,
		}

		struct SizedStruct {
		a: 8,
		}

		struct UnsizedStruct {
		_size_(array): 2,
		_reserved_: 6,
		array: 8[],
		}

		group ScalarGroup {
		a: 16
		}

		group EnumGroup {
		a: Enum16
		}

		packet ScalarParent {
		a: 8,
		_size_(_payload_): 8,
		_payload_
		}

		packet EnumParent {
		a: Enum16,
		_size_(_payload_): 8,
		_payload_
		}

		packet EmptyParent : ScalarParent {
		_payload_
		}

		// Packet bit fields

		// The parser must be able to handle bit fields with scalar values
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_Scalar_Field {
		a: 7,
		c: 57,
		}

		// The parser must be able to handle bit fields with enum values
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_Enum_Field {
		a: Enum7,
		c: 57,
		}

		// The parser must be able to handle bit fields with reserved fields
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_Reserved_Field {
		a: 7,
		_reserved_: 2,
		c: 55,
		}

		// The parser must be able to handle bit fields with size fields
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_Size_Field {
		_size_(b): 3,
		a: 61,
		b: 8[],
		}

		// The parser must be able to handle bit fields with count fields
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_Count_Field {
		_count_(b): 3,
		a: 61,
		b: 8[],
		}

		// The parser must be able to handle bit fields with fixed scalar values
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_FixedScalar_Field {
		_fixed_ = 7 : 7,
		b: 57,
		}

		// The parser must be able to handle bit fields with fixed enum values
		// up to 64 bits wide. The parser should generate a static size guard.
		packet Packet_FixedEnum_Field {
		_fixed_ = A : Enum7,
		b: 57,
		}

		// Packet payload fields

		// The parser must be able to handle sized payload fields without
		// size modifier.
		packet Packet_Payload_Field_VariableSize {
		_size_(_payload_): 3,
		_reserved_: 5,
		_payload_
		}

		// The parser must be able to handle sized payload fields with
		// size modifier.
		packet Packet_Payload_Field_SizeModifier {
		_size_(_payload_): 3,
		_reserved_: 5,
		_payload_ : [+2],
		}

		// The parser must be able to handle payload fields of unkonwn size followed
		// by fields of statically known size. The remaining span is integrated
		// in the packet.
		packet Packet_Payload_Field_UnknownSize {
		_payload_,
		a: 16,
		}

		// The parser must be able to handle payload fields of unkonwn size.
		// The remaining span is integrated in the packet.
		packet Packet_Payload_Field_UnknownSize_Terminal {
		a: 16,
		_payload_,
		}

		// Packet body fields

		// The parser must be able to handle sized body fields without
		// size modifier when the packet has no children.
		packet Packet_Body_Field_VariableSize {
		_size_(_body_): 3,
		_reserved_: 5,
		_body_
		}

		// The parser must be able to handle body fields of unkonwn size followed
		// by fields of statically known size. The remaining span is integrated
		// in the packet.
		packet Packet_Body_Field_UnknownSize {
		_body_,
		a: 16,
		}

		// The parser must be able to handle body fields of unkonwn size.
		// The remaining span is integrated in the packet.
		packet Packet_Body_Field_UnknownSize_Terminal {
		a: 16,
		_body_,
		}

		// Packet group fields

		packet Packet_ScalarGroup_Field {
		ScalarGroup { a = 42 },
		}

		packet Packet_EnumGroup_Field {
		EnumGroup { a = A },
		}

		// Packet checksum fields

		// The parser must be able to handle checksum fields if the checksum value
		// field is positioned at constant offset from the checksum start.
		// The parser should generate a checksum guard for the buffer covered by the
		// checksum.
		packet Packet_Checksum_Field_FromStart {
		_checksum_start_(crc),
		a: 16,
		b: 16,
		crc: Checksum,
		}

		// The parser must be able to handle checksum fields if the checksum value
		// field is positioned at constant offset from the end of the packet.
		// The parser should generate a checksum guard for the buffer covered by the
		// checksum.
		packet Packet_Checksum_Field_FromEnd {
		_checksum_start_(crc),
		_payload_,
		crc: Checksum,
		a: 16,
		b: 16,
		}

		// Packet typedef fields

		// The parser must be able to handle struct fields.
		// The size guard is generated by the Struct parser.
		packet Packet_Struct_Field {
		a: SizedStruct,
		b: UnsizedStruct,
		}

		// The parser must be able to handle padding fields.
		// The parser should generate a static size guard.
		packet Packet_Padding_Field {
		a: 8,
		_padding_ [1],
		b: 8,
		}

		// The parser must be able to handle custom fields of constant size.
		// The parser should generate a static size guard.
		packet Packet_Custom_Field_ConstantSize {
		a: SizedCustomField,
		}

		// The parser must be able to handle custom fields of undefined size.
		// No size guard possible.
		packet Packet_Custom_Field_VariableSize {
		a: UnsizedCustomField,
		}

		// Array fields
		// Array element types

		// Array with scalar elements.
		packet Packet_Array_Field_ScalarElement {
		array: 16[4],
		}

		// Array with enum elements.
		packet Packet_Array_Field_EnumElement {
		array: Enum16[4],
		}

		// Array with sized structure elements.
		packet Packet_Array_Field_SizedStructElement {
		array: SizedStruct[4],
		}

		// Array with variable sized structure elements.
		packet Packet_Array_Field_UnsizedStructElement {
		array: UnsizedStruct[4],
		}

		// Array configurations

		// Array with sized elements, and a variable count.
		// The parser must extract as many elements as specified by the count from
		// the remaining span. The parser should generate a dynamic array size guard.
		packet Packet_Array_Field_SizedElement_VariableCount {
		_count_(array) : 4,
		_reserved_: 4,
		array: 16[],
		}

		// The parser must extract as many elements as possible from
		// the span with the specified size. The parser should generate a
		// dynamic array size guard.
		packet Packet_Array_Field_SizedElement_VariableSize {
		_size_(array) : 4,
		_reserved_: 4,
		array: 16[],
		}

		// The parser must extract as many elements as specified by the
		// static array count. The parser should generate a static array size guard.
		packet Packet_Array_Field_SizedElement_ConstantSize {
		array: 16[8],
		}

		// The parser must extract as many elements as specified by the count from
		// the remaining span. No size guard possible.
		packet Packet_Array_Field_UnsizedElement_VariableCount {
		_count_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[],
		}

		// The parser must extract as many elements as possible from
		// the span with the specified size. The parser should generate a
		// dynamic array size guard.
		packet Packet_Array_Field_UnsizedElement_VariableSize {
		_size_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[],
		}

		// The parser must extract as many elements as specified by the
		// static array count. No size guard possible.
		packet Packet_Array_Field_UnsizedElement_ConstantSize {
		array: UnsizedStruct[8],
		}

		// The parser must support complex size modifiers on arrays whose size is
		// specified by a size field.
		packet Packet_Array_Field_UnsizedElement_SizeModifier {
		_size_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[+8*2],
		}

		// The parser must support complex size modifiers on arrays whose size is
		// specified by a count field.
		packet Packet_Array_Field_UnsizedElement_CountModifier {
		_count_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[-8*2],
		}

		// Packet inheritance

		// The parser must handle specialization into
		// any child packet of a parent packet with scalar constraints.
		packet ScalarChild_A : ScalarParent (a = 0) {
		b: 8,
		}

		// The parser must handle specialization into
		// any child packet of a parent packet with scalar constraints.
		packet ScalarChild_B : ScalarParent (a = 1) {
		c: 16,
		}

		// The parser must handle specialization into
		// any child packet of a parent packet with enum constraints.
		packet EnumChild_A : EnumParent (a = A) {
		b: 8,
		}

		// The parser must handle specialization into
		// any child packet of a parent packet with enum constraints.
		packet EnumChild_B : EnumParent (a = B) {
		c: 16,
		}

		// The parser must handle aliasing of packets
		// through inheritance with no constraints
		packet AliasedChild_A : EmptyParent (a = 2) {
		b: 8,
		}

		// The parser must handle aliasing of packets
		// through inheritance with no constraints
		packet AliasedChild_B : EmptyParent (a = 3) {
		c: 16,
		}

		// Struct bit fields

		// The parser must be able to handle bit fields with scalar values
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_Scalar_Field {
		a: 7,
		c: 57,
		}

		// The parser must be able to handle bit fields with enum values
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_Enum_Field_ {
		a: Enum7,
		c: 57,
		}
		packet Struct_Enum_Field {
		s: Struct_Enum_Field_,
		}

		// The parser must be able to handle bit fields with reserved fields
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_Reserved_Field_ {
		a: 7,
		_reserved_: 2,
		c: 55,
		}
		packet Struct_Reserved_Field {
		s: Struct_Reserved_Field_,
		}

		// The parser must be able to handle bit fields with size fields
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_Size_Field_ {
		_size_(b): 3,
		a: 61,
		b: 8[],
		}
		packet Struct_Size_Field {
		s: Struct_Size_Field_,
		}

		// The parser must be able to handle bit fields with count fields
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_Count_Field_ {
		_count_(b): 3,
		a: 61,
		b: 8[],
		}
		packet Struct_Count_Field {
		s: Struct_Count_Field_,
		}

		// The parser must be able to handle bit fields with fixed scalar values
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_FixedScalar_Field_ {
		_fixed_ = 7 : 7,
		b: 57,
		}
		packet Struct_FixedScalar_Field {
		s: Struct_FixedScalar_Field_,
		}

		// The parser must be able to handle bit fields with fixed enum values
		// up to 64 bits wide. The parser should generate a static size guard.
		struct Struct_FixedEnum_Field_ {
		_fixed_ = A : Enum7,
		b: 57,
		}
		packet Struct_FixedEnum_Field {
		s: Struct_FixedEnum_Field_,
		}

		// Struct group fields

		struct Struct_ScalarGroup_Field_ {
		ScalarGroup { a = 42 },
		}
		packet Struct_ScalarGroup_Field {
		s: Struct_ScalarGroup_Field_,
		}

		struct Struct_EnumGroup_Field_ {
		EnumGroup { a = A },
		}
		packet Struct_EnumGroup_Field {
		s: Struct_EnumGroup_Field_,
		}

		// Struct checksum fields

		// The parser must be able to handle checksum fields if the checksum value
		// field is positioned at constant offset from the checksum start.
		// The parser should generate a checksum guard for the buffer covered by the
		// checksum.
		struct Struct_Checksum_Field_FromStart_ {
		_checksum_start_(crc),
		a: 16,
		b: 16,
		crc: Checksum,
		}
		packet Struct_Checksum_Field_FromStart {
		s: Struct_Checksum_Field_FromStart_,
		}

		// The parser must be able to handle checksum fields if the checksum value
		// field is positioned at constant offset from the end of the packet.
		// The parser should generate a checksum guard for the buffer covered by the
		// checksum.
		struct Struct_Checksum_Field_FromEnd_ {
		_checksum_start_(crc),
		_payload_,
		crc: Checksum,
		a: 16,
		b: 16,
		}
		packet Struct_Checksum_Field_FromEnd {
		s: Struct_Checksum_Field_FromEnd_,
		}

		// Struct typedef fields

		// The parser must be able to handle struct fields.
		// The size guard is generated by the Struct parser.
		packet Struct_Struct_Field {
		a: SizedStruct,
		b: UnsizedStruct,
		}

		// The parser must be able to handle padding fields.
		// The parser should generate a static size guard.
		struct Struct_Padding_Field_ {
		a: 8,
		_padding_ [1],
		b: 8,
		}
		packet Struct_Padding_Field {
		s: Struct_Padding_Field_,
		}

		// The parser must be able to handle custom fields of constant size.
		// The parser should generate a static size guard.
		struct Struct_Custom_Field_ConstantSize_ {
		a: SizedCustomField,
		}
		packet Struct_Custom_Field_ConstantSize {
		s: Struct_Custom_Field_ConstantSize_,
		}

		// The parser must be able to handle custom fields of undefined size.
		// No size guard possible.
		struct Struct_Custom_Field_VariableSize_ {
		a: UnsizedCustomField,
		}
		packet Struct_Custom_Field_VariableSize {
		s: Struct_Custom_Field_VariableSize_,
		}

		// Array fields
		// Array element types

		// Array with scalar elements.
		struct Struct_Array_Field_ScalarElement_ {
		array: 16[4],
		}
		packet Struct_Array_Field_ScalarElement {
		s: Struct_Array_Field_ScalarElement_,
		}

		// Array with enum elements.
		struct Struct_Array_Field_EnumElement_ {
		array: Enum16[4],
		}
		packet Struct_Array_Field_EnumElement {
		s: Struct_Array_Field_EnumElement_,
		}

		// Array with sized structure elements.
		struct Struct_Array_Field_SizedStructElement_ {
		array: SizedStruct[4],
		}
		packet Struct_Array_Field_SizedStructElement {
		s: Struct_Array_Field_SizedStructElement_,
		}

		// Array with variable sized structure elements.
		struct Struct_Array_Field_UnsizedStructElement_ {
		array: UnsizedStruct[4],
		}
		packet Struct_Array_Field_UnsizedStructElement {
		s: Struct_Array_Field_UnsizedStructElement_,
		}

		// Array configurations

		// Array with sized elements, and a variable count.
		// The parser must extract as many elements as specified by the count from
		// the remaining span. The parser should generate a dynamic array size guard.
		struct Struct_Array_Field_SizedElement_VariableCount_ {
		_count_(array) : 4,
		_reserved_: 4,
		array: 16[],
		}
		packet Struct_Array_Field_SizedElement_VariableCount {
		s: Struct_Array_Field_SizedElement_VariableCount_,
		}

		// The parser must extract as many elements as possible from
		// the span with the specified size. The parser should generate a
		// dynamic array size guard.
		struct Struct_Array_Field_SizedElement_VariableSize_ {
		_size_(array) : 4,
		_reserved_: 4,
		array: 16[],
		}
		packet Struct_Array_Field_SizedElement_VariableSize {
		s: Struct_Array_Field_SizedElement_VariableSize_,
		}

		// The parser must extract as many elements as specified by the
		// static array count. The parser should generate a static array size guard.
		struct Struct_Array_Field_SizedElement_ConstantSize_ {
		array: 16[8],
		}
		packet Struct_Array_Field_SizedElement_ConstantSize {
		s: Struct_Array_Field_SizedElement_ConstantSize_,
		}

		// The parser must extract as many elements as specified by the count from
		// the remaining span. No size guard possible.
		struct Struct_Array_Field_UnsizedElement_VariableCount_ {
		_count_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[],
		}
		packet Struct_Array_Field_UnsizedElement_VariableCount {
		s: Struct_Array_Field_UnsizedElement_VariableCount_,
		}

		// The parser must extract as many elements as possible from
		// the span with the specified size. The parser should generate a
		// dynamic array size guard.
		struct Struct_Array_Field_UnsizedElement_VariableSize_ {
		_size_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[],
		}
		packet Struct_Array_Field_UnsizedElement_VariableSize {
		s: Struct_Array_Field_UnsizedElement_VariableSize_,
		}

		// The parser must extract as many elements as specified by the
		// static array count. No size guard possible.
		struct Struct_Array_Field_UnsizedElement_ConstantSize_ {
		array: UnsizedStruct[8],
		}
		packet Struct_Array_Field_UnsizedElement_ConstantSize {
		s: Struct_Array_Field_UnsizedElement_ConstantSize_,
		}

		// The parser must support complex size modifiers on arrays whose size is
		// specified by a size field.
		struct Struct_Array_Field_UnsizedElement_SizeModifier_ {
		_size_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[+8*2],
		}
		packet Struct_Array_Field_UnsizedElement_SizeModifier {
		s: Struct_Array_Field_UnsizedElement_SizeModifier_,
		}

		// The parser must support complex size modifiers on arrays whose size is
		// specified by a count field.
		struct Struct_Array_Field_UnsizedElement_CountModifier_ {
		_count_(array) : 4,
		_reserved_: 4,
		array: UnsizedStruct[-8*2],
		}
		packet Struct_Array_Field_UnsizedElement_CountModifier {
		s: Struct_Array_Field_UnsizedElement_CountModifier_,
		}

tools/pdl/tests/canonical/be_test_vectors.json

0 → 100644

+1122 −0

File added.

Preview size limit exceeded, changes collapsed.