Implement codegen for structures with dynamically sized arrays (1a76f85a) · Commits · e / os / android_frameworks_native

vulkan/scripts/vk_parser.py

+35 −7

Original line number	Diff line number	Diff line
		@@ -210,7 +210,7 @@ REQUIRED_DATA_MEMBERS = []
		# Mappings derived from parsing
		alias_map: Dict[str, str] = {} # Maps alias name to original name
		feature_api_map: Dict[str, List[Dict[str, str]]] = defaultdict(list) # Maps feature name to list of {struct: type_enum}
		disabled_structs = ["VkPhysicalDeviceHostImageCopyProperties"] # TODO:- b/415705512 handle VkPhysicalDeviceHostImageCopyProperties
		disabled_structs = []
		structs_with_valid_extends = []
		enum_member_map = {}
		additional_vk_format_values = {}
		@@ -1114,9 +1114,10 @@ def generate_core_struct_mapping(struct_names: list[str], vk_py_file_handle: IO[


		# --- Ancillary Mappings & Lists for vk.py ---
		def extract_list_size_mapping(struct_elements: List[ET.Element]) -> Dict[str, str]:
		def extract_list_size_mapping(struct_elements: List[ET.Element]) :
		"""Extracts mappings from list members to their corresponding size-indicating members within structs."""
		member_map = {} # Stores {list_member_name: size_member_name}
		struct_with_dynamic_size_list_variables = {} # Stores {struct_name: set of list variables}
		for type_element in struct_elements:
		struct_name = type_element.get(ATTR_NAME)
		# Skip invalid or non-physical device structs
		@@ -1128,20 +1129,47 @@ def extract_list_size_mapping(struct_elements: List[ET.Element]) -> Dict[str, st
		len_attribute_value = member_element.get(ATTR_LEN) # Name of the member holding the size
		name_tag = member_element.find(TAG_NAME)
		name_value = get_element_text(name_tag) # Name of the list/array member
		member_type_element = member_element.find(ATTR_TYPE)
		is_pointer = '*' in member_type_element.tail if member_type_element.tail else False

		# Store the mapping if valid (names exist, len is not "null-terminated")
		if name_value and len_attribute_value and len_attribute_value.strip().lower() != "null-terminated":
		member_map[name_value] = len_attribute_value.strip()
		return member_map
		if is_pointer:
		struct_with_dynamic_size_list_variables.setdefault(struct_name, set()).add(name_value)
		return member_map, struct_with_dynamic_size_list_variables


		def write_list_size_mapping(struct_elements: List[ET.Element], vk_py_file_handle: IO[str]):

		def sort_dict_by_key_and_content(input_dict):
		"""
		Sorts a dictionary first by its keys alphabetically,
		and then sorts the elements within the sets associated with each key.
		"""
		sorted_items_by_key = sorted(input_dict.items())
		sorted_dict = {}
		for key, value_set in sorted_items_by_key:
		sorted_elements = sorted(list(value_set))
		sorted_dict[key] = set(sorted_elements)

		return sorted_dict

		def write_formatted_code(list_variable_name, list_data):
		formatted_list_code = pprint.pformat(list_data, indent=4, width=100)
		formatted_content = f"""{list_variable_name} = {formatted_list_code}\n\n"""
		write_py_file(vk_py_file_handle, formatted_content)

		"""Extracts and writes the list-member-to-size-member mapping to a Python file."""
		list_size_map = extract_list_size_mapping(struct_elements)
		content = "# --- List Size Mappings (Field name to size field name) ---\n"
		list_size_str = pprint.pformat(list_size_map, indent=4, width=100)
		content += f"LIST_TYPE_FIELD_AND_SIZE_MAPPING = {list_size_str}\n\n"
		write_py_file(vk_py_file_handle, content)

		list_size_map, dynamic_list_variables = extract_list_size_mapping(struct_elements)
		write_formatted_code("LIST_TYPE_FIELD_AND_SIZE_MAPPING", list_size_map)

		sorted_dynamic_list_variables = sort_dict_by_key_and_content(dynamic_list_variables)
		write_formatted_code("STRUCT_WITH_DYNAMIC_SIZE_LIST_MAPPING", sorted_dynamic_list_variables)


		def write_all_structs(vk_py_file_handle: IO[str]) -> None:
		"""Writes a Python list of all processed PhysicalDevice struct names."""

vulkan/scripts/vkjson_gen_util.py

+206 −6

Original line number	Diff line number	Diff line
		@@ -21,11 +21,11 @@ This file contains all supporting utility functions for vkjson_generator.py
		import dataclasses
		import re
		import vk as VK
		from typing import get_origin
		from typing import List, get_origin, Tuple, Optional, get_args

		CONST_PHYSICAL_DEVICE_FEATURE_2 = "VkPhysicalDeviceFeatures2"
		CONST_PHYSICAL_DEVICE_PROPERTIES_2 = "VkPhysicalDeviceProperties2"

		PROPERTIES = "Properties"
		COPYRIGHT_WARNINGS = """///////////////////////////////////////////////////////////////////////////////
		//
		// Copyright (c) 2015-2016 The Khronos Group Inc.
		@@ -187,6 +187,42 @@ def get_struct_name(struct_name):

		return special_cases.get(variable_name, variable_name)

		def get_dataclass_list_members(class_name: str) -> List[Tuple[str, str]]:
		"""Retrieves names and types of list members within a dataclass.
		A list of tuples, where each tuple contains (list_variable_name, list_element_type).
		"""
		dataclass_type = get_dataclass_type(class_name)

		if not dataclass_type:
		return []
		"""Catches the TypeError thrown by dataclasses.fields if dataclass_type
		is not a valid dataclass class or instance.
		"""
		try:
		class_member_list = dataclasses.fields(dataclass_type)
		except TypeError:
		return []
		return [
		(field.name, get_args(field.type)[0].__name__)
		for field in class_member_list
		if get_origin(field.type) is list and get_args(field.type)
		]


		def get_class_name_from_alias(class_name):
		dataclass_type = get_dataclass_type(class_name)

		"""Handles an exception that is thrown only when running unit tests.
		"""
		if dataclass_type:
		try:
		return dataclass_type.__name__
		except AttributeError:
		return class_name
		return class_name

		def get_dataclass_type(class_name):
		return getattr(VK, class_name, None)

		def generate_extension_struct_definition(f):
		"""Generates struct definition code for extension based structs
		@@ -218,13 +254,23 @@ def generate_extension_struct_definition(f):
		for struct_name, _ in struct_map.items():
		variable_name = get_struct_name(struct_name)
		f.write(f" memset(&{variable_name}, 0, sizeof({struct_name}));\n")
		struct_entries.append(f"{struct_name} {variable_name}")
		struct_entries.append([struct_name, variable_name])

		f.write(" }\n") # End of constructor
		f.write(" bool reported;\n")
		for entry in struct_entries:
		f.write(f" {entry};\n")

		# Writing structure variables in vkjson_struct
		for struct_type, variable_name in struct_entries:
		variable_declaration = struct_type + " " + variable_name
		f.write(f" {variable_declaration};\n")

		# Populating vkjson_struct with list variables from properties structure.
		properties_and_feature_structures = [struct_name for struct_name, _ in struct_entries]
		cpp_list_variables = get_dynamic_size_list_member_from_structures(
		properties_and_feature_structures,
		)
		for list_type, variable_name in cpp_list_variables:
		list_variable_declaration = "std::vector<" + list_type + ">" + " " + variable_name + ";"
		f.write(f" {list_variable_declaration}\n")
		f.write("};\n\n") # End of struct

		return vkjson_entries
		@@ -540,6 +586,160 @@ def generate_vk_version_structs_initialization(version_data, struct_type_keyword

		return "\n".join(struct_initialization_code)

		"""
		A list of pairs, where each pair is (element_type, transformed_member_name).
		[('VkQueueFamilyProperties', 'queue_families')]
		"""
		def get_dynamic_size_list_member_from_structures(properties_and_feature_set: List) -> List[Tuple[str, str]]:
		entries: List[Tuple[str, str]] = []
		for struct_name in properties_and_feature_set:
		updated_struct_name = get_class_name_from_alias(struct_name)
		if PROPERTIES in struct_name and updated_struct_name in VK.STRUCT_WITH_DYNAMIC_SIZE_LIST_MAPPING:
		all_list_variables = get_dataclass_list_members(struct_name)
		for list_name, list_type in all_list_variables:
		if list_name in VK.STRUCT_WITH_DYNAMIC_SIZE_LIST_MAPPING[updated_struct_name]:
		name = transform_list_member_name(list_name)
		entries.append((list_type, name))
		return entries



		def transform_list_member_name(member_name: str) -> str:
		def convert_camel_to_snake_case(camel_case_string: str) -> str:
		s1 = re.sub(r'(?<!^)([A-Z][a-z])', r'_\1', camel_case_string)
		s2 = re.sub(r'(?<=[A-Z])([A-Z][a-z])', r'_\1', s1)
		return s2.lower()

		"""Converts a list member name (e.g., 'pCopySrcLayouts') to a more
		Pythonic snake_case variable name (e.g., 'copy_src_layouts').

		Removes leading 'p_' if present after initial camel to snake conversion.
		Examples:
		'pCopySrcLayouts' -> 'copy_src_layouts'
		'pDemoName' -> 'demo_name'
		'TestName' -> 'test_name'
		"""
		snake_case_name = convert_camel_to_snake_case(member_name)
		if snake_case_name.startswith('p_'):
		snake_case_name = snake_case_name[2:]
		return snake_case_name


		def generate_ext_dependent_structs_list_resizing_logic(
		vk_properties_setter_line: str,
		) -> str:
		"""
		This function iterates through a mapping of Vulkan extensions to their associated
		structs. For each relevant struct (e.g., properties structs that extend
		VkPhysicalDeviceProperties2), it generates C++ 'if' blocks. These blocks
		handle resizing of member arrays and assigning data pointers, then call
		vkGetPhysicalDeviceProperties2.
		"""
		generated_code_blocks: List[str] = []

		for extension_name, struct_definitions_list in VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"].items():
		device_extension_variable = get_vkjson_struct_variable_name(extension_name)
		structures = [list(struct_definition.keys())[0] for struct_definition in struct_definitions_list]
		code = generate_list_resizing_logic(structures,
		vk_properties_setter_line,
		device_extension_variable,
		)
		if code != "":
		generated_code_blocks.append(code)
		return "\n".join(generated_code_blocks)


		def generate_ext_independent_structs_list_resizing_logic(
		vulkan_api_version_mapping: List[dict[str, str]],
		vk_properties_setter_line: str,
		) -> str:
		vulkan_api_version_structures = {structure for mapping in vulkan_api_version_mapping for structure in
		mapping.keys()}
		filtered_structures = []
		for structure in VK.EXTENSION_INDEPENDENT_STRUCTS:
		if structure in vulkan_api_version_structures:
		filtered_structures.append(structure)

		generated_code_blocks = generate_list_resizing_logic(filtered_structures,
		vk_properties_setter_line,
		)
		return generated_code_blocks


		def generate_list_resizing_logic(
		structures: List[str],
		vk_properties_setter_line: str,
		extension: Optional[str] = None
		) -> str:
		def _generate_if_condition_codeblock(
		if_condition: str,
		list_resize_codeblock: str,
		properties_setter_line: Optional[str] = None,
		) -> str:
		vk_properties_setter_line_code = f"\n{properties_setter_line}" if properties_setter_line else ""
		return f"""
		if({if_condition}) {{
		{list_resize_codeblock}{vk_properties_setter_line_code}
		}}
		"""

		def _generate_merged_if_blocks(
		outer_if_conditions_list: List[str],
		internal_resize_assignment_logic: List[str],
		properties_setter_line: str,
		) -> str:
		formatted_conditions = [f"{cond} > 0" for cond in outer_if_conditions_list]
		combined_cpp_if_condition = " \|\| ".join(formatted_conditions)
		combined_internal_logic_body = "\n".join(internal_resize_assignment_logic)

		return f"""{_generate_if_condition_codeblock(combined_cpp_if_condition,
		combined_internal_logic_body,
		properties_setter_line)}"""

		generated_code_blocks: List[str] = []
		dot_extension = f".{extension}" if extension else ""
		for structure in structures:
		struct_category_mapping = VK.STRUCT_EXTENDS_MAPPING.get(structure)
		resolved_structure_name = get_class_name_from_alias(structure)
		is_valid_struct = struct_category_mapping and (
		CONST_PHYSICAL_DEVICE_PROPERTIES_2 in struct_category_mapping
		) and resolved_structure_name in VK.STRUCT_WITH_DYNAMIC_SIZE_LIST_MAPPING
		if is_valid_struct:
		dynamic_size_list_variables = VK.STRUCT_WITH_DYNAMIC_SIZE_LIST_MAPPING[resolved_structure_name]
		struct_instance_name = get_struct_name(structure)
		code_block_list = [] # list of pair(if condition logic, resizing code)
		for list_name in dynamic_size_list_variables:
		if list_name in VK.LIST_TYPE_FIELD_AND_SIZE_MAPPING:
		list_size_field_name = VK.LIST_TYPE_FIELD_AND_SIZE_MAPPING[list_name]
		condition_expression = f"device{dot_extension}.{struct_instance_name}.{list_size_field_name}"
		cpp_list_variable_name = transform_list_member_name(list_name)
		resize_code_block = f"""device{dot_extension}.{cpp_list_variable_name}.resize({condition_expression});
		device{dot_extension}.{struct_instance_name}.{list_name} = device{dot_extension}.{cpp_list_variable_name}.data();"""
		code_block_list.append((condition_expression, resize_code_block))

		list_resize_code_blocks: List[str] = []
		set_property_setter_line_inside = len(code_block_list) == 1
		for condition, resize_code_block in code_block_list:
		list_resize_code_blocks.append(
		_generate_if_condition_codeblock(
		f"{condition} > 0",
		resize_code_block,
		vk_properties_setter_line if set_property_setter_line_inside else None,
		)
		)

		if len(list_resize_code_blocks) == 1:
		generated_code_blocks.append(list_resize_code_blocks[0])

		elif len(list_resize_code_blocks) > 1:
		generated_code_blocks.append(
		_generate_merged_if_blocks(
		[condition_expression for condition_expression, _ in code_block_list],
		list_resize_code_blocks,
		vk_properties_setter_line,
		)
		)
		return "\n".join(generated_code_blocks)

		def find_contiguous_ranges(format_data):
		"""Finds contiguous ranges of format values from a list of enums for a given extension or Vulkan API version.

vulkan/scripts/vkjson_generator.py

+24 −2

Original line number	Diff line number	Diff line
		@@ -17,12 +17,13 @@
		"""Generates the vkjson files.
		"""
		import os

		import generator_common as gencom
		import vkjson_gen_util as util
		import vk as VK
		import importlib

		VK_PROPERTIES_SETTER_LINE = 'vkGetPhysicalDeviceProperties2(physical_device, &properties);'

		def re_import_vk():
		importlib.reload(VK)

		@@ -80,6 +81,12 @@ struct VkJsonDevice {
		for entry in struct_entries:
		f.write(entry + ";\n")

		# write list properties for memory allocation
		list_members = util.get_dynamic_size_list_member_from_structures(VK.EXTENSION_INDEPENDENT_STRUCTS)
		for list_type, list_variable_name in list_members:
		variable_entry = "std::vector<" + list_type + ">" + " " + list_variable_name + ";"
		f.write(f" {variable_entry}\n")

		f.write("""\
		std::vector<VkQueueFamilyProperties> queues;
		std::vector<VkExtensionProperties> extensions;
		@@ -798,6 +805,13 @@ bool VkJsonImageFormatPropertiesFromJson(const std::string& json,


		def gen_instance_cc():
		def write_list_resizing_codeblock(file, vk_version_mapping):
		code_block = util.generate_ext_independent_structs_list_resizing_logic(
		vk_version_mapping,
		VK_PROPERTIES_SETTER_LINE,
		)
		file.write(f"\n{code_block}\n")

		"""Generates vkjson_instance.cc file.
		"""
		genfile = os.path.join(os.path.dirname(__file__),
		@@ -876,10 +890,14 @@ VkJsonDevice VkJsonGetDevice(VkPhysicalDevice physical_device) {
		cc_code_properties = util.generate_vk_extension_structs_init_code(
		VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"], "Properties")
		f.write(f'{cc_code_properties}\n')
		f.write(f'{VK_PROPERTIES_SETTER_LINE}\n')

		extension_list_resize_logic = util.generate_ext_dependent_structs_list_resizing_logic(
		VK_PROPERTIES_SETTER_LINE,
		)
		f.write(f'\n{extension_list_resize_logic}\n')
		f.write("""\

		vkGetPhysicalDeviceProperties2(physical_device, &properties);
		device.properties = properties.properties;

		VkPhysicalDeviceFeatures2 features = {
		@@ -922,6 +940,7 @@ VkJsonDevice VkJsonGetDevice(VkPhysicalDevice physical_device) {

		f.write("""\
		vkGetPhysicalDeviceProperties2(physical_device, &properties);\n\n""")
		write_list_resizing_codeblock(f, vk_version_data)

		features_initialization_code = util.generate_vk_version_structs_initialization(vk_version_data, "Features")
		f.write(features_initialization_code)
		@@ -986,6 +1005,7 @@ VkJsonDevice VkJsonGetDevice(VkPhysicalDevice physical_device) {
		f.write(util.generate_vk_version_structs_initialization(vk_version_data, "Properties") + "\n")

		f.write(f"vkGetPhysicalDeviceProperties2(physical_device, &properties);\n\n")
		write_list_resizing_codeblock(f, vk_version_data)

		f.write(cc_code_features_11)
		f.write(cc_code_features_12)
		@@ -1006,6 +1026,7 @@ VkJsonDevice VkJsonGetDevice(VkPhysicalDevice physical_device) {
		f.write(util.generate_vk_version_structs_initialization(vk_version_data, "Properties") + "\n")
		f.write(cc_code_properties_13)
		f.write(f"vkGetPhysicalDeviceProperties2(physical_device, &properties);\n\n")
		write_list_resizing_codeblock(f, vk_version_data)

		f.write(cc_code_features_13)
		f.write(f"{util.generate_vk_version_structs_initialization(vk_version_data, "Features")}\n")
		@@ -1021,6 +1042,7 @@ VkJsonDevice VkJsonGetDevice(VkPhysicalDevice physical_device) {
		f.write(util.generate_vk_version_structs_initialization(vk_version_data, "Properties") + "\n")
		f.write(cc_code_properties_14)
		f.write(f"vkGetPhysicalDeviceProperties2(physical_device, &properties);\n\n")
		write_list_resizing_codeblock(f, vk_version_data)

		f.write("""\
		if (device.core14.properties.copySrcLayoutCount > 0 \|\| device.core14.properties.copyDstLayoutCount > 0 ) {