Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 904accc9 authored by mridulgoyal's avatar mridulgoyal Committed by Mridul Goyal
Browse files

CodeGen(vk_parser): Refine vk.xml parsing and vk.py generation for vkjson 

Updates vk_parser.py for more precise vk.xml parsing and improved vk.py
output. The generated vk.py serves as an intermediate for
vkjson_generator.py, aiding generation of vkjsons.

Key changes:
- Integrated new structs and extensions into vkjson.
- Updated naming logic in vkjson_generator.py for consistency.
- Extended vkjson_generator to handle new vkjson structures and extensions comprehensively.
- Added detailed comments to vk.py explaining the generation filters for key mappings, improving clarity.
- Improved the output formatting and overall structure of vk.py for better readability.
- Applied generation filters in vk.py to include `VkPhysicalDevice`-prefixed structs (specifically those extending `Properties2` or `Features2` for mapping generation, along with their dependencies) and extensions marked as 'supported' (or without a platform specification, or targeting 'android').
-Added code generation for fetching VKFormat from extensions and vulkan
versions tags in vk.xml
- Added 'VK_USE_PLATFORM_ANDROID_KHR' to vkjson.h for new structures defined in 'vulkan_android.h' header file (like VkPhysicalDeviceExternalFormatResolveFeaturesANDROID).

TODO:
- b/415704124 [CodeGen] Pending Codegen Improvements

BYPASS_LARGE_CHANGE_WARNING

Test: adb shell cmd gpu vkjson
Flag: NONE infeasible
Bug: b/401184058
Change-Id: I0692e601a51fa8eaa04a30afcc3330bedeb1a833
parent 94ce01a7

vulkan/scripts/code_generator.py

+4 −1
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import driver_generator 
import generator_common

import null_generator

import vkjson_generator

import vk_parser



if __name__ == '__main__':

  generator_common.parse_vulkan_registry()
@@ -31,6 +32,8 @@ if __name__ == '__main__': 
  driver_generator.gen_cpp()

  null_generator.gen_h()

  null_generator.gen_cpp()

  vk_parser.gen_vk()

  vkjson_generator.re_import_vk()

  vkjson_generator.gen_h()

  vkjson_generator.gen_cc()

  vkjson_generator.gen_instance_cc()
 
 No newline at end of file

vulkan/scripts/vk_parser.py

0 → 100644
+1420 −0

File added.

Preview size limit exceeded, changes collapsed.

vulkan/scripts/vkjson_gen_util.py

+177 −74
Original line number Diff line number Diff line
@@ -23,6 +23,9 @@ import re 
import vk as VK

from typing import get_origin



CONST_PHYSICAL_DEVICE_FEATURE_2 = "VkPhysicalDeviceFeatures2"

CONST_PHYSICAL_DEVICE_PROPERTIES_2 = "VkPhysicalDeviceProperties2"



COPYRIGHT_WARNINGS = """///////////////////////////////////////////////////////////////////////////////

//

// Copyright (c) 2015-2016 The Khronos Group Inc.
@@ -44,7 +47,6 @@ COPYRIGHT_WARNINGS = """//////////////////////////////////////////////////////// 
///////////////////////////////////////////////////////////////////////////////

"""





def get_copyright_warnings():

    return COPYRIGHT_WARNINGS
@@ -56,7 +58,18 @@ def get_vkjson_struct_name(extension_name): 
    prefix_map = {

        "VK_KHR": "VkJsonKHR",

        "VK_EXT": "VkJsonExt",

        "VK_IMG": "VkJsonIMG"

        "VK_IMG": "VkJsonIMG",

        "VK_ANDROID": "VkJsonANDROID",

        "VK_AMD": "VkJsonAMD",

        "VK_MESA": "VkJsonMESA",

        "VK_NV": "VkJsonNV",

        "VK_QCOM": "VkJsonQCOM",

        "VK_ARM": "VkJsonARM",

        "VK_HUAWEI": "VkJsonHUAWEI",

        "VK_VALVE": "VkJsonVALVE",

        "VK_MSFT": "VkJsonMSFT",

        "VK_SEC": "VkJsonSEC",

        "VK_NVX": "VKJsonNVX",

    }



    for prefix, replacement in prefix_map.items():
@@ -80,45 +93,95 @@ def get_vkjson_struct_variable_name(extension_name): 
    prefix_map = {

        "VK_KHR_": "khr_",

        "VK_EXT_": "ext_",

        "VK_IMG_": "img_"

        "VK_IMG_": "img_",

        "VK_ANDROID_": "android_",

        "VK_AMD_": "amd_",

        "VK_MESA_": "mesa_",

        "VK_NV_": "nv_",

        "VK_QCOM_": "qcom_",

        "VK_ARM_": "arm_",

        "VK_HUAWEI_": "huawei_",

        "VK_VALVE_": "valve_",

        "VK_MSFT_": "msft_",

        "VK_SEC_": "sec_",

        "VK_NVX_": "nvx_",

    }



    for prefix, replacement in prefix_map.items():

        if extension_name.startswith(prefix):

            return replacement + extension_name[len(prefix):]



            # Match: word1_digits_word2 => word1_word2_digits (Example: ext_4444_formats -> ext_formats_4444)

            return re.sub(r"^([a-zA-Z]+)_(\d+)_([a-zA-Z]+)", r"\1_\3_\2", replacement + extension_name[len(prefix) :])

    return extension_name.lower()  # Default case if no known prefix matches





def normalize_variable_name(name, base_name):

    # Convert CamelCase to snake_case

    # Example: "ShaderFloat16Int8Features" → "shader_float16_int8_features"



    # Add underscore around digit+D (e.g., 2D, 3D)

    base_name = re.sub(r"(\d)(D)", r"_\1\2_", base_name)



    # Add underscore before any uppercase letter that follows a lowercase letter or digit

    initialString = re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", base_name)



    # Add underscore between consecutive uppercase letters followed by a lowercase letter

    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", initialString).lower()



    # Replace X_d_ → Xd_

    name = re.sub(r"(\d+)_d_", r"\1d_", name)



    # Replace i_d_ or similar → id_

    name = re.sub(r"\bi_d_", "id_", name)



    # Replace X_bit_ → bitX_

    name = re.sub(r"(\d+)_bit_", r"bit\1_", name)



    # Replace 9_XX → XX_9

    if re.match(r"^\d", name):

        name = re.sub(r"^(\d+)_([a-zA-Z]+)", r"\2_\1", name)



    # Prefix with underscore if starting with digit

    if re.match(r"^\d", name):

        name = "_" + name

    return name





def get_struct_name(struct_name):

    """Gets corresponding instance name

    Example: "VkPhysicalDeviceShaderFloat16Int8FeaturesKHR" → "shader_float16_int8_features_khr"

    """

    # Remove "VkPhysicalDevice" prefix and any of the known suffixes

    base_name = (struct_name.removeprefix("VkPhysicalDevice").removesuffix("KHR")

                 .removesuffix("EXT").removesuffix("IMG"))



    # Convert CamelCase to snake_case

    # Example: "ShaderFloat16Int8Features" → "shader_float16_int8_features"

    variable_name = re.sub(r"(?<!^)(?=[A-Z])", "_", base_name).lower()

    prefixes_suffixes_to_remove = ("VkPhysicalDevice", "KHR", "EXT", "IMG", "ANDROID", "AMD", "MESA", "NV", "QCOM", "ARM", "HUAWEI", "VALVE", "MSFT")

    base_name = struct_name



    for affix in prefixes_suffixes_to_remove:

        base_name = base_name.removeprefix(affix).removesuffix(affix)

    variable_name = ""

    # Fix special cases

    variable_name = variable_name.replace("2_d_", "_2d_").replace("3_d_", "_3d_")

    variable_name = normalize_variable_name(variable_name, base_name)



    # Add back the correct suffix if it was removed

    suffix_map = {"KHR": "_khr", "EXT": "_ext", "IMG": "_img"}

    suffix_map = {

        "KHR": "_khr",

        "EXT": "_ext",

        "IMG": "_img",

        "ANDROID": "_android",

        "AMD": "_amd",

        "NV": "_nv",

        "MESA": "_mesa",

        "VALVE": "_valve",

        "MSFT": "_msft",

        "QCOM": "_qcom",

        "ARM": "_arm",

        "HUAWEI": "_huawei",

    }

    for suffix, replacement in suffix_map.items():

        if struct_name.endswith(suffix):

            variable_name += replacement

            break



    # Handle specific exceptions

    special_cases = {

        "8_bit_storage_features_khr": "bit8_storage_features_khr",

        "memory_properties": "memory",

        "16_bit_storage_features": "bit16_storage_features",

        "i_d_properties": "id_properties"

    }

    special_cases = {"memory_properties": "memory", "ycbcr2_plane444_formats_features_ext": "ycbcr_2plane_444_formats_features_ext"}



    return special_cases.get(variable_name, variable_name)
@@ -138,8 +201,7 @@ def generate_extension_struct_definition(f): 
    """

    vkjson_entries = []



    for extension_name, struct_list in (VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"]

            .items()):

    for extension_name, struct_list in VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"].items():

        vkjson_struct_name = get_vkjson_struct_name(extension_name)

        vkjson_struct_variable_name = get_vkjson_struct_variable_name(extension_name)

        vkjson_entries.append(f"{vkjson_struct_name} {vkjson_struct_variable_name}")
@@ -158,7 +220,6 @@ def generate_extension_struct_definition(f): 


        f.write("  }\n")  # End of constructor

        f.write("  bool reported;\n")



        for entry in struct_entries:

            f.write(f"  {entry};\n")
@@ -193,7 +254,6 @@ def generate_vk_core_struct_definition(f): 
            for struct_type, _ in item.items():

                field_name = "properties" if "Properties" in struct_type else "features"

                f.write(f"  {struct_type} {field_name};\n")



        if version == "Core14":

            f.write(f"std::vector<VkImageLayout> copy_src_layouts;\n")

            f.write(f"std::vector<VkImageLayout> copy_dst_layouts;\n")
@@ -212,12 +272,18 @@ def generate_memset_statements(f): 
      VkPhysicalDeviceProperties properties;

    """

    entries = []



    processed_entries = set()

    all_extension_independent_structs = VK.EXTENSION_INDEPENDENT_STRUCTS + VK.ADDITIONAL_EXTENSION_INDEPENDENT_STRUCTS

    # Process independent structs

    for dataclass_type in VK.EXTENSION_INDEPENDENT_STRUCTS:

        class_name = dataclass_type.__name__

    for dataclass_type in all_extension_independent_structs:

        class_name = dataclass_type

        variable_name = get_struct_name(class_name)

        # EDIT: updated as we are now taking the dataclass_type as 'string'

        key_str = f"{variable_name}@{class_name}"

        if key_str in processed_entries:

            continue

        f.write(f"memset(&{variable_name}, 0, sizeof({class_name}));\n")

        processed_entries.add(key_str)

        entries.append(f"{class_name} {variable_name}")



    return entries
@@ -236,8 +302,7 @@ def generate_extension_struct_template(): 
    """

    template_code = []



    for extension, struct_mappings in (VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"]

            .items()):

    for extension, struct_mappings in VK.VULKAN_EXTENSIONS_AND_STRUCTS_MAPPING["extensions"].items():

        struct_type = get_vkjson_struct_name(extension)



        template_code.append(f"template <typename Visitor>")
@@ -250,13 +315,12 @@ def generate_extension_struct_template(): 
                json_field_name = struct_name.replace("VkPhysicalDevice", "")

                json_field_name = json_field_name[0].lower() + json_field_name[1:]



                # Special case renaming

                if json_field_name == "8BitStorageFeaturesKHR":

                    json_field_name = "bit8StorageFeaturesKHR"

                # Move the leading digits after the first capitalized word

                json_field_name = re.sub(r"^(\d+)([A-Z][a-z]*)", r"\2\1", json_field_name)

                # Lowercase the first character

                json_field_name = json_field_name[0].lower() + json_field_name[1:] if json_field_name else json_field_name



                visitor_calls.append(

                    f'visitor->Visit("{json_field_name}", &structs->{get_struct_name(struct_name)})'

                )

                visitor_calls.append(f'visitor->Visit("{json_field_name}", &structs->{get_struct_name(struct_name)})')



        template_code.append(" &&\n         ".join(visitor_calls) + ";")

        template_code.append("}\n")
@@ -313,7 +377,6 @@ def generate_struct_template(data_classes): 
        processed_classes.add(struct_name)



        struct_fields = dataclasses.fields(dataclass_type)

        template_code.append("template <typename Visitor>")



        # Determine the correct variable name based on the struct type

        struct_var = "properties" if "Properties" in struct_name else "features" if "Features" in struct_name else "limits" if "Limits" in struct_name else None
@@ -321,8 +384,8 @@ def generate_struct_template(data_classes): 
        if not struct_var:

            continue  # Skip structs that don't match expected patterns



        template_code.append(

            f"inline bool Iterate(Visitor* visitor, {struct_name}* {struct_var}) {{")

        template_code.append("template <typename Visitor>")

        template_code.append(f"inline bool Iterate(Visitor* visitor, {struct_name}* {struct_var}) {{")

        template_code.append(f"return\n")



        visitor_calls = []
@@ -333,8 +396,7 @@ def generate_struct_template(data_classes): 
            if get_origin(field_type) is list:

                # Handle list types (VisitArray)

                size_field_name = VK.LIST_TYPE_FIELD_AND_SIZE_MAPPING[field_name]

                visitor_calls.append(

                    f'visitor->VisitArray("{field_name}", {struct_var}->{size_field_name}, &{struct_var}->{field_name})')

                visitor_calls.append(f'visitor->VisitArray("{field_name}", {struct_var}->{size_field_name}, &{struct_var}->{field_name})')

            else:

                # Handle other types (Visit)

                visitor_calls.append(f'visitor->Visit("{field_name}", &{struct_var}->{field_name})')
@@ -346,8 +408,7 @@ def generate_struct_template(data_classes): 




def emit_struct_visits_by_vk_version(f, version):

    """Emits visitor calls for Vulkan version structs

    """

    """Emits visitor calls for Vulkan version structs"""

    for struct_map in VK.VULKAN_VERSIONS_AND_STRUCTS_MAPPING[version]:

        for struct_name, _ in struct_map.items():

            struct_var = get_struct_name(struct_name)
@@ -357,6 +418,31 @@ def emit_struct_visits_by_vk_version(f, version): 
            f.write(f'visitor->Visit("{struct_display_name}", &device->{struct_var}) &&\n')





def generate_enum_traits():

    """Writes necessary enum definitions as Python Enum classes and generates C++ traits string."""

    cpp_enum_traits_content = ""

    # Iterate through all parsed enum types

    for enum_name, members in VK.ENUM_TRAITS_MAPPING.items():

        cpp_enum_traits_content += "\n"

        # --- C++ EnumTraits generation (skip for these enums as they are differently defined in xml) ---

        if enum_name == "VkImageLayout":

            continue

        cpp_enum_traits_content += f"template <>\nstruct EnumTraits<{enum_name}> {{\n"

        cpp_enum_traits_content += f"  static bool exist(uint32_t e) {{\n"

        cpp_enum_traits_content += f"    switch (e) {{\n"

        if members:

            for member, value in members.items():

                if value is not None and member:

                    cpp_enum_traits_content += f"      case {member}:\n"

        cpp_enum_traits_content += f"        return true;\n"

        cpp_enum_traits_content += f"    }}\n"

        cpp_enum_traits_content += f"    return false;\n"

        cpp_enum_traits_content += f"  }}\n"

        cpp_enum_traits_content += f"}};\n\n"



    return cpp_enum_traits_content





def generate_vk_core_structs_init_code(version):

    """Generates code to initialize properties and features

    for structs based on its vulkan API version dependency.
@@ -366,48 +452,66 @@ def generate_vk_core_structs_init_code(version): 
    for item in VK.VULKAN_CORES_AND_STRUCTS_MAPPING["versions"].get(version, []):

        for struct_name, struct_type in item.items():

            version_lower = version.lower()



            if "Properties" in struct_name:

                properties_code.extend([

                    f"device.{version_lower}.properties.sType = {struct_type};",

                    f"device.{version_lower}.properties.pNext = properties.pNext;",

                    f"properties.pNext = &device.{version_lower}.properties;\n\n"

                ])



            elif "Features" in struct_name:

                features_code.extend([

                    f"device.{version_lower}.features.sType = {struct_type};",

                    f"device.{version_lower}.features.pNext = features.pNext;",

                    f"features.pNext = &device.{version_lower}.features;\n\n"

                ])

            target_code_list = None

            struct_field_name = None

            mapping_name = VK.STRUCT_EXTENDS_MAPPING.get(struct_name)

            if mapping_name and CONST_PHYSICAL_DEVICE_FEATURE_2 in mapping_name:

                target_code_list = features_code

                struct_field_name = "features"

            elif mapping_name and CONST_PHYSICAL_DEVICE_PROPERTIES_2 in mapping_name:

                target_code_list = properties_code

                struct_field_name = "properties"

            else:

                # TODO: b/415707715 (Add tests for exceptions for structs not extending (VkPhysicalDeviceProperties2/VkPhysicalDeviceFeatures2))

                raise Exception(f"Warning: Unknown Mapping: ' " f"for struct '{struct_name}'")



            if target_code_list is not None:

                target_code_list.extend(

                    [

                        f"device.{version_lower}.{struct_field_name}.sType = {struct_type};",

                        f"device.{version_lower}.{struct_field_name}.pNext = {struct_field_name}.pNext;",

                        f"{struct_field_name}.pNext = &device.{version_lower}.{struct_field_name};\n\n",

                    ]

                )



    return "\n".join(properties_code), "\n".join(features_code)





def generate_vk_extension_structs_init_code(mapping, struct_category, next_pointer):

def generate_vk_extension_structs_init_code(mapping, struct_category):

    """Generates Vulkan struct initialization code for given struct category (Properties/Features)

    based on its extension dependency.

    """

    generated_code = []

    next_pointer = struct_category.lower()



    for extension, struct_mappings in mapping.items():

        struct_var_name = get_vkjson_struct_variable_name(extension)

        extension_code = [

            f"  if (HasExtension(\"{extension}\", device.extensions)) {{",

            f"    device.{struct_var_name}.reported = true;"

        ]



        extension_code = [f'  if (HasExtension("{extension}", device.extensions)) {{', f"    device.{struct_var_name}.reported = true;"]

        struct_count = 0

        for struct_mapping in struct_mappings:

            for struct_name, struct_type in struct_mapping.items():

                if struct_category in struct_name:

                # EDIT: Category detection improved using structextends mapping (VK.STRUCT_EXTENDS_MAPPING).

                # Correctly handles structs with ambiguous names (e.g., VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR)

                # that might contain 'Properties'/'Features' in name but extend only one (Features2/Properties2).

                mapping_name = VK.STRUCT_EXTENDS_MAPPING.get(struct_name)

                is_valid_struct_category = False

                if mapping_name:

                    is_valid_struct_category = (CONST_PHYSICAL_DEVICE_PROPERTIES_2 in mapping_name and struct_category == "Properties") or (

                        CONST_PHYSICAL_DEVICE_FEATURE_2 in mapping_name and struct_category == "Features"

                    )

                elif struct_category.lower() in struct_name.lower():

                    raise Exception(f"Warning: Unknown Mapping: ' " f"for struct '{struct_name}'")



                if is_valid_struct_category:

                    struct_count += 1

                    struct_instance = get_struct_name(struct_name)

                    extension_code.extend([

                    extension_code.extend(

                        [

                            f"    device.{struct_var_name}.{struct_instance}.sType = {struct_type};",

                            f"    device.{struct_var_name}.{struct_instance}.pNext = {next_pointer}.pNext;",

                        f"    {next_pointer}.pNext = &device.{struct_var_name}.{struct_instance};"

                    ])

                            f"    {next_pointer}.pNext = &device.{struct_var_name}.{struct_instance};",

                        ]

                    )



        extension_code.append("  }\n")
@@ -417,20 +521,19 @@ def generate_vk_extension_structs_init_code(mapping, struct_category, next_point 
    return "\n".join(generated_code)





def generate_vk_version_structs_initialization(version_data, struct_type_keyword, next_pointer):

def generate_vk_version_structs_initialization(version_data, struct_type_keyword):

    """Generates Vulkan struct initialization code for given struct category (Properties/Features)

    of vulkan api version s.

    """

    struct_initialization_code = []

    next_pointer = struct_type_keyword.lower()



    for struct_mapping in version_data:

        for struct_name, struct_type in struct_mapping.items():

            if struct_type_keyword in struct_name:

                struct_variable = get_struct_name(struct_name)

                struct_initialization_code.extend([

                    f"device.{struct_variable}.sType = {struct_type};",

                    f"device.{struct_variable}.pNext = {next_pointer}.pNext;",

                    f"{next_pointer}.pNext = &device.{struct_variable};\n"

                ])

                struct_initialization_code.extend(

                    [f"device.{struct_variable}.sType = {struct_type};", f"device.{struct_variable}.pNext = {next_pointer}.pNext;", f"{next_pointer}.pNext = &device.{struct_variable};\n"]

                )



    return "\n".join(struct_initialization_code)

vulkan/scripts/vkjson_generator.py

+94 −403

File changed.

Preview size limit exceeded, changes collapsed.