Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 290e014f authored by Martin Geisler's avatar Martin Geisler Committed by Gerrit Code Review
Browse files

Merge changes I787a4937,I29e32a13,I11542ff2,I3dc4c2d7,I88d7de98 

* changes:
  pdl: Set default-run for Cargo
  pdl: Re-enable rustc errors for generated code
  pdl: Use num-traits in canonical tests
  pdl: Generate canonical serialization tests
  pdl: Generate canonical parsing tests
parents 4f19db55 c64db040

tools/pdl/Android.bp

+107 −0
Original line number Diff line number Diff line
@@ -62,6 +62,113 @@ rust_test_host { 
    ],

}



genrule_defaults {

    name: "pdl_rust_generator_defaults",

    tools: [

        ":pdl",

        ":rustfmt",

    ],

}



// Generate the Rust parser+serializer backends.

genrule {

    name: "pdl_le_backend",

    defaults: ["pdl_rust_generator_defaults"],

    cmd: "$(location :pdl) --output-format rust $(in) | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/le_rust_test_file.pdl"],

    out: ["le_backend.rs"],

}



genrule {

    name: "pdl_be_backend",

    defaults: ["pdl_rust_generator_defaults"],

    cmd: "$(location :pdl) --output-format rust $(in) | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/be_rust_test_file.pdl"],

    out: ["be_backend.rs"],

}



rust_defaults {

    name: "pdl_backend_defaults",

    rustlibs: [

        "libbytes",

        "libnum_traits",

        "libtempfile",

        "libthiserror",

    ],

    proc_macros: ["libnum_derive"],

}



rust_library_host {

    name: "libpdl_le_backend",

    crate_name: "pdl_le_backend",

    srcs: [":pdl_le_backend"],

    defaults: ["pdl_backend_defaults"],

}



rust_library_host {

    name: "libpdl_be_backend",

    crate_name: "pdl_be_backend",

    srcs: [":pdl_be_backend"],

    defaults: ["pdl_backend_defaults"],

}



rust_binary_host {

    name: "pdl_generate_tests",

    srcs: ["src/bin/generate-canonical-tests.rs"],

    rustlibs: [

        "libproc_macro2",

        "libquote",

        "libserde",

        "libserde_json",

        "libsyn",

        "libtempfile",

    ],

}



genrule_defaults {

    name: "pdl_rust_generator_src_defaults",

    tools: [

        ":pdl_generate_tests",

        ":rustfmt",

    ],

}



genrule {

    name: "pdl_rust_generator_tests_le_src",

    cmd: "$(location :pdl_generate_tests) $(in) pdl_le_backend | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/le_test_vectors.json"],

    out: ["le_canonical.rs"],

    defaults: ["pdl_rust_generator_src_defaults"],

}



genrule {

    name: "pdl_rust_generator_tests_be_src",

    cmd: "$(location :pdl_generate_tests) $(in) pdl_be_backend | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/be_test_vectors.json"],

    out: ["be_canonical.rs"],

    defaults: ["pdl_rust_generator_src_defaults"],

}



rust_test_host {

    name: "pdl_rust_generator_tests_le",

    srcs: [":pdl_rust_generator_tests_le_src"],

    test_suites: ["general-tests"],

    rustlibs: [

        "libnum_traits",

        "libpdl_le_backend",

    ],

}



rust_test_host {

    name: "pdl_rust_generator_tests_be",

    srcs: [":pdl_rust_generator_tests_be_src"],

    test_suites: ["general-tests"],

    rustlibs: [

        "libnum_traits",

        "libpdl_be_backend",

    ],

}



// Defaults for PDL python backend generation.

genrule_defaults {

    name: "pdl_python_generator_defaults",

tools/pdl/Cargo.toml

+5 −0
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 
name = "pdl"

version = "0.1.0"

edition = "2021"

default-run = "pdl"



[workspace]
@@ -18,3 +19,7 @@ syn = "1.0.102" 


[dev-dependencies]

tempfile = "3.3.0"

bytes = "1.2.1"

num-derive = "0.3.3"

num-traits = "0.2.15"

thiserror = "1.0.37"

tools/pdl/src/backends/rust/preamble.rs

+3 −0
Original line number Diff line number Diff line
@@ -8,6 +8,9 @@ pub fn generate(path: &Path) -> String { 
    let filename = path.file_name().unwrap().to_str().expect("non UTF-8 filename");

    code.push_str(&format!("// @generated rust packets from {filename}\n\n"));



    // TODO(mgeisler): make the generated code clean from warnings.

    code.push_str("#![allow(warnings, missing_docs)]\n\n");



    code.push_str(&quote_block! {

        use bytes::{BufMut, Bytes, BytesMut};

        use num_derive::{FromPrimitive, ToPrimitive};

tools/pdl/src/bin/generate-canonical-tests.rs

0 → 100644
+139 −0
Original line number Diff line number Diff line 
//! Generate Rust unit tests for canonical test vectors.



use quote::{format_ident, quote};

use serde::Deserialize;

use serde_json::Value;



#[derive(Debug, Deserialize)]

struct Packet {

    #[serde(rename = "packet")]

    name: String,

    tests: Vec<TestVector>,

}



#[derive(Debug, Deserialize)]

struct TestVector {

    packed: String,

    unpacked: Value,

    packet: Option<String>,

}



// Convert a string of hexadecimal characters into a Rust vector of

// bytes.

//

// The string `"80038302"` becomes `vec![0x80, 0x03, 0x83, 0x02]`.

fn hexadecimal_to_vec(hex: &str) -> proc_macro2::TokenStream {

    assert!(hex.len() % 2 == 0, "Expects an even number of hex digits");

    let bytes = hex.as_bytes().chunks_exact(2).map(|chunk| {

        let number = format!("0x{}", std::str::from_utf8(chunk).unwrap());

        syn::parse_str::<syn::LitInt>(&number).unwrap()

    });



    quote! {

        vec![#(#bytes),*]

    }

}



fn generate_unit_tests(input: &str, packet_names: &[&str], module_name: &str) {

    eprintln!("Reading test vectors from {input}, will use {} packets", packet_names.len());



    let data = std::fs::read_to_string(input)

        .unwrap_or_else(|err| panic!("Could not read {input}: {err}"));

    let packets: Vec<Packet> = serde_json::from_str(&data).expect("Could not parse JSON");



    let module = format_ident!("{}", module_name);

    let mut tests = Vec::new();

    for packet in &packets {

        for (i, test_vector) in packet.tests.iter().enumerate() {

            let test_packet = test_vector.packet.as_deref().unwrap_or(packet.name.as_str());

            if !packet_names.contains(&test_packet) {

                eprintln!("Skipping packet {}", test_packet);

                continue;

            }

            let parse_test_name = format_ident!(

                "test_parse_{}_vector_{}_0x{}",

                test_packet,

                i + 1,

                &test_vector.packed

            );

            let serialize_test_name = format_ident!(

                "test_serialize_{}_vector_{}_0x{}",

                test_packet,

                i + 1,

                &test_vector.packed

            );

            let packed = hexadecimal_to_vec(&test_vector.packed);

            let packet_name = format_ident!("{}Packet", test_packet);

            let builder_name = format_ident!("{}Builder", test_packet);



            let object = test_vector.unpacked.as_object().unwrap_or_else(|| {

                panic!("Expected test vector object, found: {}", test_vector.unpacked)

            });

            let assertions = object.iter().map(|(key, value)| {

                let getter = format_ident!("get_{key}");

                let value_u64 = value

                    .as_u64()

                    .unwrap_or_else(|| panic!("Expected u64 for {key:?} key, got {value}"));

                let value = proc_macro2::Literal::u64_unsuffixed(value_u64);

                // We lack type information, but ToPrimitive allows us

                // to convert both integers and enums to u64.

                quote! {

                    assert_eq!(actual.#getter().to_u64().unwrap(), #value);

                }

            });



            let builder_fields = object.iter().map(|(key, value)| {

                let field = format_ident!("{key}");

                let value_u64 = value

                    .as_u64()

                    .unwrap_or_else(|| panic!("Expected u64 for {key:?} key, got {value}"));

                let value = proc_macro2::Literal::u64_unsuffixed(value_u64);

                // We lack type information, but FromPrimitive allows

                // us to convert both integers and enums to u64.

                quote! {

                    #field: FromPrimitive::from_u64(#value).unwrap()

                }

            });



            tests.push(quote! {

                #[test]

                fn #parse_test_name() {

                    let packed = #packed;

                    let actual = #module::#packet_name::parse(&packed).unwrap();

                    #(#assertions)*

                }



                #[test]

                fn #serialize_test_name() {

                    let builder =  #module::#builder_name {

                        #(#builder_fields,)*

                    };

                    let packet = builder.build();

                    let packed = #packed;

                    assert_eq!(packet.to_vec(), packed);

                }

            });

        }

    }



    // TODO(mgeisler): make the generated code clean from warnings.

    println!("#![allow(warnings, missing_docs)]");

    println!();

    println!(

        "{}",

        &quote! {

            use #module::Packet;

            use num_traits::{FromPrimitive, ToPrimitive};



            #(#tests)*

        }

    );

}



fn main() {

    let input_path = std::env::args().nth(1).expect("Need path to JSON file with test vectors");

    let module_name = std::env::args().nth(2).expect("Need name for the generated module");

    // TODO(mgeisler): remove the `packet_names` argument when we

    // support all canonical packets.

    generate_unit_tests(&input_path, &["Packet_Scalar_Field"], &module_name);

}

tools/pdl/tests/canonical/be_rust_test_file.pdl

0 → 100644
+10 −0
Original line number Diff line number Diff line 
big_endian_packets



// Packet bit fields



// The parser must be able to handle bit fields with scalar values

// up to 64 bits wide.  The parser should generate a static size guard.

packet Packet_Scalar_Field {

    a: 7,

    c: 57,

}