Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6863b993 authored by Martin Geisler's avatar Martin Geisler
Browse files

pdl: Generate canonical parsing tests 

The new binary in src/bin/generate-canonical-tests.rs will take a JSON
file with canonical test vectors (from tests/canonical/) and produce a
Rust file with unit tests for each test vector.

This allows us to verify the full end-to-end functionality of the
generated Rust code: we check that it compiles and that it gives the
correct values. This change only contains the parsing tests, we will
test serialization next.

Test: atest pdl_rust_generator_tests_{le,be}
Change-Id: I88d7de98e7a11b6c8d2ceffadc4d3282db638a14
parent 66bca36b

tools/pdl/Android.bp

+107 −0
Original line number Diff line number Diff line
@@ -62,6 +62,113 @@ rust_test_host { 
    ],

}



genrule_defaults {

    name: "pdl_rust_generator_defaults",

    tools: [

        ":pdl",

        ":rustfmt",

    ],

}



// Generate the Rust parser+serializer backends.

genrule {

    name: "pdl_le_backend",

    defaults: ["pdl_rust_generator_defaults"],

    cmd: "$(location :pdl) --output-format rust $(in) | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/le_rust_test_file.pdl"],

    out: ["le_backend.rs"],

}



genrule {

    name: "pdl_be_backend",

    defaults: ["pdl_rust_generator_defaults"],

    cmd: "$(location :pdl) --output-format rust $(in) | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/be_rust_test_file.pdl"],

    out: ["be_backend.rs"],

}



rust_defaults {

    name: "pdl_backend_defaults",

    rustlibs: [

        "libbytes",

        "libnum_traits",

        "libtempfile",

        "libthiserror",

    ],

    proc_macros: ["libnum_derive"],

    clippy_lints: "none",

    lints: "none",

}



rust_library_host {

    name: "libpdl_le_backend",

    crate_name: "pdl_le_backend",

    srcs: [":pdl_le_backend"],

    defaults: ["pdl_backend_defaults"],

}



rust_library_host {

    name: "libpdl_be_backend",

    crate_name: "pdl_be_backend",

    srcs: [":pdl_be_backend"],

    defaults: ["pdl_backend_defaults"],

}



rust_binary_host {

    name: "pdl_generate_tests",

    srcs: ["src/bin/generate-canonical-tests.rs"],

    rustlibs: [

        "libproc_macro2",

        "libquote",

        "libserde",

        "libserde_json",

        "libsyn",

        "libtempfile",

    ],

}



genrule_defaults {

    name: "pdl_rust_generator_src_defaults",

    tools: [

        ":pdl_generate_tests",

        ":rustfmt",

    ],

}



genrule {

    name: "pdl_rust_generator_tests_le_src",

    cmd: "$(location :pdl_generate_tests) $(in) pdl_le_backend | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/le_test_vectors.json"],

    out: ["le_canonical.rs"],

    defaults: ["pdl_rust_generator_src_defaults"],

}



genrule {

    name: "pdl_rust_generator_tests_be_src",

    cmd: "$(location :pdl_generate_tests) $(in) pdl_be_backend | $(location :rustfmt) > $(out)",

    srcs: ["tests/canonical/be_test_vectors.json"],

    out: ["be_canonical.rs"],

    defaults: ["pdl_rust_generator_src_defaults"],

}



rust_test_host {

    name: "pdl_rust_generator_tests_le",

    srcs: [":pdl_rust_generator_tests_le_src"],

    test_suites: ["general-tests"],

    rustlibs: ["libpdl_le_backend"],

    clippy_lints: "none",

    lints: "none",

}



rust_test_host {

    name: "pdl_rust_generator_tests_be",

    srcs: [":pdl_rust_generator_tests_be_src"],

    test_suites: ["general-tests"],

    rustlibs: ["libpdl_be_backend"],

    clippy_lints: "none",

    lints: "none",

}



// Defaults for PDL python backend generation.

genrule_defaults {

    name: "pdl_python_generator_defaults",

tools/pdl/Cargo.toml

+4 −0
Original line number Diff line number Diff line
@@ -18,3 +18,7 @@ syn = "1.0.102" 


[dev-dependencies]

tempfile = "3.3.0"

bytes = "1.2.1"

num-derive = "0.3.3"

num-traits = "0.2.15"

thiserror = "1.0.37"

tools/pdl/src/bin/generate-canonical-tests.rs

0 → 100644
+95 −0
Original line number Diff line number Diff line 
//! Generate Rust unit tests for canonical test vectors.



use quote::{format_ident, quote};

use serde::Deserialize;

use serde_json::Value;



#[derive(Debug, Deserialize)]

struct Packet {

    #[serde(rename = "packet")]

    name: String,

    tests: Vec<TestVector>,

}



#[derive(Debug, Deserialize)]

struct TestVector {

    packed: String,

    unpacked: Value,

    packet: Option<String>,

}



// Convert a string of hexadecimal characters into a Rust vector of

// bytes.

//

// The string `"80038302"` becomes `vec![0x80, 0x03, 0x83, 0x02]`.

fn hexadecimal_to_vec(hex: &str) -> proc_macro2::TokenStream {

    assert!(hex.len() % 2 == 0, "Expects an even number of hex digits");

    let bytes = hex.as_bytes().chunks_exact(2).map(|chunk| {

        let number = format!("0x{}", std::str::from_utf8(chunk).unwrap());

        syn::parse_str::<syn::LitInt>(&number).unwrap()

    });



    quote! {

        vec![#(#bytes),*]

    }

}



fn generate_unit_tests(input: &str, packet_names: &[&str], module_name: &str) {

    eprintln!("Reading test vectors from {input}, will use {} packets", packet_names.len());



    let data = std::fs::read_to_string(input)

        .unwrap_or_else(|err| panic!("Could not read {input}: {err}"));

    let packets: Vec<Packet> = serde_json::from_str(&data).expect("Could not parse JSON");



    let mut tests = Vec::new();

    for packet in &packets {

        for (i, test_vector) in packet.tests.iter().enumerate() {

            let packet_name = test_vector.packet.as_deref().unwrap_or(packet.name.as_str());

            if !packet_names.contains(&packet_name) {

                eprintln!("Skipping packet {}", packet_name);

                continue;

            }

            let test_name =

                format_ident!("{}_vector_{}_0x{}", packet_name, i + 1, &test_vector.packed);

            let packed = hexadecimal_to_vec(&test_vector.packed);

            let packet_name = format_ident!("{}Packet", packet_name);



            let object = test_vector.unpacked.as_object().unwrap_or_else(|| {

                panic!("Expected test vector object, found: {}", test_vector.unpacked)

            });

            let assertions = object.iter().map(|(key, value)| {

                let getter = format_ident!("get_{key}");

                let value_u64 = value

                    .as_u64()

                    .unwrap_or_else(|| panic!("Expected u64 for {key:?} key, got {value}"));

                let value = proc_macro2::Literal::u64_unsuffixed(value_u64);

                quote! {

                    assert_eq!(actual.#getter(), #value);

                }

            });



            let module = format_ident!("{}", module_name);

            tests.push(quote! {

                #[test]

                fn #test_name() {

                    let packed = #packed;

                    let actual = #module::#packet_name::parse(&packed).unwrap();

                    #(#assertions)*

                }

            });

        }

    }



    let code = quote! {

        #(#tests)*

    };

    println!("{code}");

}



fn main() {

    let input_path = std::env::args().nth(1).expect("Need path to JSON file with test vectors");

    let module_name = std::env::args().nth(2).expect("Need name for the generated module");

    // TODO(mgeisler): remove the `packet_names` argument when we

    // support all canonical packets.

    generate_unit_tests(&input_path, &["Packet_Scalar_Field"], &module_name);

}

tools/pdl/tests/canonical/be_rust_test_file.pdl

0 → 100644
+10 −0
Original line number Diff line number Diff line 
big_endian_packets



// Packet bit fields



// The parser must be able to handle bit fields with scalar values

// up to 64 bits wide.  The parser should generate a static size guard.

packet Packet_Scalar_Field {

    a: 7,

    c: 57,

}

tools/pdl/tests/canonical/le_rust_test_file.pdl

0 → 100644
+10 −0
Original line number Diff line number Diff line 
little_endian_packets



// Packet bit fields



// The parser must be able to handle bit fields with scalar values

// up to 64 bits wide.  The parser should generate a static size guard.

packet Packet_Scalar_Field {

    a: 7,

    c: 57,

}