Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1da0880b authored by Henri Chataing's avatar Henri Chataing
Browse files

pdl: Use the analyzer ast for the default rust backend 

The rust backend previously used the parser ast for the generation,
along with the legacy linter scope. This change migrates the generator
code to the analyzer ast while still using the legacy linter scope.

This will enable further code removal in the legacy linter as
the analyzer already inlines the group declarations.

Test: cargo test
Change-Id: Ic205b6a1452cd15d2dfd50c578c751a0245ee63f
parent 0a285c00

tools/pdl/src/ast.rs

+0 −82
Original line number Diff line number Diff line 
use crate::lint;

use codespan_reporting::diagnostic;

use codespan_reporting::files;

use serde::Serialize;
@@ -384,31 +383,6 @@ impl<A: Annotation> Decl<A> { 
        }

    }



    /// Determine the size of a declaration type in bits, if possible.

    ///

    /// If the type is dynamically sized (e.g. contains an array or

    /// payload), `None` is returned. If `skip_payload` is set,

    /// payload and body fields are counted as having size `0` rather

    /// than a variable size.

    pub fn width(&self, scope: &lint::Scope<'_>, skip_payload: bool) -> Option<usize> {

        match &self.desc {

            DeclDesc::Enum { width, .. } | DeclDesc::Checksum { width, .. } => Some(*width),

            DeclDesc::CustomField { width, .. } => *width,

            DeclDesc::Packet { fields, parent_id, .. }

            | DeclDesc::Struct { fields, parent_id, .. } => {

                let mut packet_size = match parent_id {

                    None => 0,

                    Some(id) => scope.typedef.get(id.as_str())?.width(scope, true)?,

                };

                for field in fields.iter() {

                    packet_size += field.width(scope, skip_payload)?;

                }

                Some(packet_size)

            }

            DeclDesc::Group { .. } | DeclDesc::Test { .. } => None,

        }

    }



    pub fn fields(&self) -> std::slice::Iter<'_, Field<A>> {

        match &self.desc {

            DeclDesc::Packet { fields, .. }
@@ -463,62 +437,6 @@ impl<A: Annotation> Field<A> { 
        }

    }



    pub fn is_bitfield(&self, scope: &lint::Scope<'_>) -> bool {

        match &self.desc {

            FieldDesc::Size { .. }

            | FieldDesc::Count { .. }

            | FieldDesc::ElementSize { .. }

            | FieldDesc::FixedScalar { .. }

            | FieldDesc::FixedEnum { .. }

            | FieldDesc::Reserved { .. }

            | FieldDesc::Scalar { .. } => true,

            FieldDesc::Typedef { type_id, .. } => {

                let field = scope.typedef.get(type_id.as_str());

                matches!(field, Some(Decl { desc: DeclDesc::Enum { .. }, .. }))

            }

            _ => false,

        }

    }



    pub fn declaration<'a>(

        &self,

        scope: &'a lint::Scope<'a>,

    ) -> Option<&'a crate::parser::ast::Decl> {

        match &self.desc {

            FieldDesc::FixedEnum { enum_id, .. } => scope.typedef.get(enum_id).copied(),

            FieldDesc::Array { type_id: Some(type_id), .. } => scope.typedef.get(type_id).copied(),

            FieldDesc::Typedef { type_id, .. } => scope.typedef.get(type_id.as_str()).copied(),

            _ => None,

        }

    }



    /// Determine the size of a field in bits, if possible.

    ///

    /// If the field is dynamically sized (e.g. unsized array or

    /// payload field), `None` is returned. If `skip_payload` is set,

    /// payload and body fields are counted as having size `0` rather

    /// than a variable size.

    pub fn width(&self, scope: &lint::Scope<'_>, skip_payload: bool) -> Option<usize> {

        match &self.desc {

            FieldDesc::Scalar { width, .. }

            | FieldDesc::Size { width, .. }

            | FieldDesc::Count { width, .. }

            | FieldDesc::ElementSize { width, .. }

            | FieldDesc::Reserved { width, .. }

            | FieldDesc::FixedScalar { width, .. } => Some(*width),

            FieldDesc::FixedEnum { .. } => self.declaration(scope)?.width(scope, false),

            FieldDesc::Padding { .. } => todo!(),

            FieldDesc::Array { size: Some(size), width, .. } => {

                let width = width.or_else(|| self.declaration(scope)?.width(scope, false))?;

                Some(width * size)

            }

            FieldDesc::Typedef { .. } => self.declaration(scope)?.width(scope, false),

            FieldDesc::Checksum { .. } => Some(0),

            FieldDesc::Payload { .. } | FieldDesc::Body { .. } if skip_payload => Some(0),

            _ => None,

        }

    }



    pub fn kind(&self) -> &str {

        match &self.desc {

            FieldDesc::Checksum { .. } => "payload",

tools/pdl/src/backends/rust.rs

+25 −17
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ use quote::{format_ident, quote}; 
use std::collections::BTreeSet;

use std::path::Path;



use crate::parser::ast as parser_ast;

use crate::analyzer::ast as analyzer_ast;



mod parser;

mod preamble;
@@ -56,19 +56,19 @@ pub fn mask_bits(n: usize, suffix: &str) -> syn::LitInt { 


fn generate_packet_size_getter(

    scope: &lint::Scope<'_>,

    fields: &[&parser_ast::Field],

    fields: &[&analyzer_ast::Field],

    is_packet: bool,

) -> (usize, proc_macro2::TokenStream) {

    let mut constant_width = 0;

    let mut dynamic_widths = Vec::new();



    for field in fields {

        if let Some(width) = field.width(scope, false) {

        if let Some(width) = scope.get_field_width(field, false) {

            constant_width += width;

            continue;

        }



        let decl = field.declaration(scope);

        let decl = scope.get_field_declaration(field);

        dynamic_widths.push(match &field.desc {

            ast::FieldDesc::Payload { .. } | ast::FieldDesc::Body { .. } => {

                if is_packet {
@@ -88,7 +88,7 @@ fn generate_packet_size_getter( 
            ast::FieldDesc::Array { id, width, .. } => {

                let id = format_ident!("{id}");

                match &decl {

                    Some(parser_ast::Decl {

                    Some(analyzer_ast::Decl {

                        desc: ast::DeclDesc::Struct { .. } | ast::DeclDesc::CustomField { .. },

                        ..

                    }) => {
@@ -96,9 +96,10 @@ fn generate_packet_size_getter( 
                            self.#id.iter().map(|elem| elem.get_size()).sum::<usize>()

                        }

                    }

                    Some(parser_ast::Decl { desc: ast::DeclDesc::Enum { .. }, .. }) => {

                        let width =

                            syn::Index::from(decl.unwrap().width(scope, false).unwrap() / 8);

                    Some(analyzer_ast::Decl { desc: ast::DeclDesc::Enum { .. }, .. }) => {

                        let width = syn::Index::from(

                            scope.get_decl_width(decl.unwrap(), false).unwrap() / 8,

                        );

                        let mul_width = (width.index > 1).then(|| quote!(* #width));

                        quote! {

                            self.#id.len() #mul_width
@@ -133,7 +134,7 @@ fn generate_packet_size_getter( 
    )

}



fn top_level_packet<'a>(scope: &lint::Scope<'a>, packet_name: &'a str) -> &'a parser_ast::Decl {

fn top_level_packet<'a>(scope: &lint::Scope<'a>, packet_name: &'a str) -> &'a analyzer_ast::Decl {

    let mut decl = scope.typedef[packet_name];

    while let ast::DeclDesc::Packet { parent_id: Some(parent_id), .. }

    | ast::DeclDesc::Struct { parent_id: Some(parent_id), .. } = &decl.desc
@@ -147,7 +148,7 @@ fn top_level_packet<'a>(scope: &lint::Scope<'a>, packet_name: &'a str) -> &'a pa 
fn find_constrained_fields<'a>(

    scope: &'a lint::Scope<'a>,

    id: &'a str,

) -> Vec<&'a parser_ast::Field> {

) -> Vec<&'a analyzer_ast::Field> {

    let mut fields = Vec::new();

    let mut field_names = BTreeSet::new();

    let mut children = scope.iter_children(id).collect::<Vec<_>>();
@@ -177,7 +178,7 @@ fn find_constrained_fields<'a>( 
fn find_constrained_parent_fields<'a>(

    scope: &'a lint::Scope<'a>,

    id: &'a str,

) -> impl Iterator<Item = &'a parser_ast::Field> {

) -> impl Iterator<Item = &'a analyzer_ast::Field> {

    let packet_scope = &scope.scopes[&scope.typedef[id]];

    find_constrained_fields(scope, id).into_iter().filter(|field| {

        let id = field.id().unwrap();
@@ -299,7 +300,7 @@ fn generate_data_struct( 
/// Find all parents from `id`.

///

/// This includes the `Decl` for `id` itself.

fn find_parents<'a>(scope: &lint::Scope<'a>, id: &str) -> Vec<&'a parser_ast::Decl> {

fn find_parents<'a>(scope: &lint::Scope<'a>, id: &str) -> Vec<&'a analyzer_ast::Decl> {

    let mut decl = scope.typedef[id];

    let mut parents = vec![decl];

    while let ast::DeclDesc::Packet { parent_id: Some(parent_id), .. }
@@ -727,8 +728,8 @@ fn generate_enum_decl(id: &str, tags: &[ast::Tag]) -> proc_macro2::TokenStream { 


fn generate_decl(

    scope: &lint::Scope<'_>,

    file: &parser_ast::File,

    decl: &parser_ast::Decl,

    file: &analyzer_ast::File,

    decl: &analyzer_ast::Decl,

) -> String {

    match &decl.desc {

        ast::DeclDesc::Packet { id, .. } => {
@@ -751,7 +752,7 @@ fn generate_decl( 
///

/// The code is not formatted, pipe it through `rustfmt` to get

/// readable source code.

pub fn generate(sources: &ast::SourceDatabase, file: &parser_ast::File) -> String {

pub fn generate(sources: &ast::SourceDatabase, file: &analyzer_ast::File) -> String {

    let mut code = String::new();



    let source = sources.get(file.file).expect("could not read source");
@@ -769,6 +770,7 @@ pub fn generate(sources: &ast::SourceDatabase, file: &parser_ast::File) -> Strin 
#[cfg(test)]

mod tests {

    use super::*;

    use crate::analyzer;

    use crate::ast;

    use crate::parser::parse_inline;

    use crate::test_utils::{assert_snapshot_eq, rustfmt};
@@ -779,9 +781,11 @@ mod tests { 
    /// # Panics

    ///

    /// Panics on parse errors.

    pub fn parse_str(text: &str) -> parser_ast::File {

    pub fn parse_str(text: &str) -> analyzer_ast::File {

        let mut db = ast::SourceDatabase::new();

        parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error")

        let file =

            parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error");

        analyzer::analyze(&file).expect("analyzer error")

    }



    #[track_caller]
@@ -800,12 +804,15 @@ mod tests { 
                a: 8,

                b: 8,

                c: 8,

                _payload_,

              }

              packet Child: Parent(a = 10) {

                x: 8,

                _payload_,

              }

              packet GrandChild: Child(b = 20) {

                y: 8,

                _payload_,

              }

              packet GrandGrandChild: GrandChild(c = 30) {

                z: 8,
@@ -845,6 +852,7 @@ mod tests { 
                    let code = format!("{endianness}_packets\n{}", $code);

                    let mut db = ast::SourceDatabase::new();

                    let file = parse_inline(&mut db, String::from("test"), code).unwrap();

                    let file = analyzer::analyze(&file).unwrap();

                    let actual_code = generate(&db, &file);

                    assert_snapshot_eq(

                        &format!("tests/generated/{name}_{endianness}.rs"),

tools/pdl/src/backends/rust/parser.rs

+19 −16
Original line number Diff line number Diff line 
use crate::analyzer::ast as analyzer_ast;

use crate::backends::rust::{

    constraint_to_value, find_constrained_parent_fields, mask_bits, types,

};

use crate::parser::ast as parser_ast;

use crate::{ast, lint};

use heck::ToUpperCamelCase;

use quote::{format_ident, quote};
@@ -14,7 +14,7 @@ fn size_field_ident(id: &str) -> proc_macro2::Ident { 
/// A single bit-field.

struct BitField<'a> {

    shift: usize, // The shift to apply to this field.

    field: &'a parser_ast::Field,

    field: &'a analyzer_ast::Field,

}



pub struct FieldParser<'a> {
@@ -47,16 +47,16 @@ impl<'a> FieldParser<'a> { 
        }

    }



    pub fn add(&mut self, field: &'a parser_ast::Field) {

    pub fn add(&mut self, field: &'a analyzer_ast::Field) {

        match &field.desc {

            _ if field.is_bitfield(self.scope) => self.add_bit_field(field),

            _ if self.scope.is_bitfield(field) => self.add_bit_field(field),

            ast::FieldDesc::Padding { .. } => todo!("Padding fields are not supported"),

            ast::FieldDesc::Array { id, width, type_id, size, .. } => self.add_array_field(

                id,

                *width,

                type_id.as_deref(),

                *size,

                field.declaration(self.scope),

                self.scope.get_field_declaration(field),

            ),

            ast::FieldDesc::Typedef { id, type_id } => self.add_typedef_field(id, type_id),

            ast::FieldDesc::Payload { size_modifier, .. } => {
@@ -67,9 +67,9 @@ impl<'a> FieldParser<'a> { 
        }

    }



    fn add_bit_field(&mut self, field: &'a parser_ast::Field) {

    fn add_bit_field(&mut self, field: &'a analyzer_ast::Field) {

        self.chunk.push(BitField { shift: self.shift, field });

        self.shift += field.width(self.scope, false).unwrap();

        self.shift += self.scope.get_field_width(field, false).unwrap();

        if self.shift % 8 != 0 {

            return;

        }
@@ -110,7 +110,7 @@ impl<'a> FieldParser<'a> { 
                v = quote! { (#v >> #shift) }

            }



            let width = field.width(self.scope, false).unwrap();

            let width = self.scope.get_field_width(field, false).unwrap();

            let value_type = types::Integer::new(width);

            if !single_value && width < value_type.width {

                // Mask value if we grabbed more than `width` and if
@@ -225,7 +225,7 @@ impl<'a> FieldParser<'a> { 


        let mut offset = 0;

        for field in fields {

            if let Some(width) = field.width(self.scope, false) {

            if let Some(width) = self.scope.get_field_width(field, false) {

                offset += width;

            } else {

                return None;
@@ -260,13 +260,13 @@ impl<'a> FieldParser<'a> { 
        // `size`: the size of the array in number of elements (if

        // known). If None, the array is a Vec with a dynamic size.

        size: Option<usize>,

        decl: Option<&parser_ast::Decl>,

        decl: Option<&analyzer_ast::Decl>,

    ) {

        enum ElementWidth {

            Static(usize), // Static size in bytes.

            Unknown,

        }

        let element_width = match width.or_else(|| decl?.width(self.scope, false)) {

        let element_width = match width.or_else(|| self.scope.get_decl_width(decl?, false)) {

            Some(w) => {

                assert_eq!(w % 8, 0, "Array element size ({w}) is not a multiple of 8");

                ElementWidth::Static(w / 8)
@@ -433,7 +433,7 @@ impl<'a> FieldParser<'a> { 
        let id = format_ident!("{id}");

        let type_id = format_ident!("{type_id}");



        match decl.width(self.scope, true) {

        match self.scope.get_decl_width(decl, true) {

            None => self.code.push(quote! {

                let #id = #type_id::parse_inner(&mut #span)?;

            }),
@@ -535,7 +535,7 @@ impl<'a> FieldParser<'a> { 
        span: &proc_macro2::Ident,

        width: Option<usize>,

        type_id: Option<&str>,

        decl: Option<&parser_ast::Decl>,

        decl: Option<&analyzer_ast::Decl>,

    ) -> proc_macro2::TokenStream {

        if let Some(width) = width {

            let get_uint = types::get_uint(self.endianness, width, span);
@@ -568,7 +568,7 @@ impl<'a> FieldParser<'a> { 


    pub fn done(&mut self) {

        let decl = self.scope.typedef[self.packet_name];

        if let parser_ast::DeclDesc::Struct { .. } = &decl.desc {

        if let ast::DeclDesc::Struct { .. } = &decl.desc {

            return; // Structs don't parse the child structs recursively.

        }
@@ -652,6 +652,7 @@ impl quote::ToTokens for FieldParser<'_> { 
#[cfg(test)]

mod tests {

    use super::*;

    use crate::analyzer;

    use crate::ast;

    use crate::parser::parse_inline;
@@ -660,9 +661,11 @@ mod tests { 
    /// # Panics

    ///

    /// Panics on parse errors.

    pub fn parse_str(text: &str) -> parser_ast::File {

    pub fn parse_str(text: &str) -> analyzer_ast::File {

        let mut db = ast::SourceDatabase::new();

        parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error")

        let file =

            parse_inline(&mut db, String::from("stdin"), String::from(text)).expect("parse error");

        analyzer::analyze(&file).expect("analyzer error")

    }



    #[test]

tools/pdl/src/backends/rust/serializer.rs

+13 −8
Original line number Diff line number Diff line 
use crate::analyzer::ast as analyzer_ast;

use crate::backends::rust::{mask_bits, types};

use crate::parser::ast as parser_ast;

use crate::{ast, lint};

use heck::ToUpperCamelCase;

use quote::{format_ident, quote};
@@ -39,11 +39,11 @@ impl<'a> FieldSerializer<'a> { 
        }

    }



    pub fn add(&mut self, field: &parser_ast::Field) {

    pub fn add(&mut self, field: &analyzer_ast::Field) {

        match &field.desc {

            _ if field.is_bitfield(self.scope) => self.add_bit_field(field),

            _ if self.scope.is_bitfield(field) => self.add_bit_field(field),

            ast::FieldDesc::Array { id, width, .. } => {

                self.add_array_field(id, *width, field.declaration(self.scope))

                self.add_array_field(id, *width, self.scope.get_field_declaration(field))

            }

            ast::FieldDesc::Typedef { id, type_id } => {

                self.add_typedef_field(id, type_id);
@@ -55,8 +55,8 @@ impl<'a> FieldSerializer<'a> { 
        }

    }



    fn add_bit_field(&mut self, field: &parser_ast::Field) {

        let width = field.width(self.scope, false).unwrap();

    fn add_bit_field(&mut self, field: &analyzer_ast::Field) {

        let width = self.scope.get_field_width(field, false).unwrap();

        let shift = self.shift;



        match &field.desc {
@@ -115,7 +115,7 @@ impl<'a> FieldSerializer<'a> { 
                let field_type = types::Integer::new(*width);

                // TODO: size modifier



                let value_field_decl = value_field.declaration(self.scope);

                let value_field_decl = self.scope.get_field_declaration(value_field);



                let field_size_name = format_ident!("{field_id}_size");

                let array_size = match (&value_field.desc, value_field_decl.map(|decl| &decl.desc))
@@ -240,7 +240,12 @@ impl<'a> FieldSerializer<'a> { 
        self.shift = 0;

    }



    fn add_array_field(&mut self, id: &str, width: Option<usize>, decl: Option<&parser_ast::Decl>) {

    fn add_array_field(

        &mut self,

        id: &str,

        width: Option<usize>,

        decl: Option<&analyzer_ast::Decl>,

    ) {

        // TODO: padding



        let serialize = match width {

tools/pdl/src/backends/rust/types.rs

+6 −3
Original line number Diff line number Diff line 
//! Utility functions for dealing with Rust integer types.



use crate::parser::ast as parser_ast;

use crate::analyzer::ast as analyzer_ast;

use crate::{ast, lint};

use quote::{format_ident, quote};
@@ -34,7 +34,7 @@ impl quote::ToTokens for Integer { 
    }

}



pub fn rust_type(field: &parser_ast::Field) -> proc_macro2::TokenStream {

pub fn rust_type(field: &analyzer_ast::Field) -> proc_macro2::TokenStream {

    match &field.desc {

        ast::FieldDesc::Scalar { width, .. } => {

            let field_type = Integer::new(*width);
@@ -67,7 +67,10 @@ pub fn rust_type(field: &parser_ast::Field) -> proc_macro2::TokenStream { 
    }

}



pub fn rust_borrow(field: &parser_ast::Field, scope: &lint::Scope<'_>) -> proc_macro2::TokenStream {

pub fn rust_borrow(

    field: &analyzer_ast::Field,

    scope: &lint::Scope<'_>,

) -> proc_macro2::TokenStream {

    match &field.desc {

        ast::FieldDesc::Scalar { .. } => quote!(),

        ast::FieldDesc::Typedef { type_id, .. } => match &scope.typedef[type_id].desc {