Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 19229c8b authored by Henri Chataing's avatar Henri Chataing Committed by Cherrypicker Worker
Browse files

Annotate AST nodes with loc, size information 

Test: cargo build && cargo test
(cherry picked from https://android-review.googlesource.com/q/commit:1fb992a22087afac7fb55d8c5ca5b711718cf058)
Merged-In: Ibede334254e6faa7920139a68747792e87ed05a3
Change-Id: Ibede334254e6faa7920139a68747792e87ed05a3
parent f3937c33

tools/pdl/src/ast.rs

+91 −96
Original line number Diff line number Diff line
@@ -30,6 +30,11 @@ pub struct SourceRange { 
    pub end: SourceLocation,

}



pub trait Annotation: fmt::Debug + Serialize {

    type FieldAnnotation: Default + fmt::Debug;

    type DeclAnnotation: Default + fmt::Debug;

}



#[derive(Debug, Serialize)]

#[serde(tag = "kind", rename = "comment")]

pub struct Comment {
@@ -70,34 +75,32 @@ pub struct Constraint { 


#[derive(Debug, Serialize, Clone)]

#[serde(tag = "kind")]

pub enum Field {

pub enum FieldDesc {

    #[serde(rename = "checksum_field")]

    Checksum { loc: SourceRange, field_id: String },

    Checksum { field_id: String },

    #[serde(rename = "padding_field")]

    Padding { loc: SourceRange, size: usize },

    Padding { size: usize },

    #[serde(rename = "size_field")]

    Size { loc: SourceRange, field_id: String, width: usize },

    Size { field_id: String, width: usize },

    #[serde(rename = "count_field")]

    Count { loc: SourceRange, field_id: String, width: usize },

    Count { field_id: String, width: usize },

    #[serde(rename = "elementsize_field")]

    ElementSize { loc: SourceRange, field_id: String, width: usize },

    ElementSize { field_id: String, width: usize },

    #[serde(rename = "body_field")]

    Body { loc: SourceRange },

    Body,

    #[serde(rename = "payload_field")]

    Payload { loc: SourceRange, size_modifier: Option<String> },

    Payload { size_modifier: Option<String> },

    #[serde(rename = "fixed_field")]

    Fixed {

        loc: SourceRange,

        width: Option<usize>,

        value: Option<usize>,

        enum_id: Option<String>,

        tag_id: Option<String>,

    },

    #[serde(rename = "reserved_field")]

    Reserved { loc: SourceRange, width: usize },

    Reserved { width: usize },

    #[serde(rename = "array_field")]

    Array {

        loc: SourceRange,

        id: String,

        width: Option<usize>,

        type_id: Option<String>,
@@ -105,11 +108,20 @@ pub enum Field { 
        size: Option<usize>,

    },

    #[serde(rename = "scalar_field")]

    Scalar { loc: SourceRange, id: String, width: usize },

    Scalar { id: String, width: usize },

    #[serde(rename = "typedef_field")]

    Typedef { loc: SourceRange, id: String, type_id: String },

    Typedef { id: String, type_id: String },

    #[serde(rename = "group_field")]

    Group { loc: SourceRange, group_id: String, constraints: Vec<Constraint> },

    Group { group_id: String, constraints: Vec<Constraint> },

}



#[derive(Debug, Serialize)]

pub struct Field<A: Annotation> {

    pub loc: SourceRange,

    #[serde(skip_serializing)]

    pub annot: A::FieldAnnotation,

    #[serde(flatten)]

    pub desc: FieldDesc,

}



#[derive(Debug, Serialize)]
@@ -121,42 +133,49 @@ pub struct TestCase { 


#[derive(Debug, Serialize)]

#[serde(tag = "kind")]

pub enum Decl {

pub enum DeclDesc<A: Annotation> {

    #[serde(rename = "checksum_declaration")]

    Checksum { id: String, loc: SourceRange, function: String, width: usize },

    Checksum { id: String, function: String, width: usize },

    #[serde(rename = "custom_field_declaration")]

    CustomField { id: String, loc: SourceRange, width: Option<usize>, function: String },

    CustomField { id: String, width: Option<usize>, function: String },

    #[serde(rename = "enum_declaration")]

    Enum { id: String, loc: SourceRange, tags: Vec<Tag>, width: usize },

    Enum { id: String, tags: Vec<Tag>, width: usize },

    #[serde(rename = "packet_declaration")]

    Packet {

        id: String,

        loc: SourceRange,

        constraints: Vec<Constraint>,

        fields: Vec<Field>,

        fields: Vec<Field<A>>,

        parent_id: Option<String>,

    },

    #[serde(rename = "struct_declaration")]

    Struct {

        id: String,

        loc: SourceRange,

        constraints: Vec<Constraint>,

        fields: Vec<Field>,

        fields: Vec<Field<A>>,

        parent_id: Option<String>,

    },

    #[serde(rename = "group_declaration")]

    Group { id: String, loc: SourceRange, fields: Vec<Field> },

    Group { id: String, fields: Vec<Field<A>> },

    #[serde(rename = "test_declaration")]

    Test { loc: SourceRange, type_id: String, test_cases: Vec<TestCase> },

    Test { type_id: String, test_cases: Vec<TestCase> },

}



#[derive(Debug, Serialize)]

pub struct File {

pub struct Decl<A: Annotation> {

    pub loc: SourceRange,

    #[serde(skip_serializing)]

    pub annot: A::DeclAnnotation,

    #[serde(flatten)]

    pub desc: DeclDesc<A>,

}



#[derive(Debug, Serialize)]

pub struct File<A: Annotation> {

    pub version: String,

    pub file: FileId,

    pub comments: Vec<Comment>,

    pub endianness: Endianness,

    pub declarations: Vec<Decl>,

    pub declarations: Vec<Decl<A>>,

}



impl SourceLocation {
@@ -213,8 +232,8 @@ impl ops::Add<SourceRange> for SourceRange { 
    }

}



impl File {

    pub fn new(file: FileId) -> File {

impl<A: Annotation> File<A> {

    pub fn new(file: FileId) -> File<A> {

        File {

            version: "1,0".to_owned(),

            comments: vec![],
@@ -230,92 +249,68 @@ impl File { 
    }

}



impl Decl {

    pub fn loc(&self) -> &SourceRange {

        match self {

            Decl::Checksum { loc, .. }

            | Decl::CustomField { loc, .. }

            | Decl::Enum { loc, .. }

            | Decl::Packet { loc, .. }

            | Decl::Struct { loc, .. }

            | Decl::Group { loc, .. }

            | Decl::Test { loc, .. } => loc,

        }

    }



impl<A: Annotation> Decl<A> {

    pub fn id(&self) -> Option<&str> {

        match self {

            Decl::Test { .. } => None,

            Decl::Checksum { id, .. }

            | Decl::CustomField { id, .. }

            | Decl::Enum { id, .. }

            | Decl::Packet { id, .. }

            | Decl::Struct { id, .. }

            | Decl::Group { id, .. } => Some(id),

        }

        match &self.desc {

            DeclDesc::Test { .. } => None,

            DeclDesc::Checksum { id, .. }

            | DeclDesc::CustomField { id, .. }

            | DeclDesc::Enum { id, .. }

            | DeclDesc::Packet { id, .. }

            | DeclDesc::Struct { id, .. }

            | DeclDesc::Group { id, .. } => Some(id),

        }

    }



impl Field {

    pub fn loc(&self) -> &SourceRange {

        match self {

            Field::Checksum { loc, .. }

            | Field::Padding { loc, .. }

            | Field::Size { loc, .. }

            | Field::ElementSize { loc, .. }

            | Field::Count { loc, .. }

            | Field::Body { loc, .. }

            | Field::Payload { loc, .. }

            | Field::Fixed { loc, .. }

            | Field::Reserved { loc, .. }

            | Field::Array { loc, .. }

            | Field::Scalar { loc, .. }

            | Field::Typedef { loc, .. }

            | Field::Group { loc, .. } => loc,

    pub fn new(loc: SourceRange, desc: DeclDesc<A>) -> Decl<A> {

        Decl { loc, annot: Default::default(), desc }

    }

}



impl<A: Annotation> Field<A> {

    pub fn id(&self) -> Option<&str> {

        match self {

            Field::Checksum { .. }

            | Field::Padding { .. }

            | Field::Size { .. }

            | Field::ElementSize { .. }

            | Field::Count { .. }

            | Field::Body { .. }

            | Field::Payload { .. }

            | Field::Fixed { .. }

            | Field::Reserved { .. }

            | Field::Group { .. } => None,

            Field::Array { id, .. } | Field::Scalar { id, .. } | Field::Typedef { id, .. } => {

                Some(id)

            }

        match &self.desc {

            FieldDesc::Checksum { .. }

            | FieldDesc::Padding { .. }

            | FieldDesc::Size { .. }

            | FieldDesc::Count { .. }

            | FieldDesc::ElementSize { .. }

            | FieldDesc::Body

            | FieldDesc::Payload { .. }

            | FieldDesc::Fixed { .. }

            | FieldDesc::Reserved { .. }

            | FieldDesc::Group { .. } => None,

            FieldDesc::Array { id, .. }

            | FieldDesc::Scalar { id, .. }

            | FieldDesc::Typedef { id, .. } => Some(id),

        }

    }



    pub fn is_bitfield(&self, scope: &lint::Scope<'_>) -> bool {

        match self {

            Field::Size { .. }

            | Field::Count { .. }

            | Field::Fixed { .. }

            | Field::Reserved { .. }

            | Field::Scalar { .. } => true,

            Field::Typedef { type_id, .. } => {

        match &self.desc {

            FieldDesc::Size { .. }

            | FieldDesc::Count { .. }

            | FieldDesc::ElementSize { .. }

            | FieldDesc::Fixed { .. }

            | FieldDesc::Reserved { .. }

            | FieldDesc::Scalar { .. } => true,

            FieldDesc::Typedef { type_id, .. } => {

                let field = scope.typedef.get(type_id.as_str());

                matches!(field, Some(Decl::Enum { .. }))

                matches!(field, Some(Decl { desc: DeclDesc::Enum { .. }, .. }))

            }

            _ => false,

        }

    }



    pub fn width(&self, scope: &lint::Scope<'_>) -> Option<usize> {

        match self {

            Field::Scalar { width, .. }

            | Field::Size { width, .. }

            | Field::Count { width, .. }

            | Field::Reserved { width, .. } => Some(*width),

            Field::Typedef { type_id, .. } => match scope.typedef.get(type_id.as_str()) {

                Some(Decl::Enum { width, .. }) => Some(*width),

        match &self.desc {

            FieldDesc::Scalar { width, .. }

            | FieldDesc::Size { width, .. }

            | FieldDesc::Count { width, .. }

            | FieldDesc::ElementSize { width, .. }

            | FieldDesc::Reserved { width, .. } => Some(*width),

            FieldDesc::Typedef { type_id, .. } => match scope.typedef.get(type_id.as_str()) {

                Some(Decl { desc: DeclDesc::Enum { width, .. }, .. }) => Some(*width),

                _ => None,

            },

            // TODO(mgeisler): padding, arrays, etc.

tools/pdl/src/backends/intermediate.rs

+30 −23
Original line number Diff line number Diff line 
use std::collections::{hash_map::Entry, HashMap};



use crate::ast;

use crate::parser;



pub struct Schema<'a> {

    pub packets_and_structs: HashMap<&'a str, PacketOrStruct<'a>>,
@@ -82,7 +83,7 @@ pub enum ComputedOffset<'a> { 
    Alias(ComputedOffsetId<'a>),

}



pub fn generate(file: &ast::File) -> Result<Schema, String> {

pub fn generate(file: &parser::ast::File) -> Result<Schema, String> {

    let mut schema = Schema { packets_and_structs: HashMap::new(), enums: HashMap::new() };

    match file.endianness.value {

        ast::EndiannessValue::LittleEndian => {}
@@ -96,13 +97,13 @@ pub fn generate(file: &ast::File) -> Result<Schema, String> { 
    Ok(schema)

}



fn process_decl<'a>(schema: &mut Schema<'a>, decl: &'a ast::Decl) {

    match decl {

        ast::Decl::Enum { id, tags, width, .. } => process_enum(schema, id, tags, *width),

        ast::Decl::Packet { id, fields, .. } | ast::Decl::Struct { id, fields, .. } => {

fn process_decl<'a>(schema: &mut Schema<'a>, decl: &'a parser::ast::Decl) {

    match &decl.desc {

        ast::DeclDesc::Enum { id, tags, width, .. } => process_enum(schema, id, tags, *width),

        ast::DeclDesc::Packet { id, fields, .. } | ast::DeclDesc::Struct { id, fields, .. } => {

            process_packet_or_struct(schema, id, fields)

        }

        ast::Decl::Group { .. } => todo!(),

        ast::DeclDesc::Group { .. } => todo!(),

        _ => unimplemented!("type {decl:?} not supported"),

    }

}
@@ -119,11 +120,18 @@ fn process_enum<'a>(schema: &mut Schema<'a>, id: &'a str, tags: &'a [ast::Tag], 
    );

}



fn process_packet_or_struct<'a>(schema: &mut Schema<'a>, id: &'a str, fields: &'a [ast::Field]) {

fn process_packet_or_struct<'a>(

    schema: &mut Schema<'a>,

    id: &'a str,

    fields: &'a [parser::ast::Field],

) {

    schema.packets_and_structs.insert(id, compute_getters(schema, fields));

}



fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketOrStruct<'a> {

fn compute_getters<'a>(

    schema: &Schema<'a>,

    fields: &'a [parser::ast::Field],

) -> PacketOrStruct<'a> {

    let mut prev_pos_id = None;

    let mut curr_pos_id = ComputedOffsetId::HeaderStart;

    let mut computed_values = HashMap::new();
@@ -142,16 +150,16 @@ fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketO 
        // populate this only if we are an array with a knowable size

        let mut next_prev_pos_id = None;



        let next_pos = match field {

            ast::Field::Reserved { width, .. } => {

        let next_pos = match &field.desc {

            ast::FieldDesc::Reserved { width } => {

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, *width as i64)

            }

            ast::Field::Scalar { id, width, .. } => {

            ast::FieldDesc::Scalar { id, width } => {

                computed_offsets

                    .insert(ComputedOffsetId::FieldOffset(id), ComputedOffset::Alias(curr_pos_id));

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, *width as i64)

            }

            ast::Field::Fixed { width, enum_id, .. } => {

            ast::FieldDesc::Fixed { width, enum_id, .. } => {

                let offset = match (width, enum_id) {

                    (Some(width), _) => *width,

                    (_, Some(enum_id)) => schema.enums[enum_id.as_str()].width,
@@ -159,32 +167,32 @@ fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketO 
                };

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, offset as i64)

            }

            ast::Field::Size { field_id, width, .. } => {

            ast::FieldDesc::Size { field_id, width } => {

                computed_values.insert(

                    ComputedValueId::FieldSize(field_id),

                    ComputedValue::ValueAt { offset: curr_pos_id, width: *width },

                );

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, *width as i64)

            }

            ast::Field::Count { field_id, width, .. } => {

            ast::FieldDesc::Count { field_id, width } => {

                computed_values.insert(

                    ComputedValueId::FieldCount(field_id.as_str()),

                    ComputedValue::ValueAt { offset: curr_pos_id, width: *width },

                );

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, *width as i64)

            }

            ast::Field::ElementSize { field_id, width, .. } => {

            ast::FieldDesc::ElementSize { field_id, width } => {

                computed_values.insert(

                    ComputedValueId::FieldElementSize(field_id),

                    ComputedValue::ValueAt { offset: curr_pos_id, width: *width },

                );

                ComputedOffset::ConstantPlusOffsetInBits(curr_pos_id, *width as i64)

            }

            ast::Field::Group { .. } => {

            ast::FieldDesc::Group { .. } => {

                unimplemented!("this should be removed by the linter...")

            }

            ast::Field::Checksum { .. } => unimplemented!("checksum not supported"),

            ast::Field::Body { .. } => {

            ast::FieldDesc::Checksum { .. } => unimplemented!("checksum not supported"),

            ast::FieldDesc::Body => {

                computed_offsets.insert(

                    ComputedOffsetId::FieldOffset("_body_"),

                    ComputedOffset::Alias(curr_pos_id),
@@ -202,7 +210,7 @@ fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketO 
                computed_offsets.insert(ComputedOffsetId::FieldEndOffset("_body_"), end_offset);

                end_offset

            }

            ast::Field::Payload { size_modifier, .. } => {

            ast::FieldDesc::Payload { size_modifier } => {

                if size_modifier.is_some() {

                    unimplemented!("size modifiers not supported")

                }
@@ -223,13 +231,12 @@ fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketO 
                computed_offsets.insert(ComputedOffsetId::FieldEndOffset("_payload_"), end_offset);

                end_offset

            }

            ast::Field::Array {

            ast::FieldDesc::Array {

                id,

                width,

                type_id,

                size_modifier,

                size: statically_known_count,

                ..

            } => {

                if size_modifier.is_some() {

                    unimplemented!("size modifiers not supported")
@@ -395,14 +402,14 @@ fn compute_getters<'a>(schema: &Schema<'a>, fields: &'a [ast::Field]) -> PacketO 
                    ComputedOffset::Alias(ComputedOffsetId::TrailerStart)

                }

            }

            ast::Field::Padding { size, .. } => {

            ast::FieldDesc::Padding { size } => {

                if let Some(prev_pos_id) = prev_pos_id {

                    ComputedOffset::ConstantPlusOffsetInBits(prev_pos_id, *size as i64)

                } else {

                    panic!("padding must follow array field with known total size")

                }

            }

            ast::Field::Typedef { id, type_id, .. } => {

            ast::FieldDesc::Typedef { id, type_id } => {

                computed_offsets

                    .insert(ComputedOffsetId::FieldOffset(id), ComputedOffset::Alias(curr_pos_id));

tools/pdl/src/backends/json.rs

+2 −2
Original line number Diff line number Diff line 
//! Rust compiler backend.



use crate::ast;

use crate::parser;



/// Turn the AST into a JSON representation.

pub fn generate(file: &ast::File) -> Result<String, String> {

pub fn generate(file: &parser::ast::File) -> Result<String, String> {

    serde_json::to_string_pretty(&file)

        .map_err(|err| format!("could not JSON serialize grammar: {err}"))

}

tools/pdl/src/backends/rust.rs

+13 −7
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ use crate::{ast, lint}; 
use quote::{format_ident, quote};

use std::path::Path;



use crate::parser::ast as parser_ast;



mod declarations;

mod parser;

mod preamble;
@@ -49,7 +51,7 @@ fn generate_packet_decl( 
    // Packet:

    id: &str,

    _constraints: &[ast::Constraint],

    fields: &[ast::Field],

    fields: &[parser_ast::Field],

    _parent_id: Option<&str>,

) -> proc_macro2::TokenStream {

    // TODO(mgeisler): use the convert_case crate to convert between
@@ -237,10 +239,14 @@ fn generate_enum_decl(id: &str, tags: &[ast::Tag]) -> proc_macro2::TokenStream { 
    }

}



fn generate_decl(scope: &lint::Scope<'_>, file: &ast::File, decl: &ast::Decl) -> String {

    match decl {

        ast::Decl::Packet { id, constraints, fields, parent_id, .. }

        | ast::Decl::Struct { id, constraints, fields, parent_id, .. } => generate_packet_decl(

fn generate_decl(

    scope: &lint::Scope<'_>,

    file: &parser_ast::File,

    decl: &parser_ast::Decl,

) -> String {

    match &decl.desc {

        ast::DeclDesc::Packet { id, constraints, fields, parent_id, .. }

        | ast::DeclDesc::Struct { id, constraints, fields, parent_id, .. } => generate_packet_decl(

            scope,

            file.endianness.value,

            id,
@@ -249,7 +255,7 @@ fn generate_decl(scope: &lint::Scope<'_>, file: &ast::File, decl: &ast::Decl) -> 
            parent_id.as_deref(),

        )

        .to_string(),

        ast::Decl::Enum { id, tags, .. } => generate_enum_decl(id, tags).to_string(),

        ast::DeclDesc::Enum { id, tags, .. } => generate_enum_decl(id, tags).to_string(),

        _ => todo!("unsupported Decl::{:?}", decl),

    }

}
@@ -258,7 +264,7 @@ fn generate_decl(scope: &lint::Scope<'_>, file: &ast::File, decl: &ast::Decl) -> 
///

/// The code is not formatted, pipe it through `rustfmt` to get

/// readable source code.

pub fn generate(sources: &ast::SourceDatabase, file: &ast::File) -> String {

pub fn generate(sources: &ast::SourceDatabase, file: &parser_ast::File) -> String {

    let mut code = String::new();



    let source = sources.get(file.file).expect("could not read source");

tools/pdl/src/backends/rust/declarations.rs

+6 −5
Original line number Diff line number Diff line 
use crate::ast;

use crate::backends::rust::types;

use crate::parser::ast as parser_ast;

use quote::{format_ident, quote};



pub struct FieldDeclarations {
@@ -11,23 +12,23 @@ impl FieldDeclarations { 
        FieldDeclarations { code: Vec::new() }

    }



    pub fn add(&mut self, field: &ast::Field) {

        self.code.push(match field {

            ast::Field::Scalar { id, width, .. } => {

    pub fn add(&mut self, field: &parser_ast::Field) {

        self.code.push(match &field.desc {

            ast::FieldDesc::Scalar { id, width } => {

                let id = format_ident!("{id}");

                let field_type = types::Integer::new(*width);

                quote! {

                    #id: #field_type,

                }

            }

            ast::Field::Typedef { id, type_id, .. } => {

            ast::FieldDesc::Typedef { id, type_id } => {

                let id = format_ident!("{id}");

                let field_type = format_ident!("{type_id}");

                quote! {

                    #id: #field_type,

                }

            }

            ast::Field::Reserved { .. } => {

            ast::FieldDesc::Reserved { .. } => {

                // Nothing to do here.

                quote! {}

            }