Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b13a3214 authored by Muhammad Qureshi's avatar Muhammad Qureshi
Browse files

Store annotation during collation 

- Only collate atoms in the specified module.

- Replace signature_to_modules with signatureInfoMap. This maps each
signature to another map keyed by field number and whose values are vector of
annotations.

Bug: 151102006
Test: stats-log-api-gen-test
Change-Id: I25bbe4883c8f7f86a06d04d27cd425367b6d65a0
parent 8f15e287

tools/stats_log_api_gen/Collation.cpp

+160 −57
Original line number Diff line number Diff line
@@ -27,8 +27,11 @@ using google::protobuf::EnumDescriptor; 
using google::protobuf::FieldDescriptor;

using google::protobuf::FileDescriptor;

using google::protobuf::SourceLocation;

using std::make_shared;

using std::map;



const bool dbg = false;





//

// AtomDecl class
@@ -45,6 +48,7 @@ AtomDecl::AtomDecl(const AtomDecl &that) 
        name(that.name),

        message(that.message),

        fields(that.fields),

        fieldNumberToAnnotations(that.fieldNumberToAnnotations),

        primaryFields(that.primaryFields),

        exclusiveField(that.exclusiveField),

        defaultState(that.defaultState),
@@ -52,8 +56,7 @@ AtomDecl::AtomDecl(const AtomDecl &that) 
        nested(that.nested),

        uidField(that.uidField),

        whitelisted(that.whitelisted),

        binaryFields(that.binaryFields),

        moduleNames(that.moduleNames) {}

        binaryFields(that.binaryFields) {}



AtomDecl::AtomDecl(int c, const string& n, const string& m)

    :code(c),
@@ -160,6 +163,17 @@ void collate_enums(const EnumDescriptor &enumDescriptor, AtomField *atomField) { 
    }

}



static void addAnnotationToAtomDecl(AtomDecl* atomDecl, const int fieldNumber,

        const int annotationId, const AnnotationType annotationType,

        const AnnotationValue annotationValue) {

    if (dbg) {

        printf("   Adding annotation to %s: [%d] = {id: %d, type: %d}\n",

                atomDecl->name.c_str(), fieldNumber, annotationId, annotationType);

    }

    atomDecl->fieldNumberToAnnotations[fieldNumber].insert(make_shared<Annotation>(

                annotationId, atomDecl->code, annotationType, annotationValue));

}



/**

 * Gather the info about an atom proto.

 */
@@ -279,7 +293,6 @@ int collate_atom(const Descriptor *atom, AtomDecl *atomDecl, 
      if (javaType == JAVA_TYPE_ENUM) {

        // All enums are treated as ints when it comes to function signatures.

        signature->push_back(JAVA_TYPE_INT);

        collate_enums(*field->enum_type(), &atField);

      } else if (javaType == JAVA_TYPE_OBJECT && isBinaryField) {

          signature->push_back(JAVA_TYPE_BYTE_ARRAY);

      } else {
@@ -292,64 +305,121 @@ int collate_atom(const Descriptor *atom, AtomDecl *atomDecl, 
    }

    atomDecl->fields.push_back(atField);



    if (field->options().GetExtension(os::statsd::state_field_option).option() ==

        os::statsd::StateField::PRIMARY_FIELD) {

    if (field->options().HasExtension(os::statsd::state_field_option)) {

        const int option = field->options().GetExtension(os::statsd::state_field_option).option();

        if (option != STATE_OPTION_UNSET) {

            addAnnotationToAtomDecl(atomDecl, signature->size(), ANNOTATION_ID_STATE_OPTION,

                    ANNOTATION_TYPE_INT, AnnotationValue(option));

        }



        if (option == STATE_OPTION_PRIMARY) {

            if (javaType == JAVA_TYPE_UNKNOWN ||

                    javaType == JAVA_TYPE_ATTRIBUTION_CHAIN ||

                    javaType == JAVA_TYPE_OBJECT || javaType == JAVA_TYPE_BYTE_ARRAY) {

                print_error(

                    field,

                    "Invalid primary state field: '%s'\n",

                    atom->name().c_str());

                errorCount++;

                continue;

            }

            atomDecl->primaryFields.push_back(it->first);



        }



    if (field->options().GetExtension(os::statsd::state_field_option).option() ==

        os::statsd::StateField::PRIMARY_FIELD_FIRST_UID) {

        if (option == STATE_OPTION_PRIMARY_FIELD_FIRST_UID) {

            if (javaType != JAVA_TYPE_ATTRIBUTION_CHAIN) {

                print_error(

                    field,

                    "PRIMARY_FIELD_FIRST_UID annotation is only for AttributionChains: '%s'\n",

                    atom->name().c_str());

                errorCount++;

                continue;

            } else {

                atomDecl->primaryFields.push_back(FIRST_UID_IN_CHAIN_ID);

            }

        }



    if (field->options().GetExtension(os::statsd::state_field_option).option() ==

        os::statsd::StateField::EXCLUSIVE_STATE) {

        if (option == STATE_OPTION_EXCLUSIVE) {

            if (javaType == JAVA_TYPE_UNKNOWN ||

                    javaType == JAVA_TYPE_ATTRIBUTION_CHAIN ||

                    javaType == JAVA_TYPE_OBJECT || javaType == JAVA_TYPE_BYTE_ARRAY) {

                print_error(

                    field,

                    "Invalid exclusive state field: '%s'\n",

                    atom->name().c_str());

                errorCount++;

                continue;

            }



            if (atomDecl->exclusiveField == 0) {

                atomDecl->exclusiveField = it->first;

            } else {

                print_error(

                    field,

                    "Cannot have more than one exclusive state field in an atom: '%s'\n",

                    atom->name().c_str());

                errorCount++;

                continue;

            }



            if (field->options()

                        .GetExtension(os::statsd::state_field_option)

                        .has_default_state_value()) {

            atomDecl->defaultState = field->options()

                                             .GetExtension(os::statsd::state_field_option)

                const int defaultState =

                        field->options().GetExtension(os::statsd::state_field_option)

                        .default_state_value();

                atomDecl->defaultState = defaultState;



                addAnnotationToAtomDecl(atomDecl, signature->size(), ANNOTATION_ID_DEFAULT_STATE,

                        ANNOTATION_TYPE_INT, AnnotationValue(defaultState));

            }



        if (field->options().GetExtension(os::statsd::state_field_option).has_reset_state_value()) {

            atomDecl->resetState = field->options()

            if (field->options().GetExtension(os::statsd::state_field_option)

                    .has_reset_state_value()) {

                const int resetState = field->options()

                        .GetExtension(os::statsd::state_field_option)

                        .reset_state_value();



                atomDecl->resetState = resetState;

                addAnnotationToAtomDecl(atomDecl, signature->size(), ANNOTATION_ID_RESET_STATE,

                        ANNOTATION_TYPE_INT, AnnotationValue(resetState));

            }



            if (field->options().GetExtension(os::statsd::state_field_option)

                    .has_nested()) {

                const bool nested =

                        field->options().GetExtension(os::statsd::state_field_option).nested();

                atomDecl->nested = nested;



                addAnnotationToAtomDecl(atomDecl, signature->size(), ANNOTATION_ID_STATE_NESTED,

                        ANNOTATION_TYPE_BOOL, AnnotationValue(nested));

            }

        atomDecl->nested = field->options().GetExtension(os::statsd::state_field_option).nested();

        }



    }

    if (field->options().GetExtension(os::statsd::is_uid) == true) {

        if (javaType != JAVA_TYPE_INT) {

            print_error(

                field,

                "is_uid annotation can only be applied to int32 fields: '%s'\n",

                atom->name().c_str());

            errorCount++;

            continue;

        }



        if (atomDecl->uidField == 0) {

            atomDecl->uidField = it->first;



            addAnnotationToAtomDecl(atomDecl, signature->size(), ANNOTATION_ID_IS_UID,

                    ANNOTATION_TYPE_BOOL, AnnotationValue(true));

        } else {

            print_error(

                field,

                "Cannot have more than one field in an atom with is_uid annotation: '%s'\n",

                atom->name().c_str());

            errorCount++;

            continue;

        }

    }

    // Binary field validity is already checked above.
@@ -408,17 +478,50 @@ bool get_non_chained_node(const Descriptor *atom, AtomDecl *atomDecl, 
    return has_attribution_node;

}



static void populateFieldNumberToAnnotations(

        const AtomDecl& atomDecl,

        FieldNumberToAnnotations* fieldNumberToAnnotations) {

    for (FieldNumberToAnnotations::const_iterator it = atomDecl.fieldNumberToAnnotations.begin();

            it != atomDecl.fieldNumberToAnnotations.end(); it++) {

        const int fieldNumber = it->first;

        const set<shared_ptr<Annotation>>& insertAnnotationsSource = it->second;

        set<shared_ptr<Annotation>>& insertAnnotationsTarget =

                (*fieldNumberToAnnotations)[fieldNumber];

        insertAnnotationsTarget.insert(

                insertAnnotationsSource.begin(),

                insertAnnotationsSource.end());

    }

}



/**

 * Gather the info about the atoms.

 */

int collate_atoms(const Descriptor *descriptor, Atoms *atoms) {

int collate_atoms(const Descriptor *descriptor, const string& moduleName, Atoms *atoms) {

  int errorCount = 0;

  const bool dbg = false;



  int maxPushedAtomId = 2;

  for (int i = 0; i < descriptor->field_count(); i++) {

    const FieldDescriptor *atomField = descriptor->field(i);



    if (moduleName != DEFAULT_MODULE_NAME) {

        const int moduleCount = atomField->options().ExtensionSize(os::statsd::module);

        int j;

        for (j = 0; j < moduleCount; ++j) {

            const string atomModuleName = atomField->options().GetExtension(os::statsd::module, j);

            if (atomModuleName == moduleName) {

                break;

            }

        }



        // This atom is not in the module we're interested in; skip it.

        if (moduleCount == j) {

            if (dbg) {

              printf("   Skipping %s (%d)\n", atomField->name().c_str(), atomField->number());

            }

            continue;

        }

    }



    if (dbg) {

      printf("   %s (%d)\n", atomField->name().c_str(), atomField->number());

    }
@@ -441,27 +544,27 @@ int collate_atoms(const Descriptor *descriptor, Atoms *atoms) { 
        atomDecl.whitelisted = true;

    }



    for (int j = 0; j < atomField->options().ExtensionSize(os::statsd::module); ++j) {

        const string moduleName = atomField->options().GetExtension(os::statsd::module, j);

        atomDecl.moduleNames.insert(moduleName);

    }



    vector<java_type_t> signature;

    errorCount += collate_atom(atom, &atomDecl, &signature);

    if (atomDecl.primaryFields.size() != 0 && atomDecl.exclusiveField == 0) {

        print_error(atomField,

                  "Cannot have a primary field without an exclusive field: %s\n",

                  atomField->name().c_str());

        errorCount++;

        continue;

    }



    atoms->signatures_to_modules[signature].insert(

            atomDecl.moduleNames.begin(), atomDecl.moduleNames.end());

    atoms->decls.insert(atomDecl);

    FieldNumberToAnnotations& fieldNumberToAnnotations = atoms->signatureInfoMap[signature];

    populateFieldNumberToAnnotations(atomDecl, &fieldNumberToAnnotations);



    AtomDecl nonChainedAtomDecl(atomField->number(), atomField->name(), atom->name());

    vector<java_type_t> nonChainedSignature;

    if (get_non_chained_node(atom, &nonChainedAtomDecl, &nonChainedSignature)) {

        atoms->non_chained_signatures_to_modules[nonChainedSignature].insert(

            atomDecl.moduleNames.begin(), atomDecl.moduleNames.end());

        atoms->non_chained_decls.insert(nonChainedAtomDecl);

        FieldNumberToAnnotations& fieldNumberToAnnotations =

                atoms->nonChainedSignatureInfoMap[nonChainedSignature];

        populateFieldNumberToAnnotations(atomDecl, &fieldNumberToAnnotations);

    }



    if (atomDecl.code < PULL_ATOM_START_ID && atomDecl.code > maxPushedAtomId) {
@@ -473,9 +576,9 @@ int collate_atoms(const Descriptor *descriptor, Atoms *atoms) { 


  if (dbg) {

    printf("signatures = [\n");

    for (map<vector<java_type_t>, set<string>>::const_iterator it =

             atoms->signatures_to_modules.begin();

         it != atoms->signatures_to_modules.end(); it++) {

    for (map<vector<java_type_t>, FieldNumberToAnnotations>::const_iterator it =

             atoms->signatureInfoMap.begin();

         it != atoms->signatureInfoMap.end(); it++) {

      printf("   ");

      for (vector<java_type_t>::const_iterator jt = it->first.begin();

           jt != it->first.end(); jt++){

tools/stats_log_api_gen/Collation.h

+54 −5
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 




#include <google/protobuf/descriptor.h>

#include "frameworks/base/cmds/statsd/src/atom_field_options.pb.h"



#include <set>

#include <vector>
@@ -29,6 +30,7 @@ namespace stats_log_api_gen { 


using std::map;

using std::set;

using std::shared_ptr;

using std::string;

using std::vector;

using google::protobuf::Descriptor;
@@ -38,6 +40,20 @@ const int PULL_ATOM_START_ID = 10000; 


const int FIRST_UID_IN_CHAIN_ID = 0;



const unsigned char ANNOTATION_ID_IS_UID = 1;

const unsigned char ANNOTATION_ID_TRUNCATE_TIMESTAMP = 2;

const unsigned char ANNOTATION_ID_STATE_OPTION = 3;

const unsigned char ANNOTATION_ID_DEFAULT_STATE = 4;

const unsigned char ANNOTATION_ID_RESET_STATE = 5;

const unsigned char ANNOTATION_ID_STATE_NESTED = 6;



const int STATE_OPTION_UNSET = os::statsd::StateField::STATE_FIELD_UNSET;

const int STATE_OPTION_EXCLUSIVE = os::statsd::StateField::EXCLUSIVE_STATE;

const int STATE_OPTION_PRIMARY_FIELD_FIRST_UID = os::statsd::StateField::PRIMARY_FIELD_FIRST_UID;

const int STATE_OPTION_PRIMARY = os::statsd::StateField::PRIMARY_FIELD;



const string DEFAULT_MODULE_NAME = "DEFAULT";



/**

 * The types for atom parameters.

 */
@@ -58,6 +74,38 @@ typedef enum { 
  JAVA_TYPE_BYTE_ARRAY = -2,

} java_type_t;



enum AnnotationType {

    ANNOTATION_TYPE_UNKNOWN = 0,

    ANNOTATION_TYPE_INT = 1,

    ANNOTATION_TYPE_BOOL = 2,

};



union AnnotationValue {

    int intValue;

    bool boolValue;



    AnnotationValue(const int value): intValue(value) {}

    AnnotationValue(const bool value): boolValue(value) {}

};



struct Annotation {

    const unsigned char annotationId;

    const int atomId;

    AnnotationType type;

    AnnotationValue value;



    inline Annotation(unsigned char annotationId, int atomId, AnnotationType type,

            AnnotationValue value):

            annotationId(annotationId), atomId(atomId), type(type), value(value) {}

    inline ~Annotation() {}



    inline bool operator<(const Annotation& that) const {

        return atomId == that.atomId ? annotationId < that.annotationId : atomId < that.atomId;

    }

};



using FieldNumberToAnnotations =  map<int, set<shared_ptr<Annotation>>>;



/**

 * The name and type for an atom field.

 */
@@ -72,6 +120,7 @@ struct AtomField { 
    inline AtomField(const AtomField& that) :name(that.name),

                                             javaType(that.javaType),

                                             enumValues(that.enumValues) {}



    inline AtomField(string n, java_type_t jt) :name(n), javaType(jt) {}

    inline ~AtomField() {}

};
@@ -86,6 +135,8 @@ struct AtomDecl { 
    string message;

    vector<AtomField> fields;



    FieldNumberToAnnotations fieldNumberToAnnotations;



    vector<int> primaryFields;

    int exclusiveField = 0;

    int defaultState = INT_MAX;
@@ -98,8 +149,6 @@ struct AtomDecl { 


    vector<int> binaryFields;



    set<string> moduleNames;



    AtomDecl();

    AtomDecl(const AtomDecl& that);

    AtomDecl(int code, const string& name, const string& message);
@@ -111,17 +160,17 @@ struct AtomDecl { 
};



struct Atoms {

    map<vector<java_type_t>, set<string>> signatures_to_modules;

    map<vector<java_type_t>, FieldNumberToAnnotations> signatureInfoMap;

    set<AtomDecl> decls;

    set<AtomDecl> non_chained_decls;

    map<vector<java_type_t>, set<string>> non_chained_signatures_to_modules;

    map<vector<java_type_t>, FieldNumberToAnnotations> nonChainedSignatureInfoMap;

    int maxPushedAtomId;

};



/**

 * Gather the information about the atoms.  Returns the number of errors.

 */

int collate_atoms(const Descriptor* descriptor, Atoms* atoms);

int collate_atoms(const Descriptor* descriptor, const string& moduleName, Atoms* atoms);

int collate_atom(const Descriptor *atom, AtomDecl *atomDecl, vector<java_type_t> *signature);



}  // namespace stats_log_api_gen

tools/stats_log_api_gen/java_writer.cpp

+14 −22
Original line number Diff line number Diff line
@@ -23,10 +23,8 @@ namespace stats_log_api_gen { 


static int write_java_q_logger_class(

        FILE* out,

        const map<vector<java_type_t>, set<string>>& signatures_to_modules,

        const AtomDecl &attributionDecl,

        const string& moduleName

        ) {

        const map<vector<java_type_t>, FieldNumberToAnnotations>& signatureInfoMap,

        const AtomDecl &attributionDecl) {

    fprintf(out, "\n");

    fprintf(out, "    // Write logging helper methods for statsd in Q and earlier.\n");

    fprintf(out, "    private static class QLogger {\n");
@@ -37,7 +35,7 @@ static int write_java_q_logger_class( 
    fprintf(out, "\n");

    fprintf(out, "        // Write methods.\n");

    write_java_methods_q_schema(

            out, signatures_to_modules, attributionDecl, moduleName, "        ");

            out, signatureInfoMap, attributionDecl, "        ");



    fprintf(out, "    }\n");

    return 0;
@@ -46,21 +44,15 @@ static int write_java_q_logger_class( 


static int write_java_methods(

        FILE* out,

        const map<vector<java_type_t>, set<string>>& signatures_to_modules,

        const map<vector<java_type_t>, FieldNumberToAnnotations>& signatureInfoMap,

        const AtomDecl &attributionDecl,

        const string& moduleName,

        const bool supportQ

        ) {

    for (auto signature_to_modules_it = signatures_to_modules.begin();

            signature_to_modules_it != signatures_to_modules.end(); signature_to_modules_it++) {

        // Skip if this signature is not needed for the module.

        if (!signature_needed_for_module(signature_to_modules_it->second, moduleName)) {

            continue;

        }



    for (auto signatureInfoMapIt = signatureInfoMap.begin();

            signatureInfoMapIt != signatureInfoMap.end(); signatureInfoMapIt++) {

        // Print method signature.

        fprintf(out, "    public static void write(int code");

        vector<java_type_t> signature = signature_to_modules_it->first;

        vector<java_type_t> signature = signatureInfoMapIt->first;

        int argIndex = 1;

        for (vector<java_type_t>::const_iterator arg = signature.begin();

                arg != signature.end(); arg++) {
@@ -249,7 +241,7 @@ static int write_java_methods( 
}



int write_stats_log_java(FILE* out, const Atoms& atoms, const AtomDecl &attributionDecl,

                                    const string& moduleName, const string& javaClass,

                                    const string& javaClass,

                                    const string& javaPackage, const bool supportQ,

                                    const bool supportWorkSource) {

    // Print prelude
@@ -273,24 +265,24 @@ int write_stats_log_java(FILE* out, const Atoms& atoms, const AtomDecl &attribut 
    fprintf(out, " */\n");

    fprintf(out, "public class %s {\n", javaClass.c_str());



    write_java_atom_codes(out, atoms, moduleName);

    write_java_enum_values(out, atoms, moduleName);

    write_java_atom_codes(out, atoms);

    write_java_enum_values(out, atoms);



    int errors = 0;



    // Print write methods.

    fprintf(out, "    // Write methods\n");

    errors += write_java_methods(

            out, atoms.signatures_to_modules, attributionDecl, moduleName, supportQ);

            out, atoms.signatureInfoMap, attributionDecl, supportQ);

    errors += write_java_non_chained_methods(

            out, atoms.non_chained_signatures_to_modules, moduleName);

            out, atoms.nonChainedSignatureInfoMap);

    if (supportWorkSource) {

        errors += write_java_work_source_methods(out, atoms.signatures_to_modules, moduleName);

        errors += write_java_work_source_methods(out, atoms.signatureInfoMap);

    }



    if (supportQ) {

        errors += write_java_q_logger_class(

                out, atoms.signatures_to_modules, attributionDecl, moduleName);

                out, atoms.signatureInfoMap, attributionDecl);

    }



    fprintf(out, "}\n");

tools/stats_log_api_gen/java_writer.h

+1 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ namespace stats_log_api_gen { 
using namespace std;



int write_stats_log_java(FILE* out, const Atoms& atoms, const AtomDecl &attributionDecl,

                         const string& moduleName, const string& javaClass,

                         const string& javaClass,

                         const string& javaPackage, const bool supportQ,

                         const bool supportWorkSource);

tools/stats_log_api_gen/java_writer_q.cpp

+11 −18
Original line number Diff line number Diff line
@@ -51,20 +51,14 @@ void write_java_q_logging_constants(FILE* out, const string& indent) { 


int write_java_methods_q_schema(

        FILE* out,

        const map<vector<java_type_t>, set<string>>& signatures_to_modules,

        const map<vector<java_type_t>, FieldNumberToAnnotations>& signatureInfoMap,

        const AtomDecl &attributionDecl,

        const string& moduleName,

        const string& indent) {

    int requiredHelpers = 0;

    for (auto signature_to_modules_it = signatures_to_modules.begin();

            signature_to_modules_it != signatures_to_modules.end(); signature_to_modules_it++) {

        // Skip if this signature is not needed for the module.

        if (!signature_needed_for_module(signature_to_modules_it->second, moduleName)) {

            continue;

        }



    for (auto signatureInfoMapIt = signatureInfoMap.begin();

            signatureInfoMapIt != signatureInfoMap.end(); signatureInfoMapIt++) {

        // Print method signature.

        vector<java_type_t> signature = signature_to_modules_it->first;

        vector<java_type_t> signature = signatureInfoMapIt->first;

        fprintf(out, "%spublic static void write(int code", indent.c_str());

        int argIndex = 1;

        for (vector<java_type_t>::const_iterator arg = signature.begin();
@@ -568,7 +562,7 @@ void write_java_helpers_for_q_schema_methods( 
// This method is called in main.cpp to generate StatsLog for modules that's compatible with

// Q at compile-time.

int write_stats_log_java_q_for_module(FILE* out, const Atoms& atoms,

                                      const AtomDecl &attributionDecl, const string& moduleName,

                                      const AtomDecl &attributionDecl,

                                      const string& javaClass, const string& javaPackage,

                                      const bool supportWorkSource) {

    // Print prelude
@@ -589,19 +583,18 @@ int write_stats_log_java_q_for_module(FILE* out, const Atoms& atoms, 


    write_java_q_logging_constants(out, "    ");



    write_java_atom_codes(out, atoms, moduleName);

    write_java_atom_codes(out, atoms);



    write_java_enum_values(out, atoms, moduleName);

    write_java_enum_values(out, atoms);



    int errors = 0;

    // Print write methods

    fprintf(out, "    // Write methods\n");

    errors += write_java_methods_q_schema(out, atoms.signatures_to_modules, attributionDecl,

            moduleName, "    ");

    errors += write_java_non_chained_methods(out, atoms.non_chained_signatures_to_modules,

            moduleName);

    errors += write_java_methods_q_schema(out, atoms.signatureInfoMap, attributionDecl,

            "    ");

    errors += write_java_non_chained_methods(out, atoms.nonChainedSignatureInfoMap);

    if (supportWorkSource) {

        errors += write_java_work_source_methods(out, atoms.signatures_to_modules, moduleName);

        errors += write_java_work_source_methods(out, atoms.signatureInfoMap);

    }



    fprintf(out, "}\n");