Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9e418533 authored by Jack He's avatar Jack He
Browse files

Add Python 3 Binding Generation via Pybind11 

* Created entry point packet/python3_module.cc to create
  bluetooth_packets_python3 Python module. Layer specific
  packets are submodules of this top layer module
* Define shim layer for Builder::Create() functions to
  use a std::shared_ptr instead of std::unique_ptr as
  Python does not support explicit ownership transfer
* Add py::keep_alive to make sure the shared_ptr does not
  get garbage collected until the returned object is garbage
  collected
* Redefine PacketView::Create() method as a constructor for packet view
  and throws value errors when packet is in valid so that
  users don't have to call IsValid()
* Redefine Builder::Serialize() method to return a Python
  list of bytes directly instead of using BitInserter.
  Python binding users need to manually concatenate
  sub-sections of generated packets to form a bigger packet
* Add build rules to generate and build the Python 3 library

Example:

>>> import bluetooth_packets_python3 as bp3
>>> write_ssp_enable_builder = bp3.hci_packets.WriteSimplePairingModeBuilder(bp3.hci_packets.Enable.ENABLED)
>>> write_ssp_enable_bytes = write_ssp_enable_builder.Serialize()
>>> list(map(lambda b : hex(b), write_ssp_enable_bytes))
['0x56', '0xc', '0x1', '0x1']
>>> write_ssp_enable_pkt_view = bp3.PacketViewLittleEndian(write_ssp_enable_bytes)
>>> write_ssp_enable_view = bp3.hci_packets.WriteSimplePairingModeView(bp3.hci_packets.SecurityCommandView(bp3.hci_packets.CommandPacketView(write_ssp_enable_pkt_view)))
>>> write_ssp_enable_view.GetOpCode()
OpCode.WRITE_SIMPLE_PAIRING_MODE
>>> write_ssp_enable_view.GetSimplePairingMode()
Enable.ENABLED

Bug: 143374372
Test: make and run locally
Change-Id: I8775c2d26a393aa5c1a417b7f845f7102fe4120e
parent 50524992

system/gd/Android.bp

+102 −0
Original line number Diff line number Diff line
@@ -358,6 +358,24 @@ genrule { 
    ],

}



genrule {

    name: "BluetoothGeneratedPackets_python3_cc",

    tools: [

        "bluetooth_packetgen",

    ],

    cmd: "$(location bluetooth_packetgen) --include=packages/modules/Bluetooth/system/gd --out=$(genDir) $(in)",

    srcs: [

        "hci/hci_packets.pdl",

        "l2cap/l2cap_packets.pdl",

        "security/smp_packets.pdl",

    ],

    out: [

        "hci/hci_packets_python3.cc",

        "l2cap/l2cap_packets_python3.cc",

        "security/smp_packets_python3.cc",

    ],

}



filegroup {

    name: "BluetoothFacadeProto",

    srcs: [
@@ -523,3 +541,87 @@ genrule { 
        "l2cap/classic/cert/api.pb.cc",

    ],

}



cc_defaults {

  name: "bluetooth_py3_native_extension_defaults",

  include_dirs: [

    "external/python/cpython3/Include",

  ],

  target: {

      android: {

          include_dirs: ["external/python/cpython3/android/bionic/pyconfig"],

      },

      android_arm: {

          cflags: ["-DSOABI=\"cpython-38android-arm-android-bionic\""],

          suffix: ".cpython-38android-arm-android-bionic",

      },

      android_arm64: {

          cflags: ["-DSOABI=\"cpython-38android-arm64-android-bionic\""],

          suffix: ".cpython-38android-arm64-android-bionic",

      },

      android_x86: {

          cflags: ["-DSOABI=\"cpython-38android-x86-android-bionic\""],

          suffix: ".cpython-38android-x86-android-bionic",

      },

      android_x86_64: {

          cflags: ["-DSOABI=\"cpython-38android-x86_64-android-bionic\""],

          suffix: ".cpython-38android-x86_64-android-bionic",

      },

      // Regenerate include dirs with android_regen.sh

      darwin_x86_64: {

          include_dirs: ["external/python/cpython3/android/darwin_x86_64/pyconfig"],

          cflags: [

              "-Wno-deprecated-declarations",

              "-Wno-pointer-arith",

              "-DSOABI=\"cpython-38android-x86_64-darwin\"",

          ],

          suffix: ".cpython-38android-x86_64-darwin",

      },

      linux_bionic: {

          // NB linux_bionic is a 'host' architecture but it uses the bionic libc like 'android'

          // targets so use the android pyconfig.

          include_dirs: ["external/python/cpython3/android/bionic/pyconfig"],

          cflags: ["-DSOABI=\"cpython-38android-x86_64-linux-bionic\""],

          suffix: ".cpython-38android-x86_64-linux-bionic",

      },

      linux_glibc_x86: {

          enabled: false,

      },

      linux_glibc_x86_64: {

          include_dirs: ["external/python/cpython3/android/linux_x86_64/pyconfig"],

          cflags: ["-DSOABI=\"cpython-38android-x86_64-linux-gnu\""],

          suffix: ".cpython-38android-x86_64-linux-gnu",

      },

      windows: {

          enabled: false,

      },

  },

  allow_undefined_symbols: true,

}



cc_library{

  name: "bluetooth_packets_python3",

  defaults: [

    "gd_defaults",

    "bluetooth_py3_native_extension_defaults"

  ],

  host_supported: true,

  srcs: [

    "packet/python3_module.cc",

    "l2cap/fcs.cc",

    ":BluetoothPacketSources",

  ],

  generated_headers: [

    "BluetoothGeneratedPackets_h",

  ],

  generated_sources: [

    "BluetoothGeneratedPackets_python3_cc",

  ],

  header_libs: [

    "pybind11_headers",

  ],

  cflags: [

    "-fexceptions",

  ],

  rtti: true,

}

system/gd/packet/parser/enum_gen.cc

+9 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 


#include "util.h"



EnumGen::EnumGen(EnumDef e) : e_(e) {}

EnumGen::EnumGen(EnumDef e) : e_(std::move(e)) {}



void EnumGen::GenDefinition(std::ostream& stream) {

  stream << "enum class ";
@@ -33,6 +33,14 @@ void EnumGen::GenDefinition(std::ostream& stream) { 
  stream << "};\n";

}



void EnumGen::GenDefinitionPybind11(std::ostream& stream) {

  stream << "py::enum_<" << e_.name_ << ">(m, \"" << e_.name_ << "\")";

  for (const auto& pair : e_.constants_) {

    stream << ".value(\"" << pair.second << "\", " << e_.name_ << "::" << pair.second << ")";

  }

  stream << ";\n";

}



void EnumGen::GenLogging(std::ostream& stream) {

  // Print out the switch statement that converts all the constants to strings.

  stream << "inline std::string " << e_.name_ << "Text(const " << e_.name_ << "& param) {";

system/gd/packet/parser/enum_gen.h

+2 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ class EnumGen { 


  void GenDefinition(std::ostream& stream);



  void GenDefinitionPybind11(std::ostream& stream);



  void GenLogging(std::ostream& stream);



  EnumDef e_;

system/gd/packet/parser/main.cc

+98 −0
Original line number Diff line number Diff line
@@ -230,6 +230,100 @@ bool generate_cpp_headers_one_file(const Declarations& decls, const std::filesys 
  return true;

}



bool generate_pybind11_sources_one_file(const Declarations& decls, const std::filesystem::path& input_file,

                                        const std::filesystem::path& include_dir, const std::filesystem::path& out_dir,

                                        const std::string& root_namespace) {

  auto gen_relative_path = input_file.lexically_relative(include_dir).parent_path();



  auto input_filename = input_file.filename().string().substr(0, input_file.filename().string().find(".pdl"));

  auto gen_path = out_dir / gen_relative_path;



  std::filesystem::create_directories(gen_path);



  auto gen_relative_header = gen_relative_path / (input_filename + ".h");

  auto gen_file = gen_path / (input_filename + "_python3.cc");



  std::ofstream out_file;

  out_file.open(gen_file);

  if (!out_file.is_open()) {

    std::cerr << "can't open " << gen_file << std::endl;

    return false;

  }



  out_file << "#include <pybind11/pybind11.h>\n";

  out_file << "#include <pybind11/stl.h>\n";

  out_file << "\n\n";

  out_file << "#include " << gen_relative_header << "\n";

  out_file << "\n\n";



  std::vector<std::string> namespace_list;

  parse_namespace(root_namespace, gen_relative_path, &namespace_list);

  generate_namespace_open(namespace_list, out_file);

  out_file << "\n\n";



  for (const auto& c : decls.type_defs_queue_) {

    if (c.second->GetDefinitionType() == TypeDef::Type::CUSTOM ||

        c.second->GetDefinitionType() == TypeDef::Type::CHECKSUM) {

      const auto* custom_def = dynamic_cast<const CustomFieldDef*>(c.second);

      custom_def->GenUsing(out_file);

    }

  }

  out_file << "\n\n";



  out_file << "using ::bluetooth::packet::BasePacketBuilder;";

  out_file << "using ::bluetooth::packet::BitInserter;";

  out_file << "using ::bluetooth::packet::CustomTypeChecker;";

  out_file << "using ::bluetooth::packet::Iterator;";

  out_file << "using ::bluetooth::packet::kLittleEndian;";

  out_file << "using ::bluetooth::packet::PacketBuilder;";

  out_file << "using ::bluetooth::packet::BaseStruct;";

  out_file << "using ::bluetooth::packet::PacketStruct;";

  out_file << "using ::bluetooth::packet::PacketView;";

  out_file << "using ::bluetooth::packet::parser::ChecksumTypeChecker;";

  out_file << "\n\n";



  out_file << "namespace py = pybind11;\n\n";



  out_file << "void define_" << input_filename << "_submodule(py::module& parent) {\n\n";

  out_file << "py::module m = parent.def_submodule(\"" << input_filename << "\", \"A submodule of " << input_filename

           << "\");\n\n";



  for (const auto& e : decls.type_defs_queue_) {

    if (e.second->GetDefinitionType() == TypeDef::Type::ENUM) {

      const auto* enum_def = dynamic_cast<const EnumDef*>(e.second);

      EnumGen gen(*enum_def);

      gen.GenDefinitionPybind11(out_file);

      out_file << "\n\n";

    }

  }



  for (const auto& s : decls.type_defs_queue_) {

    if (s.second->GetDefinitionType() == TypeDef::Type::STRUCT) {

      const auto* struct_def = dynamic_cast<const StructDef*>(s.second);

      struct_def->GenDefinitionPybind11(out_file);

      out_file << "\n";

    }

  }



  for (const auto& packet_def : decls.packet_defs_queue_) {

    packet_def.second.GenParserDefinitionPybind11(out_file);

    out_file << "\n\n";

  }



  for (const auto& p : decls.packet_defs_queue_) {

    p.second.GenBuilderDefinitionPybind11(out_file);

    out_file << "\n\n";

  }



  out_file << "}\n\n";



  generate_namespace_close(namespace_list, out_file);



  out_file.close();



  return true;

}



}  // namespace



// TODO(b/141583809): stop leaks
@@ -273,6 +367,10 @@ int main(int argc, const char** argv) { 
      std::cerr << "Didn't generate cpp headers for " << input_files.front() << std::endl;

      return 3;

    }

    if (!generate_pybind11_sources_one_file(declarations, input_files.front(), include_dir, out_dir, root_namespace)) {

      std::cerr << "Didn't generate pybind11 sources for " << input_files.front() << std::endl;

      return 4;

    }

    input_files.pop();

  }

system/gd/packet/parser/packet_def.cc

+134 −0
Original line number Diff line number Diff line
@@ -84,6 +84,42 @@ void PacketDef::GenParserDefinition(std::ostream& s) const { 
  s << "};\n";

}



void PacketDef::GenParserDefinitionPybind11(std::ostream& s) const {

  s << "py::class_<" << name_ << "View";

  if (parent_ != nullptr) {

    s << ", " << parent_->name_ << "View";

  } else {

    s << ", PacketView<" << (is_little_endian_ ? "" : "!") << "kLittleEndian>";

  }

  s << ">(m, \"" << name_ << "View\")";

  if (parent_ != nullptr) {

    s << ".def(py::init([](" << parent_->name_ << "View parent) {";

  } else {

    s << ".def(py::init([](PacketView<" << (is_little_endian_ ? "" : "!") << "kLittleEndian> parent) {";

  }

  s << "auto view =" << name_ << "View::Create(std::move(parent));";

  s << "if (!view.IsValid()) { throw std::invalid_argument(\"Bad packet view\"); }";

  s << "return view; }))";



  s << ".def(py::init(&" << name_ << "View::Create))";

  std::set<std::string> protected_field_types = {

      FixedScalarField::kFieldType,

      FixedEnumField::kFieldType,

      SizeField::kFieldType,

      CountField::kFieldType,

  };

  const auto& public_fields = fields_.GetFieldsWithoutTypes(protected_field_types);

  for (const auto& field : public_fields) {

    auto getter_func_name = field->GetGetterFunctionName();

    if (getter_func_name.empty()) {

      continue;

    }

    s << ".def(\"" << getter_func_name << "\", &" << name_ << "View::" << getter_func_name << ")";

  }

  s << ".def(\"IsValid\", &" << name_ << "View::IsValid)";

  s << ";\n";

}



void PacketDef::GenParserFieldGetter(std::ostream& s, const PacketField* field) const {

  // Start field offset

  auto start_field_offset = GetOffsetForField(field->GetName(), false);
@@ -290,6 +326,29 @@ void PacketDef::GenBuilderDefinition(std::ostream& s) const { 
  s << "\n";

}



void PacketDef::GenBuilderDefinitionPybind11(std::ostream& s) const {

  s << "py::class_<" << name_ << "Builder";

  if (parent_ != nullptr) {

    s << ", " << parent_->name_ << "Builder";

  } else {

    if (is_little_endian_) {

      s << ", PacketBuilder<kLittleEndian>";

    } else {

      s << ", PacketBuilder<!kLittleEndian>";

    }

  }

  s << ">(m, \"" << name_ << "Builder\")";

  if (!fields_.HasBody()) {

    GenBuilderCreatePybind11(s);

  }

  s << ".def(\"Serialize\", [](" << name_ << "Builder& builder){";

  s << "std::vector<uint8_t> bytes;";

  s << "BitInserter bi(bytes);";

  s << "builder.Serialize(bi);";

  s << "return bytes;})";

  s << ";\n";

}



void PacketDef::GenTestDefine(std::ostream& s) const {

  s << "#ifdef PACKET_TESTING\n";

  s << "#define DEFINE_AND_INSTANTIATE_" << name_ << "ReflectionTest(...)";
@@ -433,6 +492,81 @@ void PacketDef::GenBuilderCreate(std::ostream& s) const { 
  s << "}\n";

}



void PacketDef::GenBuilderCreatePybind11(std::ostream& s) const {

  s << ".def(py::init([](";

  auto params = GetParamList();

  std::vector<std::string> constructor_args;

  std::vector<std::string> keep_alive_args;

  int i = 1;

  for (const auto& param : params) {

    i++;

    std::stringstream ss;

    auto param_type = param->GetBuilderParameterType();

    if (param_type.empty()) {

      continue;

    }

    // Use shared_ptr instead of unique_ptr for the Python interface

    if (param->BuilderParameterMustBeMoved()) {

      param_type = util::StringFindAndReplaceAll(param_type, "unique_ptr", "shared_ptr");

      keep_alive_args.push_back(std::to_string(i));

    }

    ss << param_type << " " << param->GetName();

    constructor_args.push_back(ss.str());

  }

  s << util::StringJoin(",", constructor_args) << "){";



  // Deal with move only args

  for (const auto& param : params) {

    std::stringstream ss;

    auto param_type = param->GetBuilderParameterType();

    if (param_type.empty()) {

      continue;

    }

    if (!param->BuilderParameterMustBeMoved()) {

      continue;

    }

    auto move_only_param_name = param->GetName() + "_move_only";

    s << param_type << " " << move_only_param_name << ";";

    if (param->IsContainerField()) {

      // Assume single layer container

      s << "for (size_t i = 0; i < " << param->GetName() << ".size(); i++) {";

      if (param->GetFieldType() == VectorField::kFieldType) {

        s << move_only_param_name << ".emplace_back(" << param->GetName() << "[i].get());";

      } else if (param->GetFieldType() == ArrayField::kFieldType) {

        s << move_only_param_name << "[i].reset(" << param->GetName() << "[i].get());";

      } else {

        ERROR() << param << " is not supported by Pybind11";

      }

      s << "}";

    } else {

      // Release shared_ptr to unique_ptr and leave the Python copy as nullptr and to be garbage collected by Python

      s << move_only_param_name << ".reset(" << param->GetName() << ".get());";

    }

  }

  s << "return " << name_ << "Builder::Create(";

  std::vector<std::string> builder_vars;

  for (const auto& param : params) {

    std::stringstream ss;

    auto param_type = param->GetBuilderParameterType();

    if (param_type.empty()) {

      continue;

    }

    auto param_name = param->GetName();

    if (param->BuilderParameterMustBeMoved()) {

      ss << "std::move(" << param_name << "_move_only)";

    } else {

      ss << param_name;

    }

    builder_vars.push_back(ss.str());

  }

  s << util::StringJoin(",", builder_vars) << ");}";

  if (keep_alive_args.empty()) {

    s << "))";

  } else {

    s << "), py::keep_alive<1," << util::StringJoin(",", keep_alive_args) << ">())";

  }

}



void PacketDef::GenBuilderParameterChecker(std::ostream& s) const {

  FieldList params_to_validate = GetParametersToValidate();