Add Python 3 Binding Generation via Pybind11 am: 9e41853360 (55e58de5) · Commits · e / os / android_packages_modules_Bluetooth

system/gd/Android.bp

+102 −0

Original line number	Diff line number	Diff line
		@@ -358,6 +358,24 @@ genrule {
		],
		}

		genrule {
		name: "BluetoothGeneratedPackets_python3_cc",
		tools: [
		"bluetooth_packetgen",
		],
		cmd: "$(location bluetooth_packetgen) --include=packages/modules/Bluetooth/system/gd --out=$(genDir) $(in)",
		srcs: [
		"hci/hci_packets.pdl",
		"l2cap/l2cap_packets.pdl",
		"security/smp_packets.pdl",
		],
		out: [
		"hci/hci_packets_python3.cc",
		"l2cap/l2cap_packets_python3.cc",
		"security/smp_packets_python3.cc",
		],
		}

		filegroup {
		name: "BluetoothFacadeProto",
		srcs: [
		@@ -523,3 +541,87 @@ genrule {
		"l2cap/classic/cert/api.pb.cc",
		],
		}

		cc_defaults {
		name: "bluetooth_py3_native_extension_defaults",
		include_dirs: [
		"external/python/cpython3/Include",
		],
		target: {
		android: {
		include_dirs: ["external/python/cpython3/android/bionic/pyconfig"],
		},
		android_arm: {
		cflags: ["-DSOABI=\"cpython-38android-arm-android-bionic\""],
		suffix: ".cpython-38android-arm-android-bionic",
		},
		android_arm64: {
		cflags: ["-DSOABI=\"cpython-38android-arm64-android-bionic\""],
		suffix: ".cpython-38android-arm64-android-bionic",
		},
		android_x86: {
		cflags: ["-DSOABI=\"cpython-38android-x86-android-bionic\""],
		suffix: ".cpython-38android-x86-android-bionic",
		},
		android_x86_64: {
		cflags: ["-DSOABI=\"cpython-38android-x86_64-android-bionic\""],
		suffix: ".cpython-38android-x86_64-android-bionic",
		},
		// Regenerate include dirs with android_regen.sh
		darwin_x86_64: {
		include_dirs: ["external/python/cpython3/android/darwin_x86_64/pyconfig"],
		cflags: [
		"-Wno-deprecated-declarations",
		"-Wno-pointer-arith",
		"-DSOABI=\"cpython-38android-x86_64-darwin\"",
		],
		suffix: ".cpython-38android-x86_64-darwin",
		},
		linux_bionic: {
		// NB linux_bionic is a 'host' architecture but it uses the bionic libc like 'android'
		// targets so use the android pyconfig.
		include_dirs: ["external/python/cpython3/android/bionic/pyconfig"],
		cflags: ["-DSOABI=\"cpython-38android-x86_64-linux-bionic\""],
		suffix: ".cpython-38android-x86_64-linux-bionic",
		},
		linux_glibc_x86: {
		enabled: false,
		},
		linux_glibc_x86_64: {
		include_dirs: ["external/python/cpython3/android/linux_x86_64/pyconfig"],
		cflags: ["-DSOABI=\"cpython-38android-x86_64-linux-gnu\""],
		suffix: ".cpython-38android-x86_64-linux-gnu",
		},
		windows: {
		enabled: false,
		},
		},
		allow_undefined_symbols: true,
		}

		cc_library{
		name: "bluetooth_packets_python3",
		defaults: [
		"gd_defaults",
		"bluetooth_py3_native_extension_defaults"
		],
		host_supported: true,
		srcs: [
		"packet/python3_module.cc",
		"l2cap/fcs.cc",
		":BluetoothPacketSources",
		],
		generated_headers: [
		"BluetoothGeneratedPackets_h",
		],
		generated_sources: [
		"BluetoothGeneratedPackets_python3_cc",
		],
		header_libs: [
		"pybind11_headers",
		],
		cflags: [
		"-fexceptions",
		],
		rtti: true,
		}

system/gd/packet/parser/enum_gen.cc

+9 −1

Original line number	Diff line number	Diff line
		@@ -20,7 +20,7 @@

		#include "util.h"

		EnumGen::EnumGen(EnumDef e) : e_(e) {}
		EnumGen::EnumGen(EnumDef e) : e_(std::move(e)) {}

		void EnumGen::GenDefinition(std::ostream& stream) {
		stream << "enum class ";
		@@ -33,6 +33,14 @@ void EnumGen::GenDefinition(std::ostream& stream) {
		stream << "};\n";
		}

		void EnumGen::GenDefinitionPybind11(std::ostream& stream) {
		stream << "py::enum_<" << e_.name_ << ">(m, \"" << e_.name_ << "\")";
		for (const auto& pair : e_.constants_) {
		stream << ".value(\"" << pair.second << "\", " << e_.name_ << "::" << pair.second << ")";
		}
		stream << ";\n";
		}

		void EnumGen::GenLogging(std::ostream& stream) {
		// Print out the switch statement that converts all the constants to strings.
		stream << "inline std::string " << e_.name_ << "Text(const " << e_.name_ << "& param) {";

system/gd/packet/parser/enum_gen.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -27,6 +27,8 @@ class EnumGen {

		void GenDefinition(std::ostream& stream);

		void GenDefinitionPybind11(std::ostream& stream);

		void GenLogging(std::ostream& stream);

		EnumDef e_;

system/gd/packet/parser/main.cc

+98 −0

Original line number	Diff line number	Diff line
		@@ -230,6 +230,100 @@ bool generate_cpp_headers_one_file(const Declarations& decls, const std::filesys
		return true;
		}

		bool generate_pybind11_sources_one_file(const Declarations& decls, const std::filesystem::path& input_file,
		const std::filesystem::path& include_dir, const std::filesystem::path& out_dir,
		const std::string& root_namespace) {
		auto gen_relative_path = input_file.lexically_relative(include_dir).parent_path();

		auto input_filename = input_file.filename().string().substr(0, input_file.filename().string().find(".pdl"));
		auto gen_path = out_dir / gen_relative_path;

		std::filesystem::create_directories(gen_path);

		auto gen_relative_header = gen_relative_path / (input_filename + ".h");
		auto gen_file = gen_path / (input_filename + "_python3.cc");

		std::ofstream out_file;
		out_file.open(gen_file);
		if (!out_file.is_open()) {
		std::cerr << "can't open " << gen_file << std::endl;
		return false;
		}

		out_file << "#include <pybind11/pybind11.h>\n";
		out_file << "#include <pybind11/stl.h>\n";
		out_file << "\n\n";
		out_file << "#include " << gen_relative_header << "\n";
		out_file << "\n\n";

		std::vector<std::string> namespace_list;
		parse_namespace(root_namespace, gen_relative_path, &namespace_list);
		generate_namespace_open(namespace_list, out_file);
		out_file << "\n\n";

		for (const auto& c : decls.type_defs_queue_) {
		if (c.second->GetDefinitionType() == TypeDef::Type::CUSTOM \|\|
		c.second->GetDefinitionType() == TypeDef::Type::CHECKSUM) {
		const auto* custom_def = dynamic_cast<const CustomFieldDef*>(c.second);
		custom_def->GenUsing(out_file);
		}
		}
		out_file << "\n\n";

		out_file << "using ::bluetooth::packet::BasePacketBuilder;";
		out_file << "using ::bluetooth::packet::BitInserter;";
		out_file << "using ::bluetooth::packet::CustomTypeChecker;";
		out_file << "using ::bluetooth::packet::Iterator;";
		out_file << "using ::bluetooth::packet::kLittleEndian;";
		out_file << "using ::bluetooth::packet::PacketBuilder;";
		out_file << "using ::bluetooth::packet::BaseStruct;";
		out_file << "using ::bluetooth::packet::PacketStruct;";
		out_file << "using ::bluetooth::packet::PacketView;";
		out_file << "using ::bluetooth::packet::parser::ChecksumTypeChecker;";
		out_file << "\n\n";

		out_file << "namespace py = pybind11;\n\n";

		out_file << "void define_" << input_filename << "_submodule(py::module& parent) {\n\n";
		out_file << "py::module m = parent.def_submodule(\"" << input_filename << "\", \"A submodule of " << input_filename
		<< "\");\n\n";

		for (const auto& e : decls.type_defs_queue_) {
		if (e.second->GetDefinitionType() == TypeDef::Type::ENUM) {
		const auto* enum_def = dynamic_cast<const EnumDef*>(e.second);
		EnumGen gen(*enum_def);
		gen.GenDefinitionPybind11(out_file);
		out_file << "\n\n";
		}
		}

		for (const auto& s : decls.type_defs_queue_) {
		if (s.second->GetDefinitionType() == TypeDef::Type::STRUCT) {
		const auto* struct_def = dynamic_cast<const StructDef*>(s.second);
		struct_def->GenDefinitionPybind11(out_file);
		out_file << "\n";
		}
		}

		for (const auto& packet_def : decls.packet_defs_queue_) {
		packet_def.second.GenParserDefinitionPybind11(out_file);
		out_file << "\n\n";
		}

		for (const auto& p : decls.packet_defs_queue_) {
		p.second.GenBuilderDefinitionPybind11(out_file);
		out_file << "\n\n";
		}

		out_file << "}\n\n";

		generate_namespace_close(namespace_list, out_file);

		out_file.close();

		return true;
		}

		} // namespace

		// TODO(b/141583809): stop leaks
		@@ -273,6 +367,10 @@ int main(int argc, const char** argv) {
		std::cerr << "Didn't generate cpp headers for " << input_files.front() << std::endl;
		return 3;
		}
		if (!generate_pybind11_sources_one_file(declarations, input_files.front(), include_dir, out_dir, root_namespace)) {
		std::cerr << "Didn't generate pybind11 sources for " << input_files.front() << std::endl;
		return 4;
		}
		input_files.pop();
		}

system/gd/packet/parser/packet_def.cc

+134 −0

Original line number	Diff line number	Diff line
		@@ -84,6 +84,42 @@ void PacketDef::GenParserDefinition(std::ostream& s) const {
		s << "};\n";
		}

		void PacketDef::GenParserDefinitionPybind11(std::ostream& s) const {
		s << "py::class_<" << name_ << "View";
		if (parent_ != nullptr) {
		s << ", " << parent_->name_ << "View";
		} else {
		s << ", PacketView<" << (is_little_endian_ ? "" : "!") << "kLittleEndian>";
		}
		s << ">(m, \"" << name_ << "View\")";
		if (parent_ != nullptr) {
		s << ".def(py::init([](" << parent_->name_ << "View parent) {";
		} else {
		s << ".def(py::init([](PacketView<" << (is_little_endian_ ? "" : "!") << "kLittleEndian> parent) {";
		}
		s << "auto view =" << name_ << "View::Create(std::move(parent));";
		s << "if (!view.IsValid()) { throw std::invalid_argument(\"Bad packet view\"); }";
		s << "return view; }))";

		s << ".def(py::init(&" << name_ << "View::Create))";
		std::set<std::string> protected_field_types = {
		FixedScalarField::kFieldType,
		FixedEnumField::kFieldType,
		SizeField::kFieldType,
		CountField::kFieldType,
		};
		const auto& public_fields = fields_.GetFieldsWithoutTypes(protected_field_types);
		for (const auto& field : public_fields) {
		auto getter_func_name = field->GetGetterFunctionName();
		if (getter_func_name.empty()) {
		continue;
		}
		s << ".def(\"" << getter_func_name << "\", &" << name_ << "View::" << getter_func_name << ")";
		}
		s << ".def(\"IsValid\", &" << name_ << "View::IsValid)";
		s << ";\n";
		}

		void PacketDef::GenParserFieldGetter(std::ostream& s, const PacketField* field) const {
		// Start field offset
		auto start_field_offset = GetOffsetForField(field->GetName(), false);
		@@ -290,6 +326,29 @@ void PacketDef::GenBuilderDefinition(std::ostream& s) const {
		s << "\n";
		}

		void PacketDef::GenBuilderDefinitionPybind11(std::ostream& s) const {
		s << "py::class_<" << name_ << "Builder";
		if (parent_ != nullptr) {
		s << ", " << parent_->name_ << "Builder";
		} else {
		if (is_little_endian_) {
		s << ", PacketBuilder<kLittleEndian>";
		} else {
		s << ", PacketBuilder<!kLittleEndian>";
		}
		}
		s << ">(m, \"" << name_ << "Builder\")";
		if (!fields_.HasBody()) {
		GenBuilderCreatePybind11(s);
		}
		s << ".def(\"Serialize\", [](" << name_ << "Builder& builder){";
		s << "std::vector<uint8_t> bytes;";
		s << "BitInserter bi(bytes);";
		s << "builder.Serialize(bi);";
		s << "return bytes;})";
		s << ";\n";
		}

		void PacketDef::GenTestDefine(std::ostream& s) const {
		s << "#ifdef PACKET_TESTING\n";
		s << "#define DEFINE_AND_INSTANTIATE_" << name_ << "ReflectionTest(...)";
		@@ -433,6 +492,81 @@ void PacketDef::GenBuilderCreate(std::ostream& s) const {
		s << "}\n";
		}

		void PacketDef::GenBuilderCreatePybind11(std::ostream& s) const {
		s << ".def(py::init([](";
		auto params = GetParamList();
		std::vector<std::string> constructor_args;
		std::vector<std::string> keep_alive_args;
		int i = 1;
		for (const auto& param : params) {
		i++;
		std::stringstream ss;
		auto param_type = param->GetBuilderParameterType();
		if (param_type.empty()) {
		continue;
		}
		// Use shared_ptr instead of unique_ptr for the Python interface
		if (param->BuilderParameterMustBeMoved()) {
		param_type = util::StringFindAndReplaceAll(param_type, "unique_ptr", "shared_ptr");
		keep_alive_args.push_back(std::to_string(i));
		}
		ss << param_type << " " << param->GetName();
		constructor_args.push_back(ss.str());
		}
		s << util::StringJoin(",", constructor_args) << "){";

		// Deal with move only args
		for (const auto& param : params) {
		std::stringstream ss;
		auto param_type = param->GetBuilderParameterType();
		if (param_type.empty()) {
		continue;
		}
		if (!param->BuilderParameterMustBeMoved()) {
		continue;
		}
		auto move_only_param_name = param->GetName() + "_move_only";
		s << param_type << " " << move_only_param_name << ";";
		if (param->IsContainerField()) {
		// Assume single layer container
		s << "for (size_t i = 0; i < " << param->GetName() << ".size(); i++) {";
		if (param->GetFieldType() == VectorField::kFieldType) {
		s << move_only_param_name << ".emplace_back(" << param->GetName() << "[i].get());";
		} else if (param->GetFieldType() == ArrayField::kFieldType) {
		s << move_only_param_name << "[i].reset(" << param->GetName() << "[i].get());";
		} else {
		ERROR() << param << " is not supported by Pybind11";
		}
		s << "}";
		} else {
		// Release shared_ptr to unique_ptr and leave the Python copy as nullptr and to be garbage collected by Python
		s << move_only_param_name << ".reset(" << param->GetName() << ".get());";
		}
		}
		s << "return " << name_ << "Builder::Create(";
		std::vector<std::string> builder_vars;
		for (const auto& param : params) {
		std::stringstream ss;
		auto param_type = param->GetBuilderParameterType();
		if (param_type.empty()) {
		continue;
		}
		auto param_name = param->GetName();
		if (param->BuilderParameterMustBeMoved()) {
		ss << "std::move(" << param_name << "_move_only)";
		} else {
		ss << param_name;
		}
		builder_vars.push_back(ss.str());
		}
		s << util::StringJoin(",", builder_vars) << ");}";
		if (keep_alive_args.empty()) {
		s << "))";
		} else {
		s << "), py::keep_alive<1," << util::StringJoin(",", keep_alive_args) << ">())";
		}
		}

		void PacketDef::GenBuilderParameterChecker(std::ostream& s) const {
		FieldList params_to_validate = GetParametersToValidate();