Merge "In Floss-build scripts, add support for using Docker." (f6ff986e) · Commits · e / os / android_packages_modules_Bluetooth

floss/build/README.md

+28 −16

Original line number	Diff line number	Diff line
		# Podman build for Floss
		# Container build for Floss

		This repo contains the Podman image build rule, used to generate the Podman
		image necessary to build Floss. If building a new Podman image, run
		`podman-build-image.py` with the tag `floss:latest`.
		This repo contains the Container-image build rule, used to generate the
		(docker/podman) container image necessary to build Floss. If building a new
		docker/podman image, run `container-build-image.py` with the tag `floss:latest`.

		## Setting up podman
		## Container binary: setting up podman (default)

		On most Debian based machines, you should be able to simply use apt-get and
		On most Debian based machines, you should be able to simply use `apt-get` and
		install these requisite packages.
		```
		sudo apt-get install \
		@@ -21,24 +21,36 @@ sudo usermod --add-subuids 10000-75535 $USERNAME
		sudo usermod --add-subgids 10000-75535 $USERNAME
		```

		## Generating the flossbuild image
		## Container binary: setting up docker (alternative)

		Follow the installation instructions at:
		https://docs.docker.com/engine/install/, such as
		https://docs.docker.com/engine/install/debian/.

		Also consider configuring Docker to run in rootless mode:
		https://docs.docker.com/engine/security/rootless/

		## Generating the floss-build image

		Run the following to generate the required image:
		```
		podman-build-image.py --tag floss:latest
		container-build-image.py --tag floss:latest
		```

		This uses the default tag of `flossbuild:latest` so you don't have to provide it
		specifically when invoking `build-in-podman.py`.
		If you use the `docker` binary, add the flag: `--use-docker` when running
		`container-build-image.py`.

		This uses the default tag of `floss:latest` so you don't have to provide it
		specifically when invoking `build-in-container.py`.

		## Using the podman image to build
		## Using the container image to build

		Once the Podman image is built (and assuming it's tagged as `floss:latest`), you
		should use the `build-in-podman.py` script to build the current repo.
		Once the container image is built (and assuming it's tagged as `floss:latest`), you
		should use the `build-in-container.py` script to build the current repo.

		Basic build:
		```
		build-in-podman.py
		build-in-container.py
		```

		This script will use the local `floss:latest` (or pull it from the registry),
		@@ -50,6 +62,6 @@ source to `/root/src` before running these commands in the container:
		* `./build.py --libdir=/usr/lib/x86-64_linux_gnu/`

		If you want to run the build more quickly (or pass other commands), run
		`build-in-podman.py --only-start`. This will only start the container for you
		`build-in-container.py --only-start`. This will only start the container for you
		(doing the correct mounts) and will print the commands it would have run via
		`podman exec` normally.
		`<container_binary> exec` normally.

floss/build/build-in-podman.py→floss/build/build-in-container.py

+31 −28

Original line number	Diff line number	Diff line
		@@ -10,8 +10,8 @@ SRC_MOUNT = "/root/src"
		STAGING_MOUNT = "/root/.floss"


		class FlossPodmanRunner:
		"""Runs Floss build inside podman container."""
		class FlossContainerRunner:
		"""Runs Floss build inside container."""

		# Commands to run for build
		BUILD_COMMANDS = [
		@@ -28,20 +28,22 @@ class FlossPodmanRunner:
		[f'{SRC_MOUNT}/build.py', '--target', 'test'],
		]

		def __init__(self, workdir, rootdir, image_tag, volume_name, container_name, staging_dir):
		def __init__(self, workdir, rootdir, image_tag, volume_name, container_name, staging_dir, use_docker):
		""" Constructor.

		Args:
		workdir: Current working directory (should be the script path).
		rootdir: Root directory for Bluetooth.
		image_tag: Tag for podman image used for building.
		image_tag: Tag for container image used for building.
		volume_name: Volume name used for storing artifacts.
		container_name: Name for running container instance.
		staging_dir: Directory to mount for artifacts instead of using volume.
		use_docker: Use docker binary if True (or podman when False).
		"""
		self.workdir = workdir
		self.rootdir = rootdir
		self.image_tag = image_tag
		self.container_binary = 'docker' if use_docker else 'podman'
		self.env = os.environ.copy()

		# Name of running container
		@@ -79,12 +81,12 @@ class FlossPodmanRunner:
		def _create_volume_if_needed(self):
		# Check if the volume exists. Otherwise create it.
		try:
		subprocess.check_output(['podman', 'volume', 'inspect', self.volume_name])
		subprocess.check_output([self.container_binary, 'volume', 'inspect', self.volume_name])
		except:
		self.run_command('podman volume create', ['podman', 'volume', 'create', self.volume_name])
		self.run_command(self.container_binary + ' volume create', [self.container_binary, 'volume', 'create', self.volume_name])

		def start_container(self):
		"""Starts the podman container with correct mounts."""
		"""Starts the container with correct mounts."""
		# Stop any previously started container.
		self.stop_container(ignore_error=True)

		@@ -99,17 +101,17 @@ class FlossPodmanRunner:
		# Mount the source directory
		mount_src_dir = 'type=bind,src={},dst={}'.format(self.rootdir, SRC_MOUNT)

		# Run the podman image. It will run `tail` indefinitely so the container
		# doesn't close and we can run `podman exec` on it.
		self.run_command('podman run', [
		'podman', 'run', '--name', self.container_name, '--mount', mount_output_volume, '--mount', mount_src_dir,
		# Run the container image. It will run `tail` indefinitely so the container
		# doesn't close and we can run `<container_binary> exec` on it.
		self.run_command(self.container_binary + ' run', [
		self.container_binary, 'run', '--name', self.container_name, '--mount', mount_output_volume, '--mount', mount_src_dir,
		'-d', self.image_tag, 'tail', '-f', '/dev/null'
		])

		def stop_container(self, ignore_error=False):
		"""Stops the podman container for build."""
		self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name], ignore_rc=ignore_error)
		self.run_command('podman rm', ['podman', 'rm', self.container_name], ignore_rc=ignore_error)
		"""Stops the container for build."""
		self.run_command(self.container_binary + ' stop', [self.container_binary, 'stop', '-t', '1', self.container_name], ignore_rc=ignore_error)
		self.run_command(self.container_binary + ' rm', [self.container_binary, 'rm', self.container_name], ignore_rc=ignore_error)

		def do_build(self):
		"""Runs the basic build commands."""
		@@ -119,21 +121,21 @@ class FlossPodmanRunner:
		try:
		# Run all commands
		for i, cmd in enumerate(self.BUILD_COMMANDS):
		self.run_command('podman exec #{}'.format(i), ['podman', 'exec', '-it', self.container_name] + cmd)
		self.run_command(self.container_binary + ' exec #{}'.format(i), [self.container_binary, 'exec', '-it', self.container_name] + cmd)
		finally:
		# Always stop container before exiting
		self.stop_container()

		def print_do_build(self):
		"""Prints the commands for building."""
		podman_exec = ['podman', 'exec', '-it', self.container_name]
		container_exec = [self.container_binary, 'exec', '-it', self.container_name]
		print('Normally, build would run the following commands: \n')
		for cmd in self.BUILD_COMMANDS:
		print(' '.join(podman_exec + cmd))
		print(' '.join(container_exec + cmd))

		def check_podman_runnable(self):
		def check_container_runnable(self):
		try:
		subprocess.check_output(['podman', 'ps'], stderr=subprocess.STDOUT)
		subprocess.check_output([self.container_binary, 'ps'], stderr=subprocess.STDOUT)
		except subprocess.CalledProcessError as err:
		if 'denied' in err.output.decode('utf-8'):
		print('Run script as sudo')
		@@ -142,19 +144,19 @@ class FlossPodmanRunner:

		return False

		# No exception means podman is ok
		# No exception means container is ok
		return True


		if __name__ == "__main__":
		parser = argparse.ArgumentParser('Builder Floss inside podman image.')
		parser = argparse.ArgumentParser('Builder Floss inside container image.')
		parser.add_argument(
		'--only-start',
		action='store_true',
		default=False,
		help='Only start the container. Prints the commands it would have ran.')
		parser.add_argument('--only-stop', action='store_true', default=False, help='Only stop the container and exit.')
		parser.add_argument('--image-tag', default='floss:latest', help='Podman image to use to build.')
		parser.add_argument('--image-tag', default='floss:latest', help='Container image to use to build.')
		parser.add_argument(
		'--volume-tag',
		default='floss-out',
		@@ -163,21 +165,22 @@ if __name__ == "__main__":
		'--staging-dir',
		default=None,
		help='Staging directory to use instead of volume. Build artifacts will be written here.')
		parser.add_argument('--container-name', default='floss-podman-runner', help='What to name the started container')
		parser.add_argument('--container-name', default='floss-container-runner', help='What to name the started container.')
		parser.add_argument('--use-docker', action='store_true', default=False, help='Use flag to use Docker to build Floss. Defaults to using podman.')
		args = parser.parse_args()

		# cwd should be set to same directory as this script (that's where Podmanfile
		# is kept).
		# cwd should be set to same directory as this script (that's where
		# Dockerfile is kept).
		workdir = os.path.dirname(os.path.abspath(sys.argv[0]))
		rootdir = os.path.abspath(os.path.join(workdir, '../..'))

		# Determine staging directory absolute path
		staging = os.path.abspath(args.staging_dir) if args.staging_dir else None

		fdr = FlossPodmanRunner(workdir, rootdir, args.image_tag, args.volume_tag, args.container_name, staging)
		fdr = FlossContainerRunner(workdir, rootdir, args.image_tag, args.volume_tag, args.container_name, staging, args.use_docker)

		# Make sure podman is runnable before continuing
		if fdr.check_podman_runnable():
		# Make sure container is runnable before continuing
		if fdr.check_container_runnable():
		# Handle some flags
		if args.only_start:
		fdr.start_container()

floss/build/podman-build-image.py→floss/build/container-build-image.py

+32 −29

Original line number	Diff line number	Diff line
		@@ -8,16 +8,17 @@ import subprocess
		SRC_MOUNT = "/root/src"


		class PodmanImageBuilder:
		"""Builds the podman image for Floss build environment."""
		class ContainerImageBuilder:
		"""Builds the container image for Floss build environment."""

		def __init__(self, workdir, rootdir, tag):
		def __init__(self, workdir, rootdir, tag, use_docker):
		""" Constructor.

		Args:
		workdir: Working directory for this script. Dockerfile should exist here.
		workdir: Working directory for this script. Containerfile should exist here.
		rootdir: Root directory for Bluetooth.
		tag: Label in format \|name:version\|.
		use_docker: Use docker binary if True (or podman when False).
		"""
		self.workdir = workdir
		self.rootdir = rootdir
		@@ -25,9 +26,10 @@ class PodmanImageBuilder:
		self.build_tag = '{}:{}'.format(self.name, 'buildtemp')
		self.container_name = 'floss-buildtemp'
		self.final_tag = tag
		self.container_binary = 'docker' if use_docker else 'podman'
		self.env = os.environ.copy()

		# Mark dpkg builders for podman
		# Mark dpkg builders for container
		self.env['LIBCHROME_DOCKER'] = '1'
		self.env['MODP_DOCKER'] = '1'

		@@ -55,19 +57,19 @@ class PodmanImageBuilder:
		if rc != 0 and not ignore_rc:
		raise Exception("{} failed. Return code is {}".format(target, rc))

		def _podman_build(self):
		self.run_command('podman build', ['podman', 'build', '-t', self.build_tag, '.'])
		def _container_build(self):
		self.run_command(self.container_binary + ' build', [self.container_binary, 'build', '-t', self.build_tag, '.'])

		def _build_dpkg_and_commit(self):
		# Try to remove any previous instance of the container that may be
		# running if this script didn't complete cleanly last time.
		self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name], ignore_rc=True)
		self.run_command('podman rm', ['podman', 'rm', self.container_name], ignore_rc=True)
		self.run_command(self.container_binary + ' stop', [self.container_binary, 'stop', '-t', '1', self.container_name], ignore_rc=True)
		self.run_command(self.container_binary + ' rm', [self.container_binary, 'rm', self.container_name], ignore_rc=True)

		# Runs never terminating application on the newly built image in detached mode
		mount_str = 'type=bind,src={},dst={},readonly'.format(self.rootdir, SRC_MOUNT)
		self.run_command('podman run', [
		'podman', 'run', '--name', self.container_name, '--mount', mount_str, '-d', self.build_tag, 'tail', '-f',
		self.run_command(self.container_binary + ' run', [
		self.container_binary, 'run', '--name', self.container_name, '--mount', mount_str, '-d', self.build_tag, 'tail', '-f',
		'/dev/null'
		])

		@@ -94,18 +96,18 @@ class PodmanImageBuilder:
		try:
		# Run commands in container first to install everything.
		for i, cmd in enumerate(commands):
		self.run_command('podman exec #{}'.format(i), ['podman', 'exec', '-it', self.container_name] + cmd)
		self.run_command(self.container_binary + ' exec #{}'.format(i), [self.container_binary, 'exec', '-it', self.container_name] + cmd)

		# Commit changes into the final tag name
		self.run_command('podman commit', ['podman', 'commit', self.container_name, self.final_tag])
		self.run_command(self.container_binary + ' commit', [self.container_binary, 'commit', self.container_name, self.final_tag])
		finally:
		# Stop running the container and remove it
		self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name])
		self.run_command('podman rm', ['podman', 'rm', self.container_name])
		self.run_command(self.container_binary + ' stop', [self.container_binary, 'stop', '-t', '1', self.container_name])
		self.run_command(self.container_binary + ' rm', [self.container_binary, 'rm', self.container_name])

		def _check_podman_runnable(self):
		def _check_container_runnable(self):
		try:
		subprocess.check_output(['podman', 'ps'], stderr=subprocess.STDOUT)
		subprocess.check_output([self.container_binary, 'ps'], stderr=subprocess.STDOUT)
		except subprocess.CalledProcessError as err:
		if 'denied' in err.output.decode('utf-8'):
		print('Run script as sudo')
		@@ -114,33 +116,34 @@ class PodmanImageBuilder:

		return False

		# No exception means podman is ok
		# No exception means container is ok
		return True

		def build(self):
		if not self._check_podman_runnable():
		if not self._check_container_runnable():
		return

		# First build the podman image
		self._podman_build()
		# First build the container image
		self._container_build()

		# Then build libchrome and modp-b64 inside the podman image and install
		# them. Commit those changes to the final label.
		# Then build libchrome and modp-b64 inside the container image and
		# install them. Commit those changes to the final label.
		self._build_dpkg_and_commit()


		def main():
		parser = argparse.ArgumentParser(description='Build podman image for Floss build environment.')
		parser.add_argument('--tag', required=True, help='Tag for podman image. i.e. floss:latest')
		parser = argparse.ArgumentParser(description='Build container image for Floss build environment.')
		parser.add_argument('--tag', required=True, help='Tag for container image. i.e. floss:latest')
		parser.add_argument('--use-docker', action='store_true', default=False, help='Use flag to use Docker to build Floss. Defaults to using podman.')
		args = parser.parse_args()

		# cwd should be set to same directory as this script (that's where Dockerfile
		# is kept).
		# cwd should be set to same directory as this script (that's where
		# Dockerfile is kept).
		workdir = os.path.dirname(os.path.abspath(sys.argv[0]))
		rootdir = os.path.abspath(os.path.join(workdir, '../..'))

		# Build the podman image
		pib = PodmanImageBuilder(workdir, rootdir, args.tag)
		# Build the container image
		pib = ContainerImageBuilder(workdir, rootdir, args.tag, args.use_docker)
		pib.build()