Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f3c1bd87 authored by Abhishek Pandit-Subedi's avatar Abhishek Pandit-Subedi Committed by Gerrit Code Review
Browse files

Merge changes I813fb63a,Id4bb51a0,I37b795f0,I1b38a6b5 

* changes:
  floss: Replace docker with podman
  floss: Accept staging directory for build artifacts
  floss: Always stop container after building
  floss: Pin debian version and rustup script
parents 8cb66623 f05e97f5

floss/build/Dockerfile

+6 −3
Original line number Diff line number Diff line
@@ -5,7 +5,8 @@ 


# Inherit from a recent Debian version. The slim version is a smaller variant

# meant for containers.

FROM debian:bookworm-slim

# This digest is taken from the tag debian:bookworm-slim (if you want to update)

FROM debian@sha256:b66f66d473ef3128436ba2812198edcae86c268eb530dff44ff6ae26f9a2ee30



# First install all required apt packages.

RUN apt-get update && \
@@ -56,8 +57,10 @@ RUN apt-get update && \ 
    zlib1g-dev \

    ;



# Next install the Rust toolchain.

RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

# Next install the Rust toolchain. Download the toolchain to the local folder

# using curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs

ADD rustup/rustup.sh /tmp

RUN /tmp/rustup.sh -y --default-toolchain 1.59.0



# Add .cargo/bin to $PATH

ENV PATH="/root/.cargo/bin:${PATH}"

floss/build/README.md

+9 −9
Original line number Diff line number Diff line 
# Docker build for Floss

# Podman build for Floss



This repo contains the Docker image build rule, used to generate the docker

image necessary to build Floss. If building a new docker image, run

`docker-build-image.py` with the tag `floss:latest`.

This repo contains the Podman image build rule, used to generate the Podman

image necessary to build Floss. If building a new Podman image, run

`podman-build-image.py` with the tag `floss:latest`.



## Using the docker image to build

## Using the podman image to build



Once the Docker image is built (and assuming it's tagged as `floss:latest`), you

should use the `build-in-docker.py` script to build the current repo.

Once the Podman image is built (and assuming it's tagged as `floss:latest`), you

should use the `build-in-podman.py` script to build the current repo.



This script will use the local `floss:latest` (or pull it from the registry),

mount (or create) the `floss-out` volume to `/root/.floss` and the current
@@ -18,6 +18,6 @@ source to `/root/src` before running these commands in the container: 
* `./build.py --libdir=/usr/lib/x86-64_linux_gnu/`



If you want to run the build more quickly (or pass other commands), run

`build-in-docker.py --only-start`. This will only start the container for you

`build-in-podman.py --only-start`. This will only start the container for you

(doing the correct mounts) and will print the commands it would have run via

docker exec normally.
 
`podman exec` normally.

floss/build/build-in-docker.py→floss/build/build-in-podman.py

+67 −43
Original line number Diff line number Diff line
@@ -6,32 +6,38 @@ import subprocess 
import sys

import time



SRC_MOUNT = "/root/src"

STAGING_MOUNT = "/root/.floss"



class FlossDockerRunner:

    """Runs Floss build inside docker container."""



class FlossPodmanRunner:

    """Runs Floss build inside podman container."""



    # Commands to run for build

    BUILD_COMMANDS = [

        # First run bootstrap to get latest code + create symlinks

        ['/root/src/build.py', '--run-bootstrap'],

        [f'{SRC_MOUNT}/build.py', '--run-bootstrap'],



        # Clean up any previous artifacts inside the volume

        ['/root/src/build.py', '--target', 'clean'],

        [f'{SRC_MOUNT}/build.py', '--target', 'clean'],



        # Run normal code builder

        ['/root/src/build.py', '--target', 'all'],

        [f'{SRC_MOUNT}/build.py', '--target', 'all'],



        # Run tests

        ['/root/src/build.py', '--target', 'test'],

        [f'{SRC_MOUNT}/build.py', '--target', 'test'],

    ]



    def __init__(self, workdir, rootdir, image_tag, volume_tag):

    def __init__(self, workdir, rootdir, image_tag, volume_name, container_name, staging_dir):

        """ Constructor.



        Args:

            workdir: Current working directory (should be the script path).

            rootdir: Root directory for Bluetooth.

            build_tag: Tag for docker image used for building.

            image_tag: Tag for podman image used for building.

            volume_name: Volume name used for storing artifacts.

            container_name: Name for running container instance.

            staging_dir: Directory to mount for artifacts instead of using volume.

        """

        self.workdir = workdir

        self.rootdir = rootdir
@@ -39,10 +45,12 @@ class FlossDockerRunner: 
        self.env = os.environ.copy()



        # Name of running container

        self.container_name = 'floss-docker-runner'

        self.container_name = container_name



        # Name of volume where we'll send build output

        self.volume_name = volume_tag

        # Name of volume to write output.

        self.volume_name = volume_name

        # Staging dir where we send output instead of the volume.

        self.staging_dir = staging_dir



    def run_command(self, target, args, cwd=None, env=None, ignore_rc=False):

        """ Run command and stream the output.
@@ -71,56 +79,61 @@ class FlossDockerRunner: 
    def _create_volume_if_needed(self):

        # Check if the volume exists. Otherwise create it.

        try:

            subprocess.check_output(['docker', 'volume', 'inspect', self.volume_name])

        finally:

            self.run_command('docker volume create', ['docker', 'volume', 'create', self.volume_name])

            subprocess.check_output(['podman', 'volume', 'inspect', self.volume_name])

        except:

            self.run_command('podman volume create', ['podman', 'volume', 'create', self.volume_name])



    def start_container(self):

        """Starts the docker container with correct mounts."""

        """Starts the podman container with correct mounts."""

        # Stop any previously started container.

        self.stop_container(ignore_error=True)



        # Create volume and create mount string

        if self.staging_dir:

            mount_output_volume = 'type=bind,src={},dst={}'.format(self.staging_dir, STAGING_MOUNT)

        else:

            # If not using staging dir, use the volume instead

            self._create_volume_if_needed()

        mount_output_volume = 'type=volume,src={},dst=/root/.floss'.format(self.volume_name)

            mount_output_volume = 'type=volume,src={},dst={}'.format(self.volume_name, STAGING_MOUNT)



        # Mount the source directory

        mount_src_dir = 'type=bind,src={},dst=/root/src'.format(self.rootdir)

        mount_src_dir = 'type=bind,src={},dst={}'.format(self.rootdir, SRC_MOUNT)



        # Run the docker image. It will run `tail` indefinitely so the container

        # doesn't close and we can run `docker exec` on it.

        self.run_command('docker run', [

            'docker', 'run', '--name', self.container_name, '--mount', mount_output_volume, '--mount', mount_src_dir,

        # Run the podman image. It will run `tail` indefinitely so the container

        # doesn't close and we can run `podman exec` on it.

        self.run_command('podman run', [

            'podman', 'run', '--name', self.container_name, '--mount', mount_output_volume, '--mount', mount_src_dir,

            '-d', self.image_tag, 'tail', '-f', '/dev/null'

        ])



    def stop_container(self, ignore_error=False):

        """Stops the docker container for build."""

        self.run_command('docker stop', ['docker', 'stop', '-t', '1', self.container_name], ignore_rc=ignore_error)

        self.run_command('docker rm', ['docker', 'rm', self.container_name], ignore_rc=ignore_error)

        """Stops the podman container for build."""

        self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name], ignore_rc=ignore_error)

        self.run_command('podman rm', ['podman', 'rm', self.container_name], ignore_rc=ignore_error)



    def do_build(self):

        """Runs the basic build commands."""

        # Start container before building

        self.start_container()



        try:

            # Run all commands

            for i, cmd in enumerate(self.BUILD_COMMANDS):

            self.run_command('docker exec #{}'.format(i), ['docker', 'exec', '-it', self.container_name] + cmd)



        # Stop container before exiting

                self.run_command('podman exec #{}'.format(i), ['podman', 'exec', '-it', self.container_name] + cmd)

        finally:

            # Always stop container before exiting

            self.stop_container()



    def print_do_build(self):

        """Prints the commands for building."""

        docker_exec = ['docker', 'exec', '-it', self.container_name]

        podman_exec = ['podman', 'exec', '-it', self.container_name]

        print('Normally, build would run the following commands: \n')

        for cmd in self.BUILD_COMMANDS:

            print(' '.join(docker_exec + cmd))

            print(' '.join(podman_exec + cmd))



    def check_docker_runnable(self):

    def check_podman_runnable(self):

        try:

            subprocess.check_output(['docker', 'ps'], stderr=subprocess.STDOUT)

            subprocess.check_output(['podman', 'ps'], stderr=subprocess.STDOUT)

        except subprocess.CalledProcessError as err:

            if 'denied' in err.output.decode('utf-8'):

                print('Run script as sudo')
@@ -129,31 +142,42 @@ class FlossDockerRunner: 


            return False



        # No exception means docker is ok

        # No exception means podman is ok

        return True





if __name__ == "__main__":

    parser = argparse.ArgumentParser('Builder Floss inside docker image.')

    parser = argparse.ArgumentParser('Builder Floss inside podman image.')

    parser.add_argument(

        '--only-start',

        action='store_true',

        default=False,

        help='Only start the container. Prints the commands it would have ran.')

    parser.add_argument('--only-stop', action='store_true', default=False, help='Only stop the container and exit.')

    parser.add_argument('--image-tag', default='floss:latest', help='Docker image to use to build.')

    parser.add_argument('--volume-tag', default='floss-out', help='Name of volume to use.')

    parser.add_argument('--image-tag', default='floss:latest', help='Podman image to use to build.')

    parser.add_argument(

        '--volume-tag',

        default='floss-out',

        help='Name of volume to use. This is where build artifacts will be stored by default.')

    parser.add_argument(

        '--staging-dir',

        default=None,

        help='Staging directory to use instead of volume. Build artifacts will be written here.')

    parser.add_argument('--container-name', default='floss-podman-runner', help='What to name the started container')

    args = parser.parse_args()



    # cwd should be set to same directory as this script (that's where Dockerfile

    # cwd should be set to same directory as this script (that's where Podmanfile

    # is kept).

    workdir = os.path.dirname(os.path.abspath(sys.argv[0]))

    rootdir = os.path.abspath(os.path.join(workdir, '../..'))



    fdr = FlossDockerRunner(workdir, rootdir, args.image_tag, args.volume_tag)

    # Determine staging directory absolute path

    staging = os.path.abspath(args.staging_dir) if args.staging_dir else None



    fdr = FlossPodmanRunner(workdir, rootdir, args.image_tag, args.volume_tag, args.container_name, staging)



    # Make sure docker is runnable before continuing

    if fdr.check_docker_runnable():

    # Make sure podman is runnable before continuing

    if fdr.check_podman_runnable():

        # Handle some flags

        if args.only_start:

            fdr.start_container()

floss/build/docker-build-image.py→floss/build/podman-build-image.py

+35 −34
Original line number Diff line number Diff line
@@ -8,8 +8,8 @@ import subprocess 
SRC_MOUNT = "/root/src"





class DockerImageBuilder:

    """Builds the docker image for Floss build environment."""

class PodmanImageBuilder:

    """Builds the podman image for Floss build environment."""



    def __init__(self, workdir, rootdir, tag):

        """ Constructor.
@@ -27,7 +27,7 @@ class DockerImageBuilder: 
        self.final_tag = tag

        self.env = os.environ.copy()



        # Mark dpkg builders for docker

        # Mark dpkg builders for podman

        self.env['LIBCHROME_DOCKER'] = '1'

        self.env['MODP_DOCKER'] = '1'
@@ -55,19 +55,19 @@ class DockerImageBuilder: 
        if rc != 0 and not ignore_rc:

            raise Exception("{} failed. Return code is {}".format(target, rc))



    def _docker_build(self):

        self.run_command('docker build', ['docker', 'build', '-t', self.build_tag, '.'])

    def _podman_build(self):

        self.run_command('podman build', ['podman', 'build', '-t', self.build_tag, '.'])



    def _build_dpkg_and_commit(self):

        # Try to remove any previous instance of the container that may be

        # running if this script didn't complete cleanly last time.

        self.run_command('docker stop', ['docker', 'stop', '-t', '1', self.container_name], ignore_rc=True)

        self.run_command('docker rm', ['docker', 'rm', self.container_name], ignore_rc=True)

        self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name], ignore_rc=True)

        self.run_command('podman rm', ['podman', 'rm', self.container_name], ignore_rc=True)



        # Runs never terminating application on the newly built image in detached mode

        mount_str = 'type=bind,src={},dst={},readonly'.format(self.rootdir, SRC_MOUNT)

        self.run_command('docker run', [

            'docker', 'run', '--name', self.container_name, '--mount', mount_str, '-d', self.build_tag, 'tail', '-f',

        self.run_command('podman run', [

            'podman', 'run', '--name', self.container_name, '--mount', mount_str, '-d', self.build_tag, 'tail', '-f',

            '/dev/null'

        ])
@@ -76,13 +76,13 @@ class DockerImageBuilder: 
            ['mkdir', '-p', '/tmp/libchrome', '/tmp/modpb64'],



            # Run the dpkg builder for modp_b64

            ['/root/src/system/build/dpkg/modp_b64/gen-src-pkg.sh', '/tmp/modpb64'],

            [f'{SRC_MOUNT}/system/build/dpkg/modp_b64/gen-src-pkg.sh', '/tmp/modpb64'],



            # Install modp_b64 since libchrome depends on it

            ['find', '/tmp/modpb64', '-name', 'modp*.deb', '-exec', 'dpkg', '-i', '{}', '+'],



            # Run the dpkg builder for libchrome

            ['/root/src/system/build/dpkg/libchrome/gen-src-pkg.sh', '/tmp/libchrome'],

            [f'{SRC_MOUNT}/system/build/dpkg/libchrome/gen-src-pkg.sh', '/tmp/libchrome'],



            # Install libchrome.

            ['find', '/tmp/libchrome', '-name', 'libchrome_*.deb', '-exec', 'dpkg', '-i', '{}', '+'],
@@ -91,20 +91,21 @@ class DockerImageBuilder: 
            ['rm', '-rf', '/tmp/libchrome', '/tmp/modpb64'],

        ]



        try:

            # Run commands in container first to install everything.

            for i, cmd in enumerate(commands):

            self.run_command('docker exec #{}'.format(i), ['docker', 'exec', '-it', self.container_name] + cmd)

                self.run_command('podman exec #{}'.format(i), ['podman', 'exec', '-it', self.container_name] + cmd)



            # Commit changes into the final tag name

        self.run_command('docker commit', ['docker', 'commit', self.container_name, self.final_tag])



            self.run_command('podman commit', ['podman', 'commit', self.container_name, self.final_tag])

        finally:

            # Stop running the container and remove it

        self.run_command('docker stop', ['docker', 'stop', '-t', '1', self.container_name])

        self.run_command('docker rm', ['docker', 'rm', self.container_name])

            self.run_command('podman stop', ['podman', 'stop', '-t', '1', self.container_name])

            self.run_command('podman rm', ['podman', 'rm', self.container_name])



    def _check_docker_runnable(self):

    def _check_podman_runnable(self):

        try:

            subprocess.check_output(['docker', 'ps'], stderr=subprocess.STDOUT)

            subprocess.check_output(['podman', 'ps'], stderr=subprocess.STDOUT)

        except subprocess.CalledProcessError as err:

            if 'denied' in err.output.decode('utf-8'):

                print('Run script as sudo')
@@ -113,24 +114,24 @@ class DockerImageBuilder: 


            return False



        # No exception means docker is ok

        # No exception means podman is ok

        return True



    def build(self):

        if not self._check_docker_runnable():

        if not self._check_podman_runnable():

            return



        # First build the docker image

        self._docker_build()

        # First build the podman image

        self._podman_build()



        # Then build libchrome and modp-b64 inside the docker image and install

        # Then build libchrome and modp-b64 inside the podman image and install

        # them. Commit those changes to the final label.

        self._build_dpkg_and_commit()





def main():

    parser = argparse.ArgumentParser(description='Build docker image for Floss build environment.')

    parser.add_argument('--tag', required=True, help='Tag for docker image. i.e. floss:latest')

    parser = argparse.ArgumentParser(description='Build podman image for Floss build environment.')

    parser.add_argument('--tag', required=True, help='Tag for podman image. i.e. floss:latest')

    args = parser.parse_args()



    # cwd should be set to same directory as this script (that's where Dockerfile
@@ -138,9 +139,9 @@ def main(): 
    workdir = os.path.dirname(os.path.abspath(sys.argv[0]))

    rootdir = os.path.abspath(os.path.join(workdir, '../..'))



    # Build the docker image

    dib = DockerImageBuilder(workdir, rootdir, args.tag)

    dib.build()

    # Build the podman image

    pib = PodmanImageBuilder(workdir, rootdir, args.tag)

    pib.build()





if __name__ == '__main__':

floss/build/rustup/LICENSE-APACHE

0 → 100644
+201 −0

File added.

Preview size limit exceeded, changes collapsed.