From f7ff8f5717176daeb0951060069368b31cd2933b Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Fri, 1 Oct 2021 11:22:19 +0200 Subject: [PATCH 1/7] Copy build-community script in Dockerfile --- .gitlab-ci.yml | 2 -- Dockerfile.community | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2df3114..df2bc7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,8 +29,6 @@ build_prod: build_community: extends: .build_image - before_script: - - "mv build-community.sh src/build.sh" variables: IMAGE_TAG: $CI_COMMIT_REF_NAME-community IMAGE_TAG_RELEASE: community diff --git a/Dockerfile.community b/Dockerfile.community index b06f7f8..331d805 100644 --- a/Dockerfile.community +++ b/Dockerfile.community @@ -134,6 +134,7 @@ VOLUME /root/.ssh # Copy required files ##################### COPY src/ /root/ +COPY build-community.sh /root/build.sh # Create missing directories ############################ -- GitLab From 36198bdb4b8ea055b4cf8a7c588720121c6cd4de Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Fri, 1 Oct 2021 11:24:42 +0200 Subject: [PATCH 2/7] Handle e tags without spam stdout --- Dockerfile | 2 +- Dockerfile.community | 2 +- build-community.sh | 3 +-- src/build.sh | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 095c31e..d9e5063 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,7 +162,7 @@ RUN apt-get -qq update RUN apt-get -qqy upgrade RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt clang cron \ - curl flex g++-multilib gcc-multilib git git-lfs gnupg gperf imagemagick img2simg kmod \ + curl flex g++-multilib gcc-multilib git git-lfs gnupg gperf imagemagick img2simg kmod jq \ lib32ncurses5-dev libncurses5 lib32readline-dev lib32z1-dev libtinfo5 liblz4-tool \ libncurses5-dev libsdl1.2-dev libssl-dev libxml2 \ libxml2-utils lsof lzop maven openjdk-8-jdk pngcrush \ diff --git a/Dockerfile.community b/Dockerfile.community index 331d805..a45cd77 100644 --- a/Dockerfile.community +++ b/Dockerfile.community @@ -154,7 +154,7 @@ RUN apt-get -qq update RUN apt-get -qqy upgrade RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt clang cron \ - curl flex g++-multilib gcc-multilib git git-lfs gnupg gperf imagemagick kmod \ + curl flex g++-multilib gcc-multilib git git-lfs gnupg gperf imagemagick kmod jq \ lib32ncurses5-dev libncurses5 lib32readline-dev lib32z1-dev libtinfo5 liblz4-tool \ libncurses5-dev libsdl1.2-dev libssl-dev libxml2 \ libxml2-utils lsof lzop maven pngcrush \ diff --git a/build-community.sh b/build-community.sh index 84d375f..36933d0 100755 --- a/build-community.sh +++ b/build-community.sh @@ -139,8 +139,7 @@ for branch in ${BRANCH_NAME//,/ }; do yes | repo init -u "$REPO" --reference "$MIRROR_DIR" -b "$branch" &>> "$repo_log" else TAG_PREFIX="" - curl https://gitlab.e.foundation/api/v4/projects/659/repository/tags | grep "\"name\":\"$branch\"" - if [ $? == 0 ] + if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q "${BRANCH_NAME}" then echo "Branch name $branch is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" diff --git a/src/build.sh b/src/build.sh index 47e5c2b..0df2d11 100755 --- a/src/build.sh +++ b/src/build.sh @@ -94,7 +94,7 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then echo ">> [$(date)] (Re)initializing branch repository" TAG_PREFIX="" - if curl https://gitlab.e.foundation/api/v4/projects/659/repository/tags | grep "\"name\":\"${BRANCH_NAME}\"" + if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q ${BRANCH_NAME} then echo "Branch name ${BRANCH_NAME} is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" -- GitLab From e2b98cf4f048f1fdd6564edaa30225156810a999 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Fri, 1 Oct 2021 11:29:00 +0200 Subject: [PATCH 3/7] Enforce cleanup on lineageos and framework/base repo --- build-community.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-community.sh b/build-community.sh index 36933d0..fa23189 100755 --- a/build-community.sh +++ b/build-community.sh @@ -129,7 +129,7 @@ for branch in ${BRANCH_NAME//,/ }; do if [ -d "$path" ]; then cd "$path" git reset -q --hard - git clean -q -fd + git clean -q -dffx cd "$SRC_DIR/$branch_dir" fi done -- GitLab From 4b82cf87091c5dba6a377497e66a6bfa92a9f370 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Fri, 1 Oct 2021 11:32:56 +0200 Subject: [PATCH 4/7] Unify logs with date prefix --- build-community.sh | 8 ++++---- src/build.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build-community.sh b/build-community.sh index fa23189..9c611fe 100755 --- a/build-community.sh +++ b/build-community.sh @@ -141,7 +141,7 @@ for branch in ${BRANCH_NAME//,/ }; do TAG_PREFIX="" if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q "${BRANCH_NAME}" then - echo "Branch name $branch is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" + echo ">> [$(date)] Branch name $branch is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" fi @@ -297,8 +297,8 @@ for branch in ${BRANCH_NAME//,/ }; do # Start the build echo ">> [$(date)] Starting build for $codename, $branch branch" | tee -a "$DEBUG_LOG" build_successful=false - echo "ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}" - echo "Switch to Python2" + echo ">> [$(date)] ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}" + echo ">> [$(date)] Switch to Python2" ln -fs /usr/bin/python2 /usr/bin/python @@ -374,7 +374,7 @@ for branch in ${BRANCH_NAME//,/ }; do fi fi - echo "Switch back to Python3" + echo ">> [$(date)] Switch back to Python3" ln -fs /usr/bin/python3 /usr/bin/python fi done diff --git a/src/build.sh b/src/build.sh index 0df2d11..8dbf666 100755 --- a/src/build.sh +++ b/src/build.sh @@ -96,7 +96,7 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then TAG_PREFIX="" if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q ${BRANCH_NAME} then - echo "Branch name ${BRANCH_NAME} is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" + echo ">> [$(date)] Branch name ${BRANCH_NAME} is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" fi if [ -n ${REPO_INIT_DEPTH} ] && [ ${REPO_INIT_DEPTH} -gt 0 ]; then @@ -240,8 +240,8 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then # Start the build echo ">> [$(date)] Starting build for ${DEVICE}, ${BRANCH_NAME} branch" build_successful=false - echo "ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}" - echo "Switch to Python2" + echo ">> [$(date)] ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}" + echo ">> [$(date)] Switch to Python2" ln -fs /usr/bin/python2 /usr/bin/python BRUNCH_DEVICE=${DEVICE} @@ -393,7 +393,7 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then fi - echo "Switch back to Python3" + echo ">> [$(date)] Switch back to Python3" ln -fs /usr/bin/python3 /usr/bin/python fi -- GitLab From 4cd32d7ac03e85d110833cb154b86f5d5cde18b5 Mon Sep 17 00:00:00 2001 From: tcecyk Date: Tue, 9 Aug 2022 15:56:15 +0200 Subject: [PATCH 5/7] build-community: return all release repo tags by default, gitlab pagination will only return 20 items. As you never know what kind of tags the users will build, older tags will fail. Without a match, the tags-detection breaks, repo init fails, no checkout occurs, user will fail at the latest at the directory check for 'vendor/lineage', a misleading error. --- build-community.sh | 2 +- src/build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-community.sh b/build-community.sh index 9c611fe..7d7013a 100755 --- a/build-community.sh +++ b/build-community.sh @@ -139,7 +139,7 @@ for branch in ${BRANCH_NAME//,/ }; do yes | repo init -u "$REPO" --reference "$MIRROR_DIR" -b "$branch" &>> "$repo_log" else TAG_PREFIX="" - if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q "${BRANCH_NAME}" + if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags?search?${BRANCH_NAME} | jq -r ".[].name" | grep -q "${BRANCH_NAME}" then echo ">> [$(date)] Branch name $branch is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" diff --git a/src/build.sh b/src/build.sh index 8dbf666..b16bd86 100755 --- a/src/build.sh +++ b/src/build.sh @@ -94,7 +94,7 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then echo ">> [$(date)] (Re)initializing branch repository" TAG_PREFIX="" - if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags | jq -r ".[].name" | grep -q ${BRANCH_NAME} + if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags?search?${BRANCH_NAME} | jq -r ".[].name" | grep -q ${BRANCH_NAME} then echo ">> [$(date)] Branch name ${BRANCH_NAME} is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'" TAG_PREFIX="refs/tags/" -- GitLab From 624020a900d2cc4b1ce856a584fdd3223fdef0d3 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 9 Feb 2023 15:36:18 +0100 Subject: [PATCH 6/7] Update and fix docker pipeline --- .gitlab-ci.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df2bc7c..c21556e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,24 +1,18 @@ -stages: - - build - variables: - DOCKER_DRIVER: overlay2 - CONTAINER_IMAGE: registry.gitlab.e.foundation:5000/$CI_PROJECT_PATH + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: "" .build_image: stage: build image: docker:git services: - - docker:18-dind - variables: - IMAGE_TAG: $CI_COMMIT_REF_NAME - IMAGE_TAG_RELEASE: prod + - docker:20.10.16-dind script: - - "docker pull $CONTAINER_IMAGE:$IMAGE_TAG || true" - - "docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.e.foundation:5000" - - "docker build --cache-from $CONTAINER_IMAGE:$IMAGE_TAG -t $CONTAINER_IMAGE -t $CONTAINER_IMAGE:$IMAGE_TAG -t $CONTAINER_IMAGE:$IMAGE_TAG_RELEASE -f $DOCKERFILE ." - - "docker push $CONTAINER_IMAGE:$IMAGE_TAG" - - 'if [ "${CI_COMMIT_REF_NAME}" = master ] ; then docker push $CONTAINER_IMAGE:$IMAGE_TAG_RELEASE ; fi' + - "docker pull $CI_REGISTRY_IMAGE:$IMAGE_TAG || true" + - "docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY" + - "docker build --cache-from $CI_REGISTRY_IMAGE:$IMAGE_TAG -t $CI_REGISTRY_IMAGE:$IMAGE_TAG -f $DOCKERFILE ." + - "docker push $CI_REGISTRY_IMAGE:$IMAGE_TAG" + - 'if [ "${CI_COMMIT_REF_NAME}" = master ] ; then docker tag $CI_REGISTRY_IMAGE:$IMAGE_TAG $CI_REGISTRY_IMAGE:$IMAGE_TAG_RELEASE && docker push $CI_REGISTRY_IMAGE:$IMAGE_TAG_RELEASE ; fi' build_prod: extends: .build_image -- GitLab From d7a364dbddae300a2466f87b40e835b56f9d1349 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 9 Feb 2023 15:36:55 +0100 Subject: [PATCH 7/7] Replace eelo reference --- build-community.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-community.sh b/build-community.sh index 7d7013a..4d67cab 100755 --- a/build-community.sh +++ b/build-community.sh @@ -278,7 +278,7 @@ for branch in ${BRANCH_NAME//,/ }; do logsubdir= fi - DEBUG_LOG="$LOGS_DIR/$logsubdir/eelo-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" + DEBUG_LOG="$LOGS_DIR/$logsubdir/eos-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" if [ -f /root/userscripts/pre-build.sh ]; then echo ">> [$(date)] Running pre-build.sh for $codename" >> "$DEBUG_LOG" -- GitLab