#!/bin/bash # Docker build script # Copyright (c) 2017 Julian Xhokaxhiu # Copyright (C) 2017-2018 Nicola Corna # Copyright (C) 2020 eCorp Romain HUNAULT # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . repo_log="$LOGS_DIR/repo-$(date +%Y%m%d).log" # Determine the number of parallel jobs to use for repo sync nproc_value=$(nproc --all) JOBS=$((nproc_value > 10 ? 10 : nproc_value)) # cd to working directory cd "$SRC_DIR" if [ -f /root/userscripts/begin.sh ]; then echo ">> [$(date)] Running begin.sh" /root/userscripts/begin.sh fi # If requested, clean the OUT dir in order to avoid clutter if [ "$CLEAN_OUTDIR" = true ]; then echo ">> [$(date)] Cleaning '$ZIP_DIR'" rm -rf "$ZIP_DIR/"* fi # Treat DEVICE_LIST as DEVICE_LIST_ first_branch=$(cut -d ',' -f 1 <<<"$BRANCH_NAME") if [ -n "$DEVICE_LIST" ]; then device_list_first_branch="DEVICE_LIST_$(sed 's/.*-\([a-zA-Z]*\)$/\1/' <<<$first_branch)" device_list_first_branch=${device_list_first_branch^^} read $device_list_first_branch <<<"$DEVICE_LIST,${!device_list_first_branch}" fi sync_successful=true if [ "$LOCAL_MIRROR" = true ]; then cd "$MIRROR_DIR" if [ ! -d .repo ]; then echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log" yes | repo init -u "$MIRROR" --mirror --no-clone-bundle -p linux &>>"$repo_log" fi # Copy local manifests to the appropriate folder in order take them into consideration echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'" mkdir -p .repo/local_manifests rsync -a --delete --include '*.xml' --exclude '*' "$LMANIFEST_DIR/" .repo/local_manifests/ rm -f .repo/local_manifests/proprietary.xml if [ "$INCLUDE_PROPRIETARY" = true ]; then wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/mirror/default.xml" /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ "https://gitlab.com/the-muppets/manifest/raw/mirror/default.xml" .repo/local_manifests/proprietary_gitlab.xml fi echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" repo sync -j"$JOBS" --force-sync --no-clone-bundle &>>"$repo_log" if [ $? != 0 ]; then sync_successful=false else repo forall -c 'git lfs pull' fi fi for branch in ${BRANCH_NAME//,/ }; do branch_dir=$(sed 's/.*-\([a-zA-Z]*\)$/\1/' <<<$branch) branch_dir=${branch_dir^^} device_list_cur_branch="DEVICE_LIST_$branch_dir" devices=${!device_list_cur_branch} if [ -n "$branch" ] && [ -n "$devices" ]; then vendor=lineage regex_part1="^v[0-9](\.[0-9]*){0,2}(-(beta|alpha|rc)(\.[0-9]*){0,1}){0,1}-(" regex_part2=")(-[a-zA-Z0-9_]*)*$" if [[ "${BRANCH_NAME}" =~ $regex_part1"nougat"$regex_part2 ]]; then vendor="cm" themuppets_branch="cm-14.1" android_version="7.1.2" use_openjdk_from_ubuntu=true elif [[ "${BRANCH_NAME}" =~ $regex_part1"oreo"$regex_part2 ]]; then themuppets_branch="lineage-15.1" android_version="8.1" use_openjdk_from_ubuntu=true elif [[ "${BRANCH_NAME}" =~ $regex_part1"pie"$regex_part2 ]]; then themuppets_branch="lineage-16.0" android_version="9" elif [[ "${BRANCH_NAME}" =~ $regex_part1"q"$regex_part2 ]]; then themuppets_branch="lineage-17.1" android_version="10" elif [[ "${BRANCH_NAME}" =~ $regex_part1"r"$regex_part2 ]]; then themuppets_branch="lineage-18.1" android_version="11" elif [[ "${BRANCH_NAME}" =~ $regex_part1"s"$regex_part2 ]]; then themuppets_branch="lineage-19.1" android_version="12" elif [[ "${BRANCH_NAME}" =~ $regex_part1"t"$regex_part2 ]]; then themuppets_branch="lineage-20.0" android_version="13" elif [[ "${BRANCH_NAME}" =~ $regex_part1"u"$regex_part2 ]]; then themuppets_branch="lineage-21.0" android_version="14" elif [[ "${BRANCH_NAME}" =~ $regex_part1"a14"$regex_part2 || "${BRANCH_NAME}" =~ "a14" ]]; then themuppets_branch="lineage-21.0" android_version="14" elif [[ "${BRANCH_NAME}" =~ $regex_part1"a15"$regex_part2 || "${BRANCH_NAME}" =~ "a15" ]]; then themuppets_branch="lineage-22.2" android_version="15" elif [[ "${BRANCH_NAME}" =~ $regex_part1"a16"$regex_part2 || "${BRANCH_NAME}" =~ "a16" ]]; then themuppets_branch="lineage-23.0" android_version="16" else echo ">> [$(date)] Building branch $branch is not (yet) suppported" exit 1 fi android_version_major=$(cut -d '.' -f 1 <<<$android_version) mkdir -p "$SRC_DIR/$branch_dir" cd "$SRC_DIR/$branch_dir" echo ">> [$(date)] Branch: $branch" echo ">> [$(date)] Devices: $devices" # Remove previous changes of vendor/cm, vendor/lineage and frameworks/base (if they exist) for path in "vendor/cm" "vendor/lineage" "frameworks/base"; do if [ -d "$path" ]; then cd "$path" git reset -q --hard git clean -q -dffx cd "$SRC_DIR/$branch_dir" fi done echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" if [ "$LOCAL_MIRROR" = true ]; then repo_cmd="yes | repo init -u \"$REPO\" --reference \"$MIRROR_DIR\" -b \"$branch\"" else TAG_PREFIX="" # Fetch all tags and check for an exact match TAG_MATCH=$(curl -s --fail https://gitlab.e.foundation/api/v4/projects/659/repository/tags?search=${BRANCH_NAME} | jq -r ".[].name" | grep -Fx "${BRANCH_NAME}") if [ -n "$TAG_MATCH" ]; then echo ">> [$(date)] Branch name $branch is a tag on e/os/releases, prefixing with refs/tags/ for 'repo init'" | tee -a "$repo_log" TAG_PREFIX="refs/tags/" fi repo_cmd="yes | repo init -u \"$REPO\" -b \"${TAG_PREFIX}$branch\"" fi echo ">> [$(date)] Running: $repo_cmd" | tee -a "$repo_log" eval "$repo_cmd" &>>"$repo_log" || { echo ">> [$(date)] ERROR: Failed to initialize repository" | tee -a "$repo_log" exit 1 } # Copy local manifests to the appropriate folder in order take them into consideration echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'" mkdir -p .repo/local_manifests rsync -a --delete --include '*.xml' --exclude '*' "$LMANIFEST_DIR/" .repo/local_manifests/ rm -f .repo/local_manifests/proprietary.xml if [ "$INCLUDE_PROPRIETARY" = true ]; then wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml" /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml fi echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log" builddate=$(date +%Y%m%d) repo sync -c -j"$JOBS" --force-sync &>>"$repo_log" if [ $? != 0 ]; then sync_successful=false else repo forall -c 'git lfs pull' fi if [ ! -d "vendor/$vendor" ]; then echo ">> [$(date)] Missing \"vendor/$vendor\", aborting" exit 1 fi config_file="vendor/$vendor/config/version.mk" if [ ! -f "$config_file" ]; then # If version.mk doesn't exist, use common.mk config_file="vendor/$vendor/config/common.mk" fi # Extract version information PRODUCT_VERSION_MAJOR=$(grep -oP 'PRODUCT_VERSION_MAJOR = \K.*' "$config_file") PRODUCT_VERSION_MINOR=$(grep -oP 'PRODUCT_VERSION_MINOR = \K.*' "$config_file") PRODUCT_VERSION_MAINTENANCE=$(grep -oP 'PRODUCT_VERSION_MAINTENANCE := \K.*' "$config_file") PRODUCT_PRERELEASE_STRING=$(grep -oP 'PRODUCT_PRERELEASE_STRING := \K.*' "$config_file") # Check if PRODUCT_VERSION_MAINTENANCE is not equal to 0 e_ver="$PRODUCT_VERSION_MAJOR.$PRODUCT_VERSION_MINOR$PRODUCT_PRERELEASE_STRING" if [ "$PRODUCT_VERSION_MAINTENANCE" != "0" ]; then e_ver="$PRODUCT_VERSION_MAJOR.$PRODUCT_VERSION_MINOR.$PRODUCT_VERSION_MAINTENANCE$PRODUCT_PRERELEASE_STRING" fi echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type" sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,+2d" "vendor/$vendor/config/common.mk" # Set a custom updater URI if a OTA URL is provided echo ">> [$(date)] Adding OTA URL overlay (for custom URL $OTA_URL)" if ! [ -z "$OTA_URL" ]; then updater_url_overlay_dir="vendor/$vendor/overlay/microg/packages/apps/Updater/res/values/" mkdir -p "$updater_url_overlay_dir" if [ -n "$(grep updater_server_url packages/apps/Updater/res/values/strings.xml)" ]; then # "New" updater configuration: full URL (with placeholders {device}, {type} and {incr}) sed "s|{name}|updater_server_url|g; s|{url}|$OTA_URL/v1/{device}/{type}/{incr}|g" /root/packages_updater_strings.xml >"$updater_url_overlay_dir/strings.xml" elif [ -n "$(grep conf_update_server_url_def packages/apps/Updater/res/values/strings.xml)" ]; then # "Old" updater configuration: just the URL sed "s|{name}|conf_update_server_url_def|g; s|{url}|$OTA_URL|g" /root/packages_updater_strings.xml >"$updater_url_overlay_dir/strings.xml" else echo ">> [$(date)] ERROR: no known Updater URL property found" exit 1 fi fi if [ "$SIGN_BUILDS" = true ]; then echo ">> [$(date)] Adding keys path ($KEYS_DIR)" # Soong (Android 9+) complains if the signing keys are outside the build path ln -sf "$KEYS_DIR" user-keys if [ "$android_version_major" -lt "10" ]; then sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" elif [ "$android_version_major" -lt "12" ]; then sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" fi fi # Prepare the environment echo ">> [$(date)] Preparing build environment" source build/envsetup.sh >/dev/null if [ -f /root/userscripts/before.sh ]; then echo ">> [$(date)] Running before.sh" /root/userscripts/before.sh fi for codename in ${devices//,/ }; do build_device=true if ! [ -z "$codename" ]; then currentdate=$(date +%Y%m%d) if [ "$builddate" != "$currentdate" ]; then # Sync the source code builddate=$currentdate if [ "$LOCAL_MIRROR" = true ]; then echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" cd "$MIRROR_DIR" repo sync -j"$JOBS" --force-sync --no-clone-bundle &>>"$repo_log" if [ $? != 0 ]; then sync_successful=false build_device=false else repo forall -c 'git lfs pull' fi fi echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log" cd "$SRC_DIR/$branch_dir" repo sync -c --force-sync &>>"$repo_log" if [ $? != 0 ]; then sync_successful=false build_device=false else repo forall -c 'git lfs pull' fi fi if [ "$BUILD_OVERLAY" = true ]; then mkdir -p "$TMP_DIR/device" "$TMP_DIR/workdir" "$TMP_DIR/merged" mount -t overlay overlay -o lowerdir="$SRC_DIR/$branch_dir",upperdir="$TMP_DIR/device",workdir="$TMP_DIR/workdir" "$TMP_DIR/merged" source_dir="$TMP_DIR/merged" else source_dir="$SRC_DIR/$branch_dir" fi cd "$source_dir" if [ "$ZIP_SUBDIR" = true ]; then zipsubdir=$codename mkdir -p "$ZIP_DIR/$zipsubdir" else zipsubdir= fi if [ "$LOGS_SUBDIR" = true ]; then logsubdir=$codename mkdir -p "$LOGS_DIR/$logsubdir" else logsubdir= fi DEBUG_LOG="$LOGS_DIR/$logsubdir/eos-$e_ver-$builddate-$RELEASE_TYPE-$codename.log" if [ -f /root/userscripts/pre-build.sh ]; then echo ">> [$(date)] Running pre-build.sh for $codename" >>"$DEBUG_LOG" /root/userscripts/pre-build.sh $codename &>>"$DEBUG_LOG" if [ $? != 0 ]; then build_device=false fi fi if [ "$build_device" = false ]; then echo ">> [$(date)] No build for $codename" >>"$DEBUG_LOG" continue fi # Start the build echo ">> [$(date)] Starting build for $codename, $branch branch" | tee -a "$DEBUG_LOG" build_successful=false echo ">> [$(date)] ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}" echo ">> [$(date)] Switch to Python2" ln -fs /usr/bin/python2 /usr/bin/python if [[ "${BRANCH_NAME}" = *"rc"* ]] || [[ "${BRANCH_NAME}" = *"alpha"* ]] || [[ "${BRANCH_NAME}" == *"beta"* ]]; then PRODUCT_PRERELEASE=$(echo ${BRANCH_NAME} | sed -E 's/v[0-9]*\.[0-9]*(\.[0-9]*)?-(beta|alpha|rc).*/\2/') export PRODUCT_PRERELEASE if [[ "${BRANCH_NAME}" = *"rc."* ]] || [[ "${BRANCH_NAME}" = *"alpha."* ]] || [[ "${BRANCH_NAME}" == *"beta."* ]]; then PRODUCT_PRERELEASE_VERSION=$(echo ${BRANCH_NAME} | sed -E 's/v[0-9]*\.[0-9]*(\.[0-9]*)?-.*.\.([0-9]*).*/\2/') export PRODUCT_PRERELEASE_VERSION fi fi BRUNCH_DEVICE=${codename} BUILD_TYPE=userdebug if [ "${ENG_BUILD}" = true ]; then BUILD_TYPE=eng fi if [ "$android_version_major" -lt "12" ]; then if brunch ${BRUNCH_DEVICE} ${BUILD_TYPE} &>>"$DEBUG_LOG"; then currentdate=$(date +%Y%m%d) if [ "$builddate" != "$currentdate" ]; then find out/target/product/$codename -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>>"$DEBUG_LOG" fi # Move produced ZIP files to the main OUT directory echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG" cd out/target/product/$codename for build in e-*.zip; do sha256sum "$build" >"$ZIP_DIR/$zipsubdir/$build.sha256sum" done find . -maxdepth 1 -name 'e-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>>"$DEBUG_LOG" cd "$source_dir" build_successful=true else echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG" fi elif breakfast ${BRUNCH_DEVICE} ${BUILD_TYPE} &>>"$DEBUG_LOG"; then mka target-files-package otatools echo ">> [$(date)] Starting signing target-files-package" # Set the target files name BUILD_NUMBER=$(date -u +%Y%m%d)${CI_PIPELINE_ID} TARGET_FILES=lineage_$DEVICE-target_files-$BUILD_NUMBER.zip E_BRANCH_NAME=$e_ver-${branch_dir,,} if [[ "${BRANCH_NAME}" = *"rc"* ]] || [[ "${BRANCH_NAME}" = *"alpha"* ]] || [[ "${BRANCH_NAME}" == *"beta"* ]]; then E_BRANCH_NAME=$(sed 's/^v//' <<<"${BRANCH_NAME}") fi E_VERSION=e-$E_BRANCH_NAME-$BUILD_NUMBER-$RELEASE_TYPE-${DEVICE}.zip # Remove release-keys from build as its still using test-keys. if [ "$SIGN_BUILDS" != true ]; then SIGN_TARGETS=(-t "+test-keys,-release-keys") else SIGN_TARGETS=(-o -d "user-keys") fi if [ "$android_version_major" -ge 12 ] && [ "$SIGN_BUILDS" = true ]; then APEX_PACKAGE_LIST=$(cat "/root/apex.list") for PACKAGE in $APEX_PACKAGE_LIST; do if [ -f "user-keys/$PACKAGE.pem" ] && [ -f "user-keys/$PACKAGE.x509.pem" ] && [ -f "user-keys/$PACKAGE.pk8" ]; then SIGN_TARGETS+=(--extra_apks "$PACKAGE.apex=user-keys/$PACKAGE" --extra_apex_payload_key "$PACKAGE.apex=user-keys/$PACKAGE.pem") fi done APEXAPK_PACKAGE_LIST=$(cat "/root/apex_apk.list") for PACKAGE in $APEXAPK_PACKAGE_LIST; do SIGN_TARGETS+=(--extra_apks "$PACKAGE.apk=user-keys/releasekey") done fi if [ "$(find $OUT/obj/PACKAGING/target_files_intermediates/ -name *-target_files*.zip -print -quit)" ]; then sign_target_files_apks "${SIGN_TARGETS[@]}" \ $OUT/obj/PACKAGING/target_files_intermediates/*-target_files*.zip "$OUT/$TARGET_FILES" if [ "$SIGN_BUILDS" = true ]; then SIGN_KEY=(-k "user-keys/releasekey") fi ota_from_target_files "${SIGN_KEY[@]}" "$OUT/$TARGET_FILES" \ "$OUT/$E_VERSION" currentdate=$(date +%Y%m%d) if [ "$builddate" != "$currentdate" ]; then find out/target/product/$codename -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>>"$DEBUG_LOG" fi # Move produced ZIP files to the main OUT directory echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG" cd out/target/product/$codename for build in e-*.zip; do sha256sum "$build" >"$ZIP_DIR/$zipsubdir/$build.sha256sum" done find . -maxdepth 1 -name 'e-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>>"$DEBUG_LOG" cd "$source_dir" build_successful=true else echo ">> [$(date)] Unable to find $TARGET_FILES" | tee -a "$DEBUG_LOG" fi else echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG" fi # Remove old zips and logs if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then if [ "$ZIP_SUBDIR" = true ]; then /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $e_ver -N 1 "$ZIP_DIR/$zipsubdir" else /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $e_ver -N 1 -c $codename "$ZIP_DIR" fi fi if [ "$DELETE_OLD_LOGS" -gt "0" ]; then if [ "$LOGS_SUBDIR" = true ]; then /usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $e_ver -N 1 "$LOGS_DIR/$logsubdir" else /usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $e_ver -N 1 -c $codename "$LOGS_DIR" fi fi if [ -f /root/userscripts/post-build.sh ]; then echo ">> [$(date)] Running post-build.sh for $codename" >>"$DEBUG_LOG" /root/userscripts/post-build.sh $codename $build_successful &>>"$DEBUG_LOG" fi echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG" if [ "$BUILD_OVERLAY" = true ]; then # The Jack server must be stopped manually, as we want to unmount $TMP_DIR/merged cd "$TMP_DIR" if [ -f "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin" ]; then "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin kill-server" &>/dev/null || true fi lsof | grep "$TMP_DIR/merged" | awk '{ print $2 }' | sort -u | xargs -r kill &>/dev/null while [ -n "$(lsof | grep $TMP_DIR/merged)" ]; do sleep 1 done umount "$TMP_DIR/merged" fi if [ "$CLEAN_AFTER_BUILD" = true ]; then echo ">> [$(date)] Cleaning source dir for device $codename" | tee -a "$DEBUG_LOG" if [ "$BUILD_OVERLAY" = true ]; then cd "$TMP_DIR" rm -rf ./* else cd "$source_dir" mka clean &>>"$DEBUG_LOG" fi fi echo ">> [$(date)] Switch back to Python3" ln -fs /usr/bin/python3 /usr/bin/python fi done fi done if [ "$DELETE_OLD_LOGS" -gt "0" ]; then find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm fi if [ -f /root/userscripts/end.sh ]; then echo ">> [$(date)] Running end.sh" /root/userscripts/end.sh fi if [ "$build_successful" = false ] || [ "$sync_successful" = false ]; then exit 1 fi