build-community.sh

#!/bin/bash

# Docker build script
# Copyright (c) 2017 Julian Xhokaxhiu
# Copyright (C) 2017-2018 Nicola Corna <nicola@corna.info>
# Copyright (C) 2020 eCorp Romain HUNAULT <romain.hunaul@e.email>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

repo_log="$LOGS_DIR/repo-$(date +%Y%m%d).log"

# Determine the number of parallel jobs to use for repo sync
nproc_value=$(nproc --all)
JOBS=$(( nproc_value > 10 ? 10 : nproc_value ))

# cd to working directory
cd "$SRC_DIR"

if [ -f /root/userscripts/begin.sh ]; then
  echo ">> [$(date)] Running begin.sh"
  /root/userscripts/begin.sh
fi

# If requested, clean the OUT dir in order to avoid clutter
if [ "$CLEAN_OUTDIR" = true ]; then
  echo ">> [$(date)] Cleaning '$ZIP_DIR'"
  rm -rf "$ZIP_DIR/"*
fi

# Treat DEVICE_LIST as DEVICE_LIST_<first_branch>
first_branch=$(cut -d ',' -f 1 <<<"$BRANCH_NAME")
if [ -n "$DEVICE_LIST" ]; then
  device_list_first_branch="DEVICE_LIST_$(sed 's/.*-\([a-zA-Z]*\)$/\1/' <<<$first_branch)"
  device_list_first_branch=${device_list_first_branch^^}
  read $device_list_first_branch <<<"$DEVICE_LIST,${!device_list_first_branch}"
fi

sync_successful=true

if [ "$LOCAL_MIRROR" = true ]; then

  cd "$MIRROR_DIR"

  if [ ! -d .repo ]; then
    echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log"
    yes | repo init -u "$MIRROR" --mirror --no-clone-bundle -p linux &>>"$repo_log"
  fi

  # Copy local manifests to the appropriate folder in order take them into consideration
  echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
  mkdir -p .repo/local_manifests
  rsync -a --delete --include '*.xml' --exclude '*' "$LMANIFEST_DIR/" .repo/local_manifests/

  rm -f .repo/local_manifests/proprietary.xml
  if [ "$INCLUDE_PROPRIETARY" = true ]; then
    wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/mirror/default.xml"
    /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \
      "https://gitlab.com/the-muppets/manifest/raw/mirror/default.xml" .repo/local_manifests/proprietary_gitlab.xml
  fi

  echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log"
  repo sync -j"$JOBS" --force-sync --no-clone-bundle &>>"$repo_log"

  if [ $? != 0 ]; then
    sync_successful=false
  else
    repo forall -c 'git lfs pull'
  fi
fi

for branch in ${BRANCH_NAME//,/ }; do
  branch_dir=$(sed 's/.*-\([a-zA-Z]*\)$/\1/' <<<$branch)
  branch_dir=${branch_dir^^}
  device_list_cur_branch="DEVICE_LIST_$branch_dir"
  devices=${!device_list_cur_branch}

  if [ -n "$branch" ] && [ -n "$devices" ]; then
    vendor=lineage
    regex_part1="^v[0-9](\.[0-9]*){0,2}(-(beta|alpha|rc)(\.[0-9]*){0,1}){0,1}-("
    regex_part2=")(-[a-zA-Z0-9_]*)*$"

    if [[ "${BRANCH_NAME}" =~ $regex_part1"nougat"$regex_part2 ]]; then
      vendor="cm"
      themuppets_branch="cm-14.1"
      android_version="7.1.2"
      use_openjdk_from_ubuntu=true
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"oreo"$regex_part2 ]]; then
      themuppets_branch="lineage-15.1"
      android_version="8.1"
      use_openjdk_from_ubuntu=true
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"pie"$regex_part2 ]]; then
      themuppets_branch="lineage-16.0"
      android_version="9"
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"q"$regex_part2 ]]; then
      themuppets_branch="lineage-17.1"
      android_version="10"
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"r"$regex_part2 ]]; then
      themuppets_branch="lineage-18.1"
      android_version="11"
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"s"$regex_part2 ]]; then
      themuppets_branch="lineage-19.1"
      android_version="12"
    elif [[ "${BRANCH_NAME}" =~ $regex_part1"t"$regex_part2 ]]; then
      themuppets_branch="lineage-20.0"
      android_version="13"
    else
      echo ">> [$(date)] Building branch $branch is not (yet) suppported"
      exit 1
    fi

    android_version_major=$(cut -d '.' -f 1 <<<$android_version)

    mkdir -p "$SRC_DIR/$branch_dir"
    cd "$SRC_DIR/$branch_dir"

    echo ">> [$(date)] Branch:  $branch"
    echo ">> [$(date)] Devices: $devices"

    # Remove previous changes of vendor/cm, vendor/lineage and frameworks/base (if they exist)
    for path in "vendor/cm" "vendor/lineage" "frameworks/base"; do
      if [ -d "$path" ]; then
        cd "$path"
        git reset -q --hard
        git clean -q -dffx
        cd "$SRC_DIR/$branch_dir"
      fi
    done

    echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log"
    if [ "$LOCAL_MIRROR" = true ]; then
      yes | repo init -u "$REPO" --reference "$MIRROR_DIR" -b "$branch" &>>"$repo_log"
    else
      TAG_PREFIX=""
      if curl -s https://gitlab.e.foundation/api/v4/projects/659/repository/tags?search=${BRANCH_NAME} | jq -r ".[].name" | grep -q "${BRANCH_NAME}"; then
        echo ">> [$(date)] Branch name $branch is a tag on e/os/releases, prefix with refs/tags/ for 'repo init'"
        TAG_PREFIX="refs/tags/"
      fi

      yes | repo init -u "$REPO" -b "${TAG_PREFIX}$branch" &>>"$repo_log"
    fi

    # Copy local manifests to the appropriate folder in order take them into consideration
    echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
    mkdir -p .repo/local_manifests
    rsync -a --delete --include '*.xml' --exclude '*' "$LMANIFEST_DIR/" .repo/local_manifests/

    rm -f .repo/local_manifests/proprietary.xml
    if [ "$INCLUDE_PROPRIETARY" = true ]; then
      wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml"
      /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \
        "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml
    fi

    echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log"
    builddate=$(date +%Y%m%d)
    repo sync -c -j"$JOBS" --force-sync &>>"$repo_log"

    if [ $? != 0 ]; then
      sync_successful=false
    else
      repo forall -c 'git lfs pull'
    fi

    if [ ! -d "vendor/$vendor" ]; then
      echo ">> [$(date)] Missing \"vendor/$vendor\", aborting"
      exit 1
    fi

    config_file="vendor/$vendor/config/version.mk"
    if [ ! -f "$config_file" ]; then
      # If version.mk doesn't exist, use common.mk
      config_file="vendor/$vendor/config/common.mk"
    fi

    # Extract version information
    PRODUCT_VERSION_MAJOR=$(grep -oP 'PRODUCT_VERSION_MAJOR = \K.*' "$config_file")
    PRODUCT_VERSION_MINOR=$(grep -oP 'PRODUCT_VERSION_MINOR = \K.*' "$config_file")
    PRODUCT_VERSION_MAINTENANCE=$(grep -oP 'PRODUCT_VERSION_MAINTENANCE := \K.*' "$config_file")
    PRODUCT_PRERELEASE_STRING=$(grep -oP 'PRODUCT_PRERELEASE_STRING := \K.*' "$config_file")

    # Check if PRODUCT_VERSION_MAINTENANCE is not equal to 0
    e_ver="$PRODUCT_VERSION_MAJOR.$PRODUCT_VERSION_MINOR$PRODUCT_PRERELEASE_STRING"
    if [ "$PRODUCT_VERSION_MAINTENANCE" != "0" ]; then
      e_ver="$PRODUCT_VERSION_MAJOR.$PRODUCT_VERSION_MINOR.$PRODUCT_VERSION_MAINTENANCE$PRODUCT_PRERELEASE_STRING"
    fi

    echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type"
    sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,+2d" "vendor/$vendor/config/common.mk"

    # Set a custom updater URI if a OTA URL is provided
    echo ">> [$(date)] Adding OTA URL overlay (for custom URL $OTA_URL)"
    if ! [ -z "$OTA_URL" ]; then
      updater_url_overlay_dir="vendor/$vendor/overlay/microg/packages/apps/Updater/res/values/"
      mkdir -p "$updater_url_overlay_dir"

      if [ -n "$(grep updater_server_url packages/apps/Updater/res/values/strings.xml)" ]; then
        # "New" updater configuration: full URL (with placeholders {device}, {type} and {incr})
        sed "s|{name}|updater_server_url|g; s|{url}|$OTA_URL/v1/{device}/{type}/{incr}|g" /root/packages_updater_strings.xml >"$updater_url_overlay_dir/strings.xml"
      elif [ -n "$(grep conf_update_server_url_def packages/apps/Updater/res/values/strings.xml)" ]; then
        # "Old" updater configuration: just the URL
        sed "s|{name}|conf_update_server_url_def|g; s|{url}|$OTA_URL|g" /root/packages_updater_strings.xml >"$updater_url_overlay_dir/strings.xml"
      else
        echo ">> [$(date)] ERROR: no known Updater URL property found"
        exit 1
      fi
    fi

    if [ "$SIGN_BUILDS" = true ]; then
      echo ">> [$(date)] Adding keys path ($KEYS_DIR)"
      # Soong (Android 9+) complains if the signing keys are outside the build path
      ln -sf "$KEYS_DIR" user-keys
      if [ "$android_version_major" -lt "10" ]; then
        sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
      fi
      if [ "$android_version_major" -lt "12" ]; then
        sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
      fi
    fi

    # Prepare the environment
    echo ">> [$(date)] Preparing build environment"
    source build/envsetup.sh >/dev/null

    if [ -f /root/userscripts/before.sh ]; then
      echo ">> [$(date)] Running before.sh"
      /root/userscripts/before.sh
    fi

    for codename in ${devices//,/ }; do
      build_device=true
      if ! [ -z "$codename" ]; then

        currentdate=$(date +%Y%m%d)
        if [ "$builddate" != "$currentdate" ]; then
          # Sync the source code
          builddate=$currentdate

          if [ "$LOCAL_MIRROR" = true ]; then
            echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log"
            cd "$MIRROR_DIR"
            repo sync -j"$JOBS" --force-sync --no-clone-bundle &>>"$repo_log"

            if [ $? != 0 ]; then
              sync_successful=false
              build_device=false
            else
              repo forall -c 'git lfs pull'
            fi
          fi

          echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log"
          cd "$SRC_DIR/$branch_dir"
          repo sync -c --force-sync &>>"$repo_log"

          if [ $? != 0 ]; then
            sync_successful=false
            build_device=false
          else
            repo forall -c 'git lfs pull'
          fi
        fi

        if [ "$BUILD_OVERLAY" = true ]; then
          mkdir -p "$TMP_DIR/device" "$TMP_DIR/workdir" "$TMP_DIR/merged"
          mount -t overlay overlay -o lowerdir="$SRC_DIR/$branch_dir",upperdir="$TMP_DIR/device",workdir="$TMP_DIR/workdir" "$TMP_DIR/merged"
          source_dir="$TMP_DIR/merged"
        else
          source_dir="$SRC_DIR/$branch_dir"
        fi
        cd "$source_dir"

        if [ "$ZIP_SUBDIR" = true ]; then
          zipsubdir=$codename
          mkdir -p "$ZIP_DIR/$zipsubdir"
        else
          zipsubdir=
        fi
        if [ "$LOGS_SUBDIR" = true ]; then
          logsubdir=$codename
          mkdir -p "$LOGS_DIR/$logsubdir"
        else
          logsubdir=
        fi

        DEBUG_LOG="$LOGS_DIR/$logsubdir/eos-$e_ver-$builddate-$RELEASE_TYPE-$codename.log"

        if [ -f /root/userscripts/pre-build.sh ]; then
          echo ">> [$(date)] Running pre-build.sh for $codename" >>"$DEBUG_LOG"
          /root/userscripts/pre-build.sh $codename &>>"$DEBUG_LOG"

          if [ $? != 0 ]; then
            build_device=false
          fi
        fi

        if [ "$build_device" = false ]; then
          echo ">> [$(date)] No build for $codename" >>"$DEBUG_LOG"
          continue
        fi

        # Start the build
        echo ">> [$(date)] Starting build for $codename, $branch branch" | tee -a "$DEBUG_LOG"
        build_successful=false
        echo ">> [$(date)] ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}"
        echo ">> [$(date)] Switch to Python2"
        ln -fs /usr/bin/python2 /usr/bin/python

        if [[ "${BRANCH_NAME}" = *"rc"* ]] || [[ "${BRANCH_NAME}" = *"alpha"* ]] || [[ "${BRANCH_NAME}" == *"beta"* ]]; then
          PRODUCT_PRERELEASE=$(echo ${BRANCH_NAME} | sed -E 's/v[0-9]*\.[0-9]*(\.[0-9]*)?-(beta|alpha|rc).*/\2/')
          export PRODUCT_PRERELEASE
          if [[ "${BRANCH_NAME}" = *"rc."* ]] || [[ "${BRANCH_NAME}" = *"alpha."* ]] || [[ "${BRANCH_NAME}" == *"beta."* ]]; then
            PRODUCT_PRERELEASE_VERSION=$(echo ${BRANCH_NAME} | sed -E 's/v[0-9]*\.[0-9]*(\.[0-9]*)?-.*.\.([0-9]*).*/\2/')
            export PRODUCT_PRERELEASE_VERSION
          fi
        fi

        BRUNCH_DEVICE=${codename}

        if [ "${ENG_BUILD}" = true ]; then
          BRUNCH_DEVICE=lineage_${codename}-eng
        fi

        if [ "$android_version_major" -lt "12" ]; then
          if brunch ${BRUNCH_DEVICE} &>>"$DEBUG_LOG"; then
            currentdate=$(date +%Y%m%d)
            if [ "$builddate" != "$currentdate" ]; then
              find out/target/product/$codename -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>>"$DEBUG_LOG"
            fi

            # Move produced ZIP files to the main OUT directory
            echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG"
            cd out/target/product/$codename
            for build in e-*.zip; do
              sha256sum "$build" >"$ZIP_DIR/$zipsubdir/$build.sha256sum"
            done
            find . -maxdepth 1 -name 'e-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>>"$DEBUG_LOG"
            cd "$source_dir"
            build_successful=true
          else
            echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG"
          fi
        elif breakfast ${BRUNCH_DEVICE} &>>"$DEBUG_LOG"; then
          mka target-files-package otatools
          echo ">> [$(date)] Starting signing target-files-package"

          # Set the target files name
          BUILD_NUMBER=$(date -u +%Y%m%d)${CI_PIPELINE_ID}
          TARGET_FILES=lineage_$DEVICE-target_files-$BUILD_NUMBER.zip
          E_BRANCH_NAME=$e_ver-${branch_dir,,}
          if [[ "${BRANCH_NAME}" = *"rc"* ]] || [[ "${BRANCH_NAME}" = *"alpha"* ]] || [[ "${BRANCH_NAME}" == *"beta"* ]]; then
            E_BRANCH_NAME=$(sed 's/^v//' <<<"${BRANCH_NAME}")
          fi
          E_VERSION=e-$E_BRANCH_NAME-$BUILD_NUMBER-$RELEASE_TYPE-${DEVICE}.zip

          # Remove release-keys from build as its still using test-keys.
          if [ "$SIGN_BUILDS" != true ]; then
            SIGN_TARGETS=(-t "+test-keys,-release-keys")
          else
            SIGN_TARGETS=(-o -d "user-keys")
          fi

          if [ "$android_version_major" -ge 12 ]; then
            APEX_PACKAGE_LIST=$(cat "/root/apex.list")
            for PACKAGE in $APEX_PACKAGE_LIST; do
              if [ -f "user-keys/$PACKAGE.pem" ] &&
                [ -f "user-keys/$PACKAGE.x509.pem" ] &&
                [ -f "user-keys/$PACKAGE.pk8" ]; then
                SIGN_TARGETS+=(--extra_apks "$PACKAGE.apex=user-keys/$PACKAGE"
                  --extra_apex_payload_key "$PACKAGE.apex=user-keys/$PACKAGE.pem")
              fi
            done
          fi

          if [ "$(find $OUT/obj/PACKAGING/target_files_intermediates/ -name *-target_files*.zip -print -quit)" ]; then
            sign_target_files_apks "${SIGN_TARGETS[@]}" \
              $OUT/obj/PACKAGING/target_files_intermediates/*-target_files*.zip "$OUT/$TARGET_FILES"

            if [ "$SIGN_BUILDS" = true ]; then
              SIGN_KEY=(-k "user-keys/releasekey")
            fi

            ota_from_target_files "${SIGN_KEY[@]}" "$OUT/$TARGET_FILES" \
              "$OUT/$E_VERSION"

            currentdate=$(date +%Y%m%d)
            if [ "$builddate" != "$currentdate" ]; then
              find out/target/product/$codename -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>>"$DEBUG_LOG"
            fi

            # Move produced ZIP files to the main OUT directory
            echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG"
            cd out/target/product/$codename
            for build in e-*.zip; do
              sha256sum "$build" >"$ZIP_DIR/$zipsubdir/$build.sha256sum"
            done
            find . -maxdepth 1 -name 'e-*.zip*' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \; &>>"$DEBUG_LOG"
            cd "$source_dir"
            build_successful=true
          else
            echo ">> [$(date)] Unable to find $TARGET_FILES" | tee -a "$DEBUG_LOG"
          fi
        else
          echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG"
        fi

        # Remove old zips and logs
        if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then
          if [ "$ZIP_SUBDIR" = true ]; then
            /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $e_ver -N 1 "$ZIP_DIR/$zipsubdir"
          else
            /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $e_ver -N 1 -c $codename "$ZIP_DIR"
          fi
        fi
        if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
          if [ "$LOGS_SUBDIR" = true ]; then
            /usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $e_ver -N 1 "$LOGS_DIR/$logsubdir"
          else
            /usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $e_ver -N 1 -c $codename "$LOGS_DIR"
          fi
        fi
        if [ -f /root/userscripts/post-build.sh ]; then
          echo ">> [$(date)] Running post-build.sh for $codename" >>"$DEBUG_LOG"
          /root/userscripts/post-build.sh $codename $build_successful &>>"$DEBUG_LOG"
        fi
        echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG"

        if [ "$BUILD_OVERLAY" = true ]; then
          # The Jack server must be stopped manually, as we want to unmount $TMP_DIR/merged
          cd "$TMP_DIR"
          if [ -f "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin" ]; then
            "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin kill-server" &>/dev/null || true
          fi
          lsof | grep "$TMP_DIR/merged" | awk '{ print $2 }' | sort -u | xargs -r kill &>/dev/null

          while [ -n "$(lsof | grep $TMP_DIR/merged)" ]; do
            sleep 1
          done

          umount "$TMP_DIR/merged"
        fi

        if [ "$CLEAN_AFTER_BUILD" = true ]; then
          echo ">> [$(date)] Cleaning source dir for device $codename" | tee -a "$DEBUG_LOG"
          if [ "$BUILD_OVERLAY" = true ]; then
            cd "$TMP_DIR"
            rm -rf ./*
          else
            cd "$source_dir"
            mka clean &>>"$DEBUG_LOG"
          fi
        fi

        echo ">> [$(date)] Switch back to Python3"
        ln -fs /usr/bin/python3 /usr/bin/python
      fi
    done
  fi
done

if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
  find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm
fi

if [ -f /root/userscripts/end.sh ]; then
  echo ">> [$(date)] Running end.sh"
  /root/userscripts/end.sh
fi

if [ "$build_successful" = false ] || [ "$sync_successful" = false ]; then
  exit 1
fi