Merge branch 'master' into save-img (0dba2cb8) · Commits · e / os / docker-lineage-cicd

.gitlab-ci.yml

+23 −7

Original line number	Diff line number	Diff line
		@@ -5,14 +5,30 @@ variables:
		DOCKER_DRIVER: overlay2
		CONTAINER_IMAGE: registry.gitlab.e.foundation:5000/$CI_PROJECT_PATH

		build_image:
		.build_image:
		stage: build
		image: docker:git
		services:
		- docker:dind
		- docker:18-dind
		script:
		- docker pull $CONTAINER_IMAGE:$CI_COMMIT_REF_NAME \|\| true
		- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.e.foundation:5000
		- docker build --cache-from $CONTAINER_IMAGE:latest -t $CONTAINER_IMAGE -t $CONTAINER_IMAGE:$CI_COMMIT_REF_NAME .
		- docker push $CONTAINER_IMAGE:$CI_COMMIT_REF_NAME
		- if [ "${CI_COMMIT_REF_NAME}" = master ] ; then docker push $CONTAINER_IMAGE:latest ; fi
		- "docker pull $CONTAINER_IMAGE:$IMAGE_TAG \|\| true"
		- "docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.e.foundation:5000"
		- "docker build --cache-from $CONTAINER_IMAGE:$IMAGE_TAG -t $CONTAINER_IMAGE -t $CONTAINER_IMAGE:$IMAGE_TAG -t $CONTAINER_IMAGE:$IMAGE_TAG_RELEASE ."
		- "docker push $CONTAINER_IMAGE:$IMAGE_TAG"
		- 'if [ "${CI_COMMIT_REF_NAME}" = master ] ; then docker push $CONTAINER_IMAGE:$IMAGE_TAG_RELEASE ; fi'

		build_prod:
		extends: .build_image
		after_script:
		- 'if [ "${CI_COMMIT_REF_NAME}" = master ] ; then docker push $CONTAINER_IMAGE:latest ; fi'
		variables:
		IMAGE_TAG: $CI_COMMIT_REF_NAME
		IMAGE_TAG_RELEASE: prod

		build_debug:
		extends: .build_image
		before_script:
		- "mv build.dev.sh src/build.sh"
		variables:
		IMAGE_TAG: $CI_COMMIT_REF_NAME-debug
		IMAGE_TAG_RELEASE: debug

Dockerfile

+0 −1

Original line number	Diff line number	Diff line
		@@ -177,7 +177,6 @@ RUN echo 'deb http://deb.debian.org/debian experimental main' >> /etc/apt/source
		RUN apt-get -qq update
		RUN apt-get -qqy upgrade

		RUN apt-get install -y openjdk-7-jdk
		RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \
		curl flex g++-multilib gcc-multilib git gnupg gperf imagemagick kmod \
		lib32ncurses5-dev lib32readline-dev lib32z1-dev libesd0-dev liblz4-tool \

build.dev.sh

0 → 100755

+438 −0

Original line number	Diff line number	Diff line
		#!/bin/bash

		# Docker build script
		# Copyright (c) 2017 Julian Xhokaxhiu
		# Copyright (C) 2017-2018 Nicola Corna <nicola@corna.info>
		#
		# This program is free software: you can redistribute it and/or modify
		# it under the terms of the GNU General Public License as published by
		# the Free Software Foundation, either version 3 of the License, or
		# (at your option) any later version.
		#
		# This program is distributed in the hope that it will be useful,
		# but WITHOUT ANY WARRANTY; without even the implied warranty of
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		# GNU General Public License for more details.
		#
		# You should have received a copy of the GNU General Public License
		# along with this program. If not, see <http://www.gnu.org/licenses/>.

		# cd to working directory
		cd "$SRC_DIR"

		if [ -f /root/userscripts/begin.sh ]; then
		echo ">> [$(date)] Running begin.sh"
		/root/userscripts/begin.sh
		fi

		# If requested, clean the OUT dir in order to avoid clutter
		if [ "$CLEAN_OUTDIR" = true ]; then
		echo ">> [$(date)] Cleaning '$ZIP_DIR'"
		rm -rf "$ZIP_DIR/"*
		fi

		# Treat DEVICE_LIST as DEVICE_LIST_<first_branch>
		first_branch=$(cut -d ',' -f 1 <<< "$BRANCH_NAME")
		if [ -n "$DEVICE_LIST" ]; then
		device_list_first_branch="DEVICE_LIST_$(sed 's/.-$[a-zA-Z]$$/\1/' <<< $first_branch)"
		device_list_first_branch=${device_list_first_branch^^}
		read $device_list_first_branch <<< "$DEVICE_LIST,${!device_list_first_branch}"
		fi

		sync_successful=true

		if [ "$LOCAL_MIRROR" = true ]; then

		cd "$MIRROR_DIR"

		if [ ! -d .repo ]; then
		echo ">> [$(date)] Initializing mirror repository"
		yes \| repo init -u "$MIRROR" --mirror --no-clone-bundle -p linux
		fi

		# Copy local manifests to the appropriate folder in order take them into consideration
		echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
		mkdir -p .repo/local_manifests
		rsync -a --delete --include '.xml' --exclude '' "$LMANIFEST_DIR/" .repo/local_manifests/

		rm -f .repo/local_manifests/proprietary.xml
		if [ "$INCLUDE_PROPRIETARY" = true ]; then
		wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/mirror/default.xml"
		fi

		echo ">> [$(date)] Syncing mirror repository"
		repo sync --force-sync --no-clone-bundle

		if [ $? != 0 ]; then
		sync_successful=false
		fi
		fi

		for branch in ${BRANCH_NAME//,/ }; do
		branch_dir=$(sed 's/.-$[a-zA-Z]$$/\1/' <<< $branch)
		branch_dir=${branch_dir^^}
		device_list_cur_branch="DEVICE_LIST_$branch_dir"
		devices=${!device_list_cur_branch}

		if [ -n "$branch" ] && [ -n "$devices" ]; then

		mkdir -p "$SRC_DIR/$branch_dir"
		cd "$SRC_DIR/$branch_dir"

		echo ">> [$(date)] Branch: $branch"
		echo ">> [$(date)] Devices: $devices"

		# Remove previous changes of vendor/cm, vendor/lineage and frameworks/base (if they exist)
		for path in "vendor/cm" "vendor/lineage" "frameworks/base"; do
		if [ -d "$path" ]; then
		cd "$path"
		git reset -q --hard
		git clean -q -fd
		cd "$SRC_DIR/$branch_dir"
		fi
		done

		echo ">> [$(date)] (Re)initializing branch repository"
		if [ "$LOCAL_MIRROR" = true ]; then
		yes \| repo init -u "$REPO" --reference "$MIRROR_DIR" -b "$branch"
		else
		yes \| repo init -u "$REPO" -b "$branch"
		fi

		# Copy local manifests to the appropriate folder in order take them into consideration
		echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
		mkdir -p .repo/local_manifests
		rsync -a --delete --include '.xml' --exclude '' "$LMANIFEST_DIR/" .repo/local_manifests/

		rm -f .repo/local_manifests/proprietary.xml
		if [ "$INCLUDE_PROPRIETARY" = true ]; then
		if [[ $branch =~ nougat$ ]]; then
		themuppets_branch=cm-14.1
		echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets"
		elif [[ $branch =~ oreo$ ]]; then
		themuppets_branch=lineage-15.1
		echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets"
		elif [[ $branch =~ pie$ ]]; then
		themuppets_branch=lineage-16.0
		echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets"
		else
		themuppets_branch=cm-14.1
		echo ">> [$(date)] Can't find a matching branch on github.com/TheMuppets, using $themuppets_branch"
		fi
		wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml"
		fi

		echo ">> [$(date)] Syncing branch repository"
		builddate=$(date +%Y%m%d)
		repo sync -c --force-sync

		if [ $? != 0 ]; then
		sync_successful=false
		fi

		android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk)
		if [ -z $android_version ]; then
		android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk)
		if [ -z $android_version ]; then
		android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk)
		if [ -z $android_version ]; then
		echo ">> [$(date)] Can't detect the android version"
		exit 1
		fi
		fi
		fi
		android_version_major=$(cut -d '.' -f 1 <<< $android_version)

		if [ "$android_version_major" -ge "8" ]; then
		vendor="lineage"
		else
		vendor="cm"
		fi

		if [ ! -d "vendor/$vendor" ]; then
		echo ">> [$(date)] Missing \"vendor/$vendor\", aborting"
		exit 1
		fi

		# Set up our overlay
		mkdir -p "vendor/$vendor/overlay/microg/"
		sed -i "1s;^;PRODUCT_PACKAGE_OVERLAYS := vendor/$vendor/overlay/microg\n;" "vendor/$vendor/config/common.mk"

		los_ver_major=$(sed -n -e 's/^\s*PRODUCT_VERSION_MAJOR = //p' "vendor/$vendor/config/common.mk")
		los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "vendor/$vendor/config/common.mk")
		los_ver="$los_ver_major.$los_ver_minor"

		# If needed, apply the microG's signature spoofing patch
		if [ "$SIGNATURE_SPOOFING" = "yes" ] \|\| [ "$SIGNATURE_SPOOFING" = "restricted" ]; then
		# Determine which patch should be applied to the current Android source tree
		patch_name=""
		case $android_version in
		4.4* ) patch_name="android_frameworks_base-KK-LP.patch" ;;
		5.* ) patch_name="android_frameworks_base-KK-LP.patch" ;;
		6.* ) patch_name="android_frameworks_base-M.patch" ;;
		7.* ) patch_name="android_frameworks_base-N.patch" ;;
		8.* ) patch_name="android_frameworks_base-O.patch" ;;
		9* ) patch_name="android_frameworks_base-P.patch" ;; #not sure why 9 not 9.0 but here's a fix that will work until android 90
		esac

		if ! [ -z $patch_name ]; then
		cd frameworks/base
		if [ "$SIGNATURE_SPOOFING" = "yes" ]; then
		echo ">> [$(date)] Applying the standard signature spoofing patch ($patch_name) to frameworks/base"
		echo ">> [$(date)] WARNING: the standard signature spoofing patch introduces a security threat"
		patch --quiet -p1 -i "/root/signature_spoofing_patches/$patch_name"
		else
		echo ">> [$(date)] Applying the restricted signature spoofing patch (based on $patch_name) to frameworks/base"
		sed 's/android:protectionLevel="dangerous"/android:protectionLevel="signature\|privileged"/' "/root/signature_spoofing_patches/$patch_name" \| patch --quiet -p1
		fi
		git clean -q -f
		cd ../..

		# Override device-specific settings for the location providers
		mkdir -p "vendor/$vendor/overlay/microg/frameworks/base/core/res/res/values/"
		cp /root/signature_spoofing_patches/frameworks_base_config.xml "vendor/$vendor/overlay/microg/frameworks/base/core/res/res/values/config.xml"
		else
		echo ">> [$(date)] ERROR: can't find a suitable signature spoofing patch for the current Android version ($android_version)"
		exit 1
		fi
		fi

		echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type"
		sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,+2d" "vendor/$vendor/config/common.mk"

		# Set a custom updater URI if a OTA URL is provided
		echo ">> [$(date)] Adding OTA URL overlay (for custom URL $OTA_URL)"
		if ! [ -z "$OTA_URL" ]; then
		updater_url_overlay_dir="vendor/$vendor/overlay/microg/packages/apps/Updater/res/values/"
		mkdir -p "$updater_url_overlay_dir"

		if [ -n "$(grep updater_server_url packages/apps/Updater/res/values/strings.xml)" ]; then
		# "New" updater configuration: full URL (with placeholders {device}, {type} and {incr})
		sed "s\|{name}\|updater_server_url\|g; s\|{url}\|$OTA_URL/v1/{device}/{type}/{incr}\|g" /root/packages_updater_strings.xml > "$updater_url_overlay_dir/strings.xml"
		elif [ -n "$(grep conf_update_server_url_def packages/apps/Updater/res/values/strings.xml)" ]; then
		# "Old" updater configuration: just the URL
		sed "s\|{name}\|conf_update_server_url_def\|g; s\|{url}\|$OTA_URL\|g" /root/packages_updater_strings.xml > "$updater_url_overlay_dir/strings.xml"
		else
		echo ">> [$(date)] ERROR: no known Updater URL property found"
		exit 1
		fi
		fi

		# Add custom packages to be installed
		if ! [ -z "$CUSTOM_PACKAGES" ]; then
		echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)"
		sed -i "1s;^;PRODUCT_PACKAGES += $CUSTOM_PACKAGES\n\n;" "vendor/$vendor/config/common.mk"
		fi

		if [ "$SIGN_BUILDS" = true ]; then
		echo ">> [$(date)] Adding keys path ($KEYS_DIR)"
		# Soong (Android 9+) complains if the signing keys are outside the build path
		ln -sf "$KEYS_DIR" user-keys
		sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk"
		fi

		if [ "$android_version_major" -ge "7" ]; then
		jdk_version=8
		elif [ "$android_version_major" -ge "5" ]; then
		jdk_version=7
		else
		echo ">> [$(date)] ERROR: $branch requires a JDK version too old (< 7); aborting"
		exit 1
		fi

		echo ">> [$(date)] Using OpenJDK $jdk_version"
		update-java-alternatives -s java-1.$jdk_version.0-openjdk-amd64 &> /dev/null

		# Prepare the environment
		echo ">> [$(date)] Preparing build environment"
		source build/envsetup.sh > /dev/null

		if [ -f /root/userscripts/before.sh ]; then
		echo ">> [$(date)] Running before.sh"
		/root/userscripts/before.sh
		fi

		for codename in ${devices//,/ }; do
		build_device=true
		if ! [ -z "$codename" ]; then

		currentdate=$(date +%Y%m%d)
		if [ "$builddate" != "$currentdate" ]; then
		# Sync the source code
		builddate=$currentdate

		if [ "$LOCAL_MIRROR" = true ]; then
		echo ">> [$(date)] Syncing mirror repository"
		cd "$MIRROR_DIR"
		repo sync --force-sync --no-clone-bundle

		if [ $? != 0 ]; then
		sync_successful=false
		build_device=false
		fi
		fi

		echo ">> [$(date)] Syncing branch repository"
		cd "$SRC_DIR/$branch_dir"
		repo sync -c --force-sync

		if [ $? != 0 ]; then
		sync_successful=false
		build_device=false
		fi
		fi

		if [ "$BUILD_OVERLAY" = true ]; then
		mkdir -p "$TMP_DIR/device" "$TMP_DIR/workdir" "$TMP_DIR/merged"
		mount -t overlay overlay -o lowerdir="$SRC_DIR/$branch_dir",upperdir="$TMP_DIR/device",workdir="$TMP_DIR/workdir" "$TMP_DIR/merged"
		source_dir="$TMP_DIR/merged"
		else
		source_dir="$SRC_DIR/$branch_dir"
		fi
		cd "$source_dir"

		if [ "$ZIP_SUBDIR" = true ]; then
		zipsubdir=$codename
		mkdir -p "$ZIP_DIR/$zipsubdir"
		else
		zipsubdir=
		fi
		if [ "$LOGS_SUBDIR" = true ]; then
		logsubdir=$codename
		mkdir -p "$LOGS_DIR/$logsubdir"
		else
		logsubdir=
		fi

		if [ -f /root/userscripts/pre-build.sh ]; then
		echo ">> [$(date)] Running pre-build.sh for $codename"
		/root/userscripts/pre-build.sh $codename

		if [ $? != 0 ]; then
		build_device=false
		fi
		fi

		if [ "$build_device" = false ]; then
		echo ">> [$(date)] No build for $codename"
		continue
		fi

		# Start the build
		echo ">> [$(date)] Starting build for $codename, $branch branch"
		build_successful=false
		echo "ANDROID_JACK_VM_ARGS=${ANDROID_JACK_VM_ARGS}"
		if brunch $codename ; then
		currentdate=$(date +%Y%m%d)
		if [ "$builddate" != "$currentdate" ]; then
		find out/target/product/$codename -maxdepth 1 -name "e--$currentdate-.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \;
		fi

		if [ "$BUILD_DELTA" = true ]; then
		if [ -d "delta_last/$codename/" ]; then
		# If not the first build, create delta files
		echo ">> [$(date)] Generating delta files for $codename"
		cd /root/delta
		if ./opendelta.sh $codename; then
		echo ">> [$(date)] Delta generation for $codename completed"
		else
		echo ">> [$(date)] Delta generation for $codename failed"
		fi
		if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then
		/usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS -V $los_ver -N 1 "$DELTA_DIR/$codename"
		fi
		cd "$source_dir"
		else
		# If the first build, copy the current full zip in $source_dir/delta_last/$codename/
		echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta"
		mkdir -p delta_last/$codename/
		find out/target/product/$codename -maxdepth 1 -name 'e-*.zip' -type f -exec cp {} "$source_dir/delta_last/$codename/" \;
		fi
		fi
		# Move produced ZIP files to the main OUT directory
		echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'"
		cd out/target/product/$codename
		for build in e-*.zip; do
		sha256sum "$build" > "$ZIP_DIR/$zipsubdir/$build.sha256sum"
		done
		find . -maxdepth 1 -name 'e-.zip' -type f -exec mv {} "$ZIP_DIR/$zipsubdir/" \;
		cd "$source_dir"
		build_successful=true
		else
		echo ">> [$(date)] Failed build for $codename"
		fi

		# Remove old zips and logs
		if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then
		if [ "$ZIP_SUBDIR" = true ]; then
		/usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $los_ver -N 1 "$ZIP_DIR/$zipsubdir"
		else
		/usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS -V $los_ver -N 1 -c $codename "$ZIP_DIR"
		fi
		fi
		if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
		if [ "$LOGS_SUBDIR" = true ]; then
		/usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $los_ver -N 1 "$LOGS_DIR/$logsubdir"
		else
		/usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS -V $los_ver -N 1 -c $codename "$LOGS_DIR"
		fi
		fi
		if [ -f /root/userscripts/post-build.sh ]; then
		echo ">> [$(date)] Running post-build.sh for $codename"
		/root/userscripts/post-build.sh $codename $build_successful
		fi
		echo ">> [$(date)] Finishing build for $codename"

		if [ "$BUILD_OVERLAY" = true ]; then
		# The Jack server must be stopped manually, as we want to unmount $TMP_DIR/merged
		cd "$TMP_DIR"
		if [ -f "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin" ]; then
		"$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin kill-server" &> /dev/null \|\| true
		fi
		lsof \| grep "$TMP_DIR/merged" \| awk '{ print $2 }' \| sort -u \| xargs -r kill &> /dev/null

		while [ -n "$(lsof \| grep $TMP_DIR/merged)" ]; do
		sleep 1
		done

		umount "$TMP_DIR/merged"
		fi

		if [ "$CLEAN_AFTER_BUILD" = true ]; then
		echo ">> [$(date)] Cleaning source dir for device $codename"
		if [ "$BUILD_OVERLAY" = true ]; then
		cd "$TMP_DIR"
		rm -rf ./*
		else
		cd "$source_dir"
		mka clean
		fi
		fi

		fi
		done

		fi
		done

		# Create the OpenDelta's builds JSON file
		if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then
		echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)"
		if [ "$ZIP_SUBDIR" != true ]; then
		echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true"
		fi
		/usr/bin/python /root/opendelta_builds_json.py "$ZIP_DIR" -o "$ZIP_DIR/$OPENDELTA_BUILDS_JSON"
		fi

		if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
		find "$LOGS_DIR" -maxdepth 1 -name repo-*.log \| sort \| head -n -$DELETE_OLD_LOGS \| xargs -r rm
		fi

		if [ -f /root/userscripts/end.sh ]; then
		echo ">> [$(date)] Running end.sh"
		/root/userscripts/end.sh
		fi

		if [ "$build_successful" = false ] \|\| [ "$sync_successful" = false ]; then
		exit 1
		fi

src/init.sh

+1 −1

Original line number	Diff line number	Diff line
		@@ -60,7 +60,7 @@ fi
		# Define memory to use for jack (depending of runner tag)
		if [[ -n ${CI_RUNNER_TAGS} ]]
		then
		jack_memory=$(echo ${CI_RUNNER_TAGS} \| grep GB \| sed 's/.* $[0-9]G$B./\1/')
		jack_memory=$(echo ${CI_RUNNER_TAGS} \| grep GB \| sed 's/.ram:$[0-9]G$B.*/\1/')
		if [ -n ${jack_memory} ]
		then
		ANDROID_JACK_VM_ARGS="-Dfile.encoding=UTF-8 -XX:+TieredCompilation -Xmx"${jack_memory}