diff --git a/Dockerfile b/Dockerfile index 1a7fd48ccb03c11c049ccda9a178b1dec580127b..d213ba251443153fe4adaf76b28f573f1c3b8dc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM ubuntu:20.04 MAINTAINER Romain Hunault # Environment variables @@ -8,7 +8,6 @@ ENV SRC_DIR /srv/src ENV CCACHE_DIR /srv/ccache ENV ZIP_DIR /srv/zips ENV LMANIFEST_DIR /srv/local_manifests -ENV DELTA_DIR /srv/delta ENV KEYS_DIR /srv/keys ENV LOGS_DIR /srv/logs ENV USERSCRIPTS_DIR /srv/userscripts @@ -28,9 +27,12 @@ ENV USE_CCACHE 1 # for no limit. ENV CCACHE_SIZE 50G +# We need to specify the ccache binary since it is no longer packaged along with AOSP +ENV CCACHE_EXEC /usr/bin/ccache + # Environment for the /e/ branches name # See https://github.com/LineageOS/android_vendor_cm/branches for possible options -ENV BRANCH_NAME 'v1-pie' +ENV BRANCH_NAME 'v1-q' # Environment for the device # eg. DEVICE=hammerhead @@ -46,7 +48,7 @@ ENV REPO 'https://gitlab.e.foundation/e/os/android.git' ENV USER_NAME '/e/ robot' ENV USER_MAIL 'erobot@e.email' -# Include proprietary files, downloaded automatically from github.com/TheMuppets/ +# Include proprietary files, downloaded automatically from github.com/TheMuppets/ and gitlab.com/the-muppets/ # Only some branches are supported ENV INCLUDE_PROPRIETARY true @@ -82,24 +84,12 @@ ENV LOGS_SUBDIR true # Backup the .img in addition to zips ENV BACKUP_IMG false -# Generate delta files -ENV BUILD_DELTA false - # Delete old zips in $ZIP_DIR, keep only the N latest one (0 to disable) ENV DELETE_OLD_ZIPS 0 -# Delete old deltas in $DELTA_DIR, keep only the N latest one (0 to disable) -ENV DELETE_OLD_DELTAS 0 - # Delete old logs in $LOGS_DIR, keep only the N latest one (0 to disable) ENV DELETE_OLD_LOGS 0 -# Create a JSON file that indexes the build zips at the end of the build process -# (for the updates in OpenDelta). The file will be created in $ZIP_DIR with the -# specified name; leave empty to skip it. -# Requires ZIP_SUBDIR. -ENV OPENDELTA_BUILDS_JSON '' - # Save recovery image ENV RECOVERY_IMG false @@ -121,7 +111,6 @@ VOLUME $SRC_DIR VOLUME $CCACHE_DIR VOLUME $ZIP_DIR VOLUME $LMANIFEST_DIR -VOLUME $DELTA_DIR VOLUME $KEYS_DIR VOLUME $LOGS_DIR VOLUME $USERSCRIPTS_DIR @@ -137,24 +126,19 @@ RUN mkdir -p $SRC_DIR RUN mkdir -p $CCACHE_DIR RUN mkdir -p $ZIP_DIR RUN mkdir -p $LMANIFEST_DIR -RUN mkdir -p $DELTA_DIR RUN mkdir -p $KEYS_DIR RUN mkdir -p $LOGS_DIR RUN mkdir -p $USERSCRIPTS_DIR # Install build dependencies ############################ -COPY apt_preferences /etc/apt/preferences - -RUN echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list -RUN echo 'deb http://deb.debian.org/debian experimental main' >> /etc/apt/sources.list RUN apt-get -qq update RUN apt-get -qqy upgrade -RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \ +RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt clang cron \ curl flex g++-multilib gcc-multilib git gnupg gperf imagemagick kmod \ lib32ncurses5-dev libncurses5 lib32readline-dev lib32z1-dev libtinfo5 liblz4-tool \ - libncurses5-dev libsdl1.2-dev libssl-dev libwxgtk3.0-dev libxml2 \ + libncurses5-dev libsdl1.2-dev libssl-dev libxml2 \ libxml2-utils lsof lzop maven pngcrush \ procps python python3 rsync schedtool squashfs-tools software-properties-common wget xdelta3 xsltproc yasm \ zip zlib1g-dev @@ -169,24 +153,6 @@ RUN add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ RUN apt-get -qq update && apt-get install -y adoptopenjdk-8-hotspot RUN update-alternatives --set java /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java -# Download and build delta tools -################################ -RUN cd /root/ && \ - mkdir delta && \ - git clone --depth=1 https://gitlab.e.foundation/e/os/android_packages_apps_OpenDelta.git OpenDelta && \ - gcc -o delta/zipadjust OpenDelta/jni/zipadjust.c OpenDelta/jni/zipadjust_run.c -lz && \ - cp OpenDelta/server/minsignapk.jar OpenDelta/server/opendelta.sh delta/ && \ - chmod +x delta/opendelta.sh && \ - rm -rf OpenDelta/ && \ - sed -i -e 's|^\s*HOME=.*|HOME=/root|; \ - s|^\s*BIN_XDELTA=.*|BIN_XDELTA=xdelta3|; \ - s|^\s*FILE_MATCH=.*|FILE_MATCH=lineage-\*.zip|; \ - s|^\s*PATH_CURRENT=.*|PATH_CURRENT=$SRC_DIR/out/target/product/$DEVICE|; \ - s|^\s*PATH_LAST=.*|PATH_LAST=$SRC_DIR/delta_last/$DEVICE|; \ - s|^\s*KEY_X509=.*|KEY_X509=$KEYS_DIR/releasekey.x509.pem|; \ - s|^\s*KEY_PK8=.*|KEY_PK8=$KEYS_DIR/releasekey.pk8|; \ - s|publish|$DELTA_DIR|g' /root/delta/opendelta.sh - # Set the work directory ######################## WORKDIR $SRC_DIR diff --git a/Dockerfile.community b/Dockerfile.community index db51ef482b395c7a4056a7fa6eeb70dddfebc7a0..8ec19d6bfe5dbf040199a773270313fccdb8cf95 100644 --- a/Dockerfile.community +++ b/Dockerfile.community @@ -1,4 +1,4 @@ -FROM debian:buster +FROM ubuntu:20.04 MAINTAINER Romain Hunault # Environment variables @@ -10,7 +10,6 @@ ENV TMP_DIR /srv/tmp ENV CCACHE_DIR /srv/ccache ENV ZIP_DIR /srv/zips ENV LMANIFEST_DIR /srv/local_manifests -ENV DELTA_DIR /srv/delta ENV KEYS_DIR /srv/keys ENV LOGS_DIR /srv/logs ENV USERSCRIPTS_DIR /srv/userscripts @@ -30,9 +29,12 @@ ENV USE_CCACHE 1 # for no limit. ENV CCACHE_SIZE 50G +# We need to specify the ccache binary since it is no longer packaged along with AOSP +ENV CCACHE_EXEC /usr/bin/ccache + # Environment for the /e/ branches name # See https://github.com/LineageOS/android_vendor_cm/branches for possible options -ENV BRANCH_NAME 'v1-pie' +ENV BRANCH_NAME 'v1-q' # Environment for the device list (separate by comma if more than one) # eg. DEVICE_LIST=hammerhead,bullhead,angler @@ -55,7 +57,7 @@ ENV OTA_URL '' ENV USER_NAME 'LineageOS Buildbot' ENV USER_MAIL 'lineageos-buildbot@docker.host' -# Include proprietary files, downloaded automatically from github.com/TheMuppets/ +# Include proprietary files, downloaded automatically from github.com/TheMuppets/ and gitlab.com/the-muppets/ # Only some branches are supported ENV INCLUDE_PROPRIETARY true @@ -97,24 +99,12 @@ ENV ZIP_SUBDIR true # Write the verbose logs to $LOGS_DIR/$codename instead of $LOGS_DIR/ ENV LOGS_SUBDIR true -# Generate delta files -ENV BUILD_DELTA false - # Delete old zips in $ZIP_DIR, keep only the N latest one (0 to disable) ENV DELETE_OLD_ZIPS 0 -# Delete old deltas in $DELTA_DIR, keep only the N latest one (0 to disable) -ENV DELETE_OLD_DELTAS 0 - # Delete old logs in $LOGS_DIR, keep only the N latest one (0 to disable) ENV DELETE_OLD_LOGS 0 -# Create a JSON file that indexes the build zips at the end of the build process -# (for the updates in OpenDelta). The file will be created in $ZIP_DIR with the -# specified name; leave empty to skip it. -# Requires ZIP_SUBDIR. -ENV OPENDELTA_BUILDS_JSON '' - # You can optionally specify a USERSCRIPTS_DIR volume containing these scripts: # * begin.sh, run at the very beginning # * before.sh, run after the syncing and patching, before starting the builds @@ -132,7 +122,6 @@ VOLUME $TMP_DIR VOLUME $CCACHE_DIR VOLUME $ZIP_DIR VOLUME $LMANIFEST_DIR -VOLUME $DELTA_DIR VOLUME $KEYS_DIR VOLUME $LOGS_DIR VOLUME $USERSCRIPTS_DIR @@ -150,24 +139,19 @@ RUN mkdir -p $TMP_DIR RUN mkdir -p $CCACHE_DIR RUN mkdir -p $ZIP_DIR RUN mkdir -p $LMANIFEST_DIR -RUN mkdir -p $DELTA_DIR RUN mkdir -p $KEYS_DIR RUN mkdir -p $LOGS_DIR RUN mkdir -p $USERSCRIPTS_DIR # Install build dependencies ############################ -COPY apt_preferences /etc/apt/preferences - -RUN echo 'deb http://deb.debian.org/debian sid main' >> /etc/apt/sources.list -RUN echo 'deb http://deb.debian.org/debian experimental main' >> /etc/apt/sources.list RUN apt-get -qq update RUN apt-get -qqy upgrade -RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt cron \ +RUN apt-get install -y bc bison bsdmainutils build-essential ccache cgpt clang cron \ curl flex g++-multilib gcc-multilib git gnupg gperf imagemagick kmod \ lib32ncurses5-dev libncurses5 lib32readline-dev lib32z1-dev libtinfo5 liblz4-tool \ - libncurses5-dev libsdl1.2-dev libssl-dev libwxgtk3.0-dev libxml2 \ + libncurses5-dev libsdl1.2-dev libssl-dev libxml2 \ libxml2-utils lsof lzop maven pngcrush \ procps python python3 rsync schedtool squashfs-tools software-properties-common wget xdelta3 xsltproc yasm \ zip zlib1g-dev @@ -182,24 +166,6 @@ RUN add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ RUN apt-get -qq update && apt-get install -y adoptopenjdk-8-hotspot RUN update-alternatives --set java /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java -# Download and build delta tools -################################ -RUN cd /root/ && \ - mkdir delta && \ - git clone --depth=1 https://gitlab.e.foundation/e/os/android_packages_apps_OpenDelta.git OpenDelta && \ - gcc -o delta/zipadjust OpenDelta/jni/zipadjust.c OpenDelta/jni/zipadjust_run.c -lz && \ - cp OpenDelta/server/minsignapk.jar OpenDelta/server/opendelta.sh delta/ && \ - chmod +x delta/opendelta.sh && \ - rm -rf OpenDelta/ && \ - sed -i -e 's|^\s*HOME=.*|HOME=/root|; \ - s|^\s*BIN_XDELTA=.*|BIN_XDELTA=xdelta3|; \ - s|^\s*FILE_MATCH=.*|FILE_MATCH=lineage-\*.zip|; \ - s|^\s*PATH_CURRENT=.*|PATH_CURRENT=$SRC_DIR/out/target/product/$DEVICE|; \ - s|^\s*PATH_LAST=.*|PATH_LAST=$SRC_DIR/delta_last/$DEVICE|; \ - s|^\s*KEY_X509=.*|KEY_X509=$KEYS_DIR/releasekey.x509.pem|; \ - s|^\s*KEY_PK8=.*|KEY_PK8=$KEYS_DIR/releasekey.pk8|; \ - s|publish|$DELTA_DIR|g' /root/delta/opendelta.sh - # Set the work directory ######################## WORKDIR $SRC_DIR diff --git a/README.md b/README.md index f6792bca041fc3f995c447c15eb55436c0afe36c..677fda43e8ed5dd6bb4e10e3c90a302baa72ec4a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ TL;DR - go to the [Examples](#examples) The two fundamental settings are: - * `BRANCH_NAME (cm-14.1)`: LineageOS branch, see the branch list + * `BRANCH_NAME (lineage-16.0)`: LineageOS branch, see the branch list [here][los-branches] (multiple comma-separated branches can be specified) * `DEVICE_LIST`: comma-separated list of devices to build * `REPO (https://github.com/LineageOS/android.git)`: LineageOS repo use for build @@ -91,7 +91,8 @@ three ways: * by [pulling them from a running LineageOS][blobs-pull] * by [extracting them from a LineageOS ZIP][blobs-extract] - * by downloading them from [TheMuppets repos][blobs-themuppets] (unofficial) + * by downloading them from TheMuppets [GitHub][blobs-themuppets] and + [GitLab][blobs-the-muppets] repositories (unofficial) The third way is the easiest one and is enabled by default; if you're OK with that just move on, otherwise set `INCLUDE_PROPRIETARY (true)` to `false` and @@ -164,12 +165,12 @@ When `LOCAL_MIRROR` is `true`: ## Examples -### Build for thea (cm-14.1, officially supported), test keys, no patches +### Build for bacon (lineage-16.0, officially supported), test keys, no patches ``` docker run \ - -e "BRANCH_NAME=cm-14.1" \ - -e "DEVICE_LIST=thea" \ + -e "BRANCH_NAME=lineage-16.0" \ + -e "DEVICE_LIST=bacon" \ -v "/home/user/lineage:/srv/src" \ -v "/home/user/zips:/srv/zips" \ -v "/home/user/logs:/srv/logs" \ @@ -177,12 +178,12 @@ docker run \ lineageos4microg/docker-lineage-cicd ``` -### Build for dumpling (lineage-15.1, officially supported), custom keys, restricted signature spoofing with integrated microG and FDroid +### Build for angler (lineage-15.1, officially supported), custom keys, restricted signature spoofing with integrated microG and FDroid ``` docker run \ -e "BRANCH_NAME=lineage-15.1" \ - -e "DEVICE_LIST=dumpling" \ + -e "DEVICE_LIST=angler" \ -e "SIGN_BUILDS=true" \ -e "SIGNATURE_SPOOFING=restricted" \ -e "CUSTOM_PACKAGES=GmsCore GsfProxy FakeStore MozillaNlpBackend NominatimNlpBackend com.google.android.maps.jar FDroid FDroidPrivilegedExtension " \ @@ -212,13 +213,13 @@ it ends with `.xml`) in the `/home/user/manifests` folder with this content: ``` -### Build for four devices on cm-14.1 and lineage-15.1 (officially supported), custom keys, restricted signature spoofing with integrated microG and FDroid, custom OTA server +### Build for four devices on lineage-15.1 and lineage-16.0 (officially supported), custom keys, restricted signature spoofing with integrated microG and FDroid, custom OTA server ``` docker run \ - -e "BRANCH_NAME=cm-14.1,lineage-15.1" \ - -e "DEVICE_LIST_CM_14_1=onyx,thea" \ - -e "DEVICE_LIST_LINEAGE_15_1=cheeseburger,dumpling" \ + -e "BRANCH_NAME=lineage-15.1,lineage-16.0" \ + -e "DEVICE_LIST_LINEAGE_15_1=angler,oneplus2" \ + -e "DEVICE_LIST_LINEAGE_16_0=bacon,dumpling" \ -e "SIGN_BUILDS=true" \ -e "SIGNATURE_SPOOFING=restricted" \ -e "CUSTOM_PACKAGES=GmsCore GsfProxy FakeStore MozillaNlpBackend NominatimNlpBackend com.google.android.maps.jar FDroid FDroidPrivilegedExtension " \ @@ -309,6 +310,7 @@ docker run \ [blobs-pull]: https://wiki.lineageos.org/devices/bacon/build#extract-proprietary-blobs [blobs-extract]: https://wiki.lineageos.org/extracting_blobs_from_zips.html [blobs-themuppets]: https://github.com/TheMuppets/manifests +[blobs-the-muppets]: https://gitlab.com/the-muppets/manifest [lineageota]: https://github.com/julianxhokaxhiu/LineageOTA [los-extras]: https://download.lineageos.org/extras [dockerfile]: Dockerfile diff --git a/build-community.sh b/build-community.sh index e5feb4d58ff031956fcd935c9e465a5b0cbe5437..6588bf9fe80d7c5a29e69d926390a0dfa989534e 100755 --- a/build-community.sh +++ b/build-community.sh @@ -61,6 +61,8 @@ if [ "$LOCAL_MIRROR" = true ]; then rm -f .repo/local_manifests/proprietary.xml if [ "$INCLUDE_PROPRIETARY" = true ]; then wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/mirror/default.xml" + /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ + "https://gitlab.com/the-muppets/manifest/raw/mirror/default.xml" .repo/local_manifests/proprietary_gitlab.xml fi echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" @@ -78,6 +80,35 @@ for branch in ${BRANCH_NAME//,/ }; do devices=${!device_list_cur_branch} if [ -n "$branch" ] && [ -n "$devices" ]; then + vendor=lineage + case "$branch" in + *nougat*) + vendor="cm" + themuppets_branch="cm-14.1" + android_version="7.1.2" + ;; + *oreo*) + themuppets_branch="lineage-15.1" + android_version="8.1" + ;; + *pie*) + themuppets_branch="lineage-16.0" + android_version="9" + ;; + q*) + themuppets_branch="lineage-17.1" + android_version="10" + ;; + ;; + + *) + echo ">> [$(date)] Building branch $branch is not (yet) suppported" + exit 1 + ;; + esac + + android_version_major=$(cut -d '.' -f 1 <<< $android_version) + mkdir -p "$SRC_DIR/$branch_dir" cd "$SRC_DIR/$branch_dir" @@ -117,23 +148,9 @@ for branch in ${BRANCH_NAME//,/ }; do rm -f .repo/local_manifests/proprietary.xml if [ "$INCLUDE_PROPRIETARY" = true ]; then - if [[ $branch =~ nougat$ ]]; then - themuppets_branch=cm-14.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ $branch =~ oreo$ ]]; then - themuppets_branch=lineage-15.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ $branch =~ pie$ ]]; then - themuppets_branch=lineage-16.0 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ $branch =~ q$ ]]; then - themuppets_branch=lineage-17.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - else - themuppets_branch=cm-14.1 - echo ">> [$(date)] Can't find a matching branch on github.com/TheMuppets, using $themuppets_branch" - fi wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml" + /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ + "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml fi echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log" @@ -144,25 +161,6 @@ for branch in ${BRANCH_NAME//,/ }; do sync_successful=false fi - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - echo ">> [$(date)] Can't detect the android version" - exit 1 - fi - fi - fi - android_version_major=$(cut -d '.' -f 1 <<< $android_version) - - if [ "$android_version_major" -ge "8" ]; then - vendor="lineage" - else - vendor="cm" - fi - if [ ! -d "vendor/$vendor" ]; then echo ">> [$(date)] Missing \"vendor/$vendor\", aborting" exit 1 @@ -201,21 +199,15 @@ for branch in ${BRANCH_NAME//,/ }; do echo ">> [$(date)] Adding keys path ($KEYS_DIR)" # Soong (Android 9+) complains if the signing keys are outside the build path ln -sf "$KEYS_DIR" user-keys - sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" - fi + if [ "$android_version_major" -lt "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi - if [ "$android_version_major" -ge "7" ]; then - jdk_version=8 - elif [ "$android_version_major" -ge "5" ]; then - jdk_version=7 - else - echo ">> [$(date)] ERROR: $branch requires a JDK version too old (< 7); aborting" - exit 1 + if [ "$android_version_major" -ge "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi fi - echo ">> [$(date)] Using OpenJDK $jdk_version" - update-java-alternatives -s java-1.$jdk_version.0-openjdk-amd64 &> /dev/null - # Prepare the environment echo ">> [$(date)] Preparing build environment" source build/envsetup.sh > /dev/null @@ -305,27 +297,6 @@ for branch in ${BRANCH_NAME//,/ }; do find out/target/product/$codename -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; &>> "$DEBUG_LOG" fi - if [ "$BUILD_DELTA" = true ]; then - if [ -d "delta_last/$codename/" ]; then - # If not the first build, create delta files - echo ">> [$(date)] Generating delta files for $codename" | tee -a "$DEBUG_LOG" - cd /root/delta - if ./opendelta.sh $codename &>> "$DEBUG_LOG"; then - echo ">> [$(date)] Delta generation for $codename completed" | tee -a "$DEBUG_LOG" - else - echo ">> [$(date)] Delta generation for $codename failed" | tee -a "$DEBUG_LOG" - fi - if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then - /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS -V $los_ver -N 1 "$DELTA_DIR/$codename" &>> $DEBUG_LOG - fi - cd "$source_dir" - else - # If the first build, copy the current full zip in $source_dir/delta_last/$codename/ - echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta" | tee -a "$DEBUG_LOG" - mkdir -p delta_last/$codename/ &>> "$DEBUG_LOG" - find out/target/product/$codename -maxdepth 1 -name 'e-*.zip' -type f -exec cp {} "$source_dir/delta_last/$codename/" \; &>> "$DEBUG_LOG" - fi - fi # Move produced ZIP files to the main OUT directory echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG" cd out/target/product/$codename @@ -393,14 +364,6 @@ for branch in ${BRANCH_NAME//,/ }; do fi done -# Create the OpenDelta's builds JSON file -if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then - echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)" - if [ "$ZIP_SUBDIR" != true ]; then - echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true" - fi - /usr/bin/python /root/opendelta_builds_json.py "$ZIP_DIR" -o "$ZIP_DIR/$OPENDELTA_BUILDS_JSON" -fi if [ "$DELETE_OLD_LOGS" -gt "0" ]; then find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm diff --git a/src/build.sh b/src/build.sh index 28e396479e22455cc23e3fb38b2ae52068db47fa..24bb23d57f92637c03f5a027ea7bcaa6239d9bf9 100755 --- a/src/build.sh +++ b/src/build.sh @@ -37,6 +37,32 @@ branch_dir=$(sed 's/.*-\([a-zA-Z]*\)$/\1/' <<< ${BRANCH_NAME}) branch_dir=${branch_dir^^} if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then + vendor=lineage + case "$BRANCH_NAME" in + *nougat*) + vendor="cm" + themuppets_branch="cm-14.1" + android_version="7.1.2" + ;; + *oreo*) + themuppets_branch="lineage-15.1" + android_version="8.1" + ;; + *pie*) + themuppets_branch="lineage-16.0" + android_version="9" + ;; + *q*) + themuppets_branch="lineage-17.1" + android_version="10" + ;; + *) + echo ">> [$(date)] Building branch $branch is not (yet) suppported" + exit 1 + ;; + esac + + android_version_major=$(cut -d '.' -f 1 <<< $android_version) mkdir -p "$SRC_DIR/$branch_dir" cd "$SRC_DIR/$branch_dir" @@ -72,23 +98,10 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then rm -f .repo/local_manifests/proprietary.xml if [ "$INCLUDE_PROPRIETARY" = true ]; then - if [[ ${BRANCH_NAME} =~ nougat$ ]]; then - themuppets_branch=cm-14.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ ${BRANCH_NAME} =~ oreo$ ]]; then - themuppets_branch=lineage-15.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ ${BRANCH_NAME} =~ pie$ ]]; then - themuppets_branch=lineage-16.0 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - elif [[ ${BRANCH_NAME} =~ q$ ]]; then - themuppets_branch=lineage-17.1 - echo ">> [$(date)] Use branch $themuppets_branch on github.com/TheMuppets" - else - themuppets_branch=cm-14.1 - echo ">> [$(date)] Can't find a matching branch on github.com/TheMuppets, using $themuppets_branch" - fi wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml" + /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ + "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml + fi echo ">> [$(date)] Syncing branch repository" @@ -99,28 +112,6 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then sync_successful=false fi - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.QP1A := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.OPM1 := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION\.PPR1 := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - android_version=$(sed -n -e 's/^\s*PLATFORM_VERSION := //p' build/core/version_defaults.mk) - if [ -z $android_version ]; then - echo ">> [$(date)] Can't detect the android version" - exit 1 - fi - fi - fi - fi - android_version_major=$(cut -d '.' -f 1 <<< $android_version) - - if [ "$android_version_major" -ge "8" ]; then - vendor="lineage" - else - vendor="cm" - fi - if [ ! -d "vendor/$vendor" ]; then echo ">> [$(date)] Missing \"vendor/$vendor\", aborting" exit 1 @@ -134,11 +125,14 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then echo ">> [$(date)] Adding keys path ($KEYS_DIR)" # Soong (Android 9+) complains if the signing keys are outside the build path ln -sf "$KEYS_DIR" user-keys - sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" - fi + if [ "$android_version_major" -lt "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi - echo ">> [$(date)] Using OpenJDK $jdk_version" - update-java-alternatives -s java-1.$jdk_version.0-openjdk-amd64 &> /dev/null + if [ "$android_version_major" -ge "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi + fi # Prepare the environment echo ">> [$(date)] Preparing build environment" @@ -209,27 +203,6 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then find out/target/product/${DEVICE} -maxdepth 1 -name "e-*-$currentdate-*.zip*" -type f -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; fi - if [ "$BUILD_DELTA" = true ]; then - if [ -d "delta_last/${DEVICE}/" ]; then - # If not the first build, create delta files - echo ">> [$(date)] Generating delta files for ${DEVICE}" - cd /root/delta - if ./opendelta.sh ${DEVICE}; then - echo ">> [$(date)] Delta generation for ${DEVICE} completed" - else - echo ">> [$(date)] Delta generation for ${DEVICE} failed" - fi - if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then - /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS -V $los_ver -N 1 "$DELTA_DIR/${DEVICE}" - fi - cd "$source_dir" - else - # If the first build, copy the current full zip in $source_dir/delta_last/${DEVICE}/ - echo ">> [$(date)] No previous build for ${DEVICE}; using current build as base for the next delta" - mkdir -p delta_last/${DEVICE}/ - find out/target/product/${DEVICE} -maxdepth 1 -name 'e-*.zip' -type f -exec cp {} "$source_dir/delta_last/${DEVICE}/" \; - fi - fi # Move produced ZIP files to the main OUT directory echo ">> [$(date)] Moving build artifacts for ${DEVICE} to '$ZIP_DIR/$zipsubdir'" cd out/target/product/${DEVICE} @@ -293,15 +266,6 @@ if [ -n "${BRANCH_NAME}" ] && [ -n "${DEVICE}" ]; then fi -# Create the OpenDelta's builds JSON file -if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then - echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)" - if [ "$ZIP_SUBDIR" != true ]; then - echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true" - fi - /usr/bin/python /root/opendelta_builds_json.py "$ZIP_DIR" -o "$ZIP_DIR/$OPENDELTA_BUILDS_JSON" -fi - if [ "$DELETE_OLD_LOGS" -gt "0" ]; then find "$LOGS_DIR" -maxdepth 1 -name repo-*.log | sort | head -n -$DELETE_OLD_LOGS | xargs -r rm fi diff --git a/src/build_manifest.py b/src/build_manifest.py new file mode 100755 index 0000000000000000000000000000000000000000..32f9411ab84238b9a22e5f60826c30e81589c8bb --- /dev/null +++ b/src/build_manifest.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# +# Copyright (C) 2019 Nicola Corna +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from xml.dom import minidom +import xml.etree.ElementTree as ET +import argparse + +try: + from urllib.request import urlopen, Request +except ImportError: + from urllib2 import urlopen, Request + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Build an Android repo manifest') + parser.add_argument('url', type=str, help='URL of the source manifest') + parser.add_argument('out', type=str, help='Output path') + parser.add_argument('--remote', type=str, help='Remote URL') + parser.add_argument('--remotename', type=str, help='Remote name') + + args = parser.parse_args() + + request = Request(args.url, headers={"User-Agent": "Mozilla/5.0"}) + source_manifest = urlopen(request).read() + + xmlin = ET.fromstring(source_manifest) + xmlout = ET.Element("manifest") + + if args.remote: + ET.SubElement(xmlout, 'remote', attrib={"name": args.remotename, + "fetch": args.remote}) + + for child in xmlin: + if child.tag == "project": + attributes = {} + attributes["name"] = child.attrib["name"] + + if "path" in child.attrib: + attributes["path"] = child.attrib["path"] + + if args.remote: + attributes["remote"] = args.remotename + + ET.SubElement(xmlout, 'project', attrib=attributes) + + xmlstr = minidom.parseString(ET.tostring(xmlout)).toprettyxml(indent=" ", encoding="UTF-8") + with open(args.out, "w") as f: + f.write(xmlstr) diff --git a/src/init.sh b/src/init.sh index 57656d597070d7a3c693f1b57b0d73578e63df38..977f9355fee6202b0e32a828289dce469b007d6c 100755 --- a/src/init.sh +++ b/src/init.sh @@ -35,12 +35,12 @@ git config --global user.email $USER_MAIL if [ "$SIGN_BUILDS" = true ]; then if [ -z "$(ls -A "$KEYS_DIR")" ]; then echo ">> [$(date)] SIGN_BUILDS = true but empty \$KEYS_DIR, generating new keys" - for c in releasekey platform shared media; do + for c in releasekey platform shared media networkstack; do echo ">> [$(date)] Generating $c..." /root/make_key "$KEYS_DIR/$c" "$KEYS_SUBJECT" <<< '' &> /dev/null done else - for c in releasekey platform shared media; do + for c in releasekey platform shared media networkstack; do for e in pk8 x509.pem; do if [ ! -f "$KEYS_DIR/$c.$e" ]; then echo ">> [$(date)] SIGN_BUILDS = true and not empty \$KEYS_DIR, but \"\$KEYS_DIR/$c.$e\" is missing" diff --git a/src/opendelta_builds_json.py b/src/opendelta_builds_json.py deleted file mode 100755 index 9ae338ea963853d7316a92d9c75e447debe95076..0000000000000000000000000000000000000000 --- a/src/opendelta_builds_json.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2017 Nicola Corna -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from sys import argv -from argparse import ArgumentParser -import os -import json - -if __name__ == "__main__": - parser = ArgumentParser(description='Generate an OpenDelta\'s builds.json ' - 'file') - parser.add_argument('path', metavar='PATH', type=str, help='the directory ' - 'containing the zips') - parser.add_argument('-o', "--output", type=str, help='output file; ' - 'if unspecified, print to stdout') - args = parser.parse_args() - - data = {} - builddirs = ['./' + s for s in os.listdir(args.path)] - for builddir in builddirs: - try: - builds = os.listdir(os.path.join(args.path, builddir)) - data[builddir] = [dict() for x in range(len(builds))] - for i in range(0, len(builds)): - data[builddir][i]["filename"] = builds[i] - data[builddir][i]["timestamp"] = int(os.path.getmtime( - os.path.join(args.path, builddir, builds[i]))) - except OSError: - pass - - if args.output: - with open(args.output, "w") as f: - f.write(json.dumps(data, separators=(',',':'))) - else: - print(json.dumps(data, separators=(',',':')))