From 3753d60e52f8dc2d2c6381b7b0aa849fc52e8055 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 29 Sep 2021 17:24:15 +0530 Subject: [PATCH 01/72] Updated version numbers of all images and their references --- scripts/generate-signup-link.sh | 2 +- scripts/postfixadmin-mailbox-postdeletion.sh | 2 +- scripts/ssl-renew.sh | 2 +- .../docker-compose/docker-compose-base.yml | 37 ++++++------------- .../docker-compose-networks.yml | 7 ++-- templates/nginx/sites-enabled/rspamd.conf | 2 +- 6 files changed, 18 insertions(+), 34 deletions(-) diff --git a/scripts/generate-signup-link.sh b/scripts/generate-signup-link.sh index bbe839a..3704879 100755 --- a/scripts/generate-signup-link.sh +++ b/scripts/generate-signup-link.sh @@ -29,4 +29,4 @@ echo -e "to:$EMAIL from:drive@$DOMAIN subject:Signup for $DOMAIN You can now sign up for your $DOMAIN account at $SIGNUP_URL" | \ - docker exec -i $(docker-compose ps -q eelomailserver) sendmail -t + docker exec -i $(docker-compose ps -q mailserver) sendmail -t diff --git a/scripts/postfixadmin-mailbox-postdeletion.sh b/scripts/postfixadmin-mailbox-postdeletion.sh index f3c4781..e89e9e3 100755 --- a/scripts/postfixadmin-mailbox-postdeletion.sh +++ b/scripts/postfixadmin-mailbox-postdeletion.sh @@ -7,7 +7,7 @@ # the script is actually run by the pfexec user -# the script handles deletion in a bind-mounted dir shared with eelomailserver +# the script handles deletion in a bind-mounted dir shared with mailserver # so pfexec user has no right over it. it needs a specific sudo perm #to be able to only run this script # the /etc/sudoers line added to the container during install : diff --git a/scripts/ssl-renew.sh b/scripts/ssl-renew.sh index 0dfb204..61c3a6b 100755 --- a/scripts/ssl-renew.sh +++ b/scripts/ssl-renew.sh @@ -27,7 +27,7 @@ cat "$CONFIG" | while read DOMAIN; do echo "Certificate for $DOMAIN renewed and is valid until: $VALID_UNTIL" docker-compose exec -T nginx nginx -s reload if [ "$DOMAIN" = "$MAILHOST" ]; then - docker-compose restart eelomailserver + docker-compose restart mailserver fi fi fi diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index dc1873e..438effe 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -1,9 +1,9 @@ -version: '2.1' +version: '3' services: - eelomailserver: - image: hardware/mailserver:1.1-stable - container_name: eelomailserver + mailserver: + image: mailserver2/mailserver:1.1.4 + container_name: mailserver domainname: ${DOMAIN} # Mail server A/MX/FQDN & reverse PTR = mail.${DOMAIN}. hostname: mail restart: always @@ -40,7 +40,7 @@ services: - redis postfixadmin: - image: registry.gitlab.e.foundation:5000/e/infra/docker-postfixadmin:1.0.0 + image: registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1 container_name: postfixadmin restart: always networks: @@ -55,11 +55,11 @@ services: - /mnt/repo-base/volumes/mail:/var/mail - /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh depends_on: - - eelomailserver + - mailserver - mariadb mariadb: - image: mariadb:10.3.17 + image: mariadb:10.3 container_name: mariadb restart: always networks: @@ -75,7 +75,7 @@ services: - /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d redis: - image: redis:5.0-alpine + image: redis:6.0-alpine container_name: redis restart: always networks: @@ -86,7 +86,7 @@ services: - /mnt/repo-base/volumes/redis/tmp:/tmp/redis welcome: - image: registry.gitlab.e.foundation:5000/e/infra/docker-welcome:1.1.0 + image: registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2 container_name: welcome environment: - DOMAINS=${VHOSTS_ACCOUNTS} @@ -112,7 +112,7 @@ services: - mariadb nextcloud: - image: nextcloud:16.0.5-fpm + image: nextcloud:20.0.12-fpm container_name: nextcloud restart: always networks: @@ -129,7 +129,7 @@ services: - mariadb automx: - image: registry.gitlab.e.foundation:5000/e/infra/docker-mailstack:automx-0.1.0 + image: registry.gitlab.e.foundation/e/infra/docker-mailstack:automx-0.1.0 container_name: automx hostname: automx environment: @@ -142,18 +142,3 @@ services: volumes: - /mnt/repo-base/config-dynamic/automx/automx.conf:/etc/automx.conf - create-account: - image: registry.gitlab.e.foundation:5000/e/infra/docker-create-account:1.0.1 - container_name: create-account - restart: always - environment: - - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} - - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} - - DOMAIN=${DOMAIN} - - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} - networks: - - serverbase - depends_on: - - nextcloud - - postfixadmin diff --git a/templates/docker-compose/docker-compose-networks.yml b/templates/docker-compose/docker-compose-networks.yml index 84840ec..099ccc1 100644 --- a/templates/docker-compose/docker-compose-networks.yml +++ b/templates/docker-compose/docker-compose-networks.yml @@ -1,8 +1,8 @@ nginx: - image: registry.gitlab.e.foundation:5000/e/infra/docker-nginx:1.17 + image: nginx:1.19-alpine container_name: nginx - restart: always + restart: unless-stopped networks: - serverbase ports: @@ -18,11 +18,10 @@ - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ depends_on: - nextcloud - - create-account - automx - postfixadmin - welcome - - eelomailserver + - mailserver #- onlyoffice-community-server networks: diff --git a/templates/nginx/sites-enabled/rspamd.conf b/templates/nginx/sites-enabled/rspamd.conf index 067b466..fc722b9 100644 --- a/templates/nginx/sites-enabled/rspamd.conf +++ b/templates/nginx/sites-enabled/rspamd.conf @@ -26,7 +26,7 @@ server { #auth_basic_user_file /passwds/.htpasswd; location / { - proxy_pass http://eelomailserver:11334; + proxy_pass http://mailserver:11334; include /etc/nginx/params/proxy_params; } } -- GitLab From 79767fc7722e3c3f647981808ce97355a6a1b9f7 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 1 Oct 2021 15:15:46 +0530 Subject: [PATCH 02/72] Changed db volume location and removed blank option from postinstall.sh --- scripts/postinstall.sh | 2 +- templates/docker-compose/docker-compose-base.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 78c204d..be42165 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -18,7 +18,7 @@ docker-compose exec -T --user www-data nextcloud php occ maintenance:install \ --admin-user="$NEXTCLOUD_ADMIN_USER" --admin-pass="$NEXTCLOUD_ADMIN_PASSWORD" \ --admin-email="$ALT_EMAIL" --database="mysql" --database-pass="$MYSQL_PASSWORD_NC" \ --database-name="$MYSQL_DATABASE_NC" --database-host="mariadb" --database-user="$MYSQL_USER_NC" \ - --database-port="3306" --database-table-prefix="" + --database-port="3306" docker-compose exec -T --user www-data nextcloud php occ db:convert-filecache-bigint --no-interaction # Nextcloud resets trusted_domains to localhost during installation, so we have to set it again diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 438effe..9e87c01 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -71,7 +71,7 @@ services: - MYSQL_USER=${PFDB_USR} - MYSQL_PASSWORD=${DBPASS} volumes: - - /mnt/repo-base/volumes/mysql/db:/var/lib/mysql + - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql - /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d redis: -- GitLab From 5668b3fc4de112b09c9efa04c7afd3968f145e6b Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 6 Oct 2021 14:12:17 +0530 Subject: [PATCH 03/72] edited postinstall.sh to enable apps instead of install them using occ --- scripts/postinstall.sh | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index be42165..2949c56 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -23,14 +23,24 @@ docker-compose exec -T --user www-data nextcloud php occ db:convert-filecache-bi # Nextcloud resets trusted_domains to localhost during installation, so we have to set it again docker-compose exec -T --user www-data nextcloud php occ config:system:set trusted_domains 0 --value="$DOMAIN" +docker-compose exec -T --user www-data nextcloud php occ app:disable theming + +echo "Enabling nextcloud apps" +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable calendar +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable notes +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable user_backend_sql_raw +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable rainloop +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable quota_warning +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable contacts +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable news +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable email-recovery +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud_drop_account +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud-theme-helper +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud-launcher +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 -echo "Installing nextcloud plugins" -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install calendar docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install notes -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install user_backend_sql_raw -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install rainloop -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install photos git clone --single-branch https://framagit.org/tcit/drop_user.git volumes/nextcloud/custom_apps/drop_account docker-compose exec -T --user www-data nextcloud php occ app:enable drop_account @@ -39,8 +49,6 @@ echo "Installing custom ecloud drop account plugin" docker-compose exec -T --user www-data nextcloud php occ config:system:set e_welcome_secret --value="$WELCOME_SECRET" # Add VHOST_ACCOUNTS from .env file as a system config value, to be used by our ecloud_drop_account plugin docker-compose exec -T --user www-data nextcloud php occ config:system:set e_welcome_domain --value="welcome.$DOMAIN" -git clone --single-branch https://gitlab.e.foundation/e/infra/selfhost/nextcloud-apps/ecloud-drop-account.git volumes/nextcloud/custom_apps/ecloud_drop_account -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud_drop_account echo "Installing Nextcloud theme" @@ -69,11 +77,6 @@ curl --silent -L https://mail.$DOMAIN/setup.php > /dev/null echo "Adding Postfix admin superadmin account" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli admin add $ALT_EMAIL --password $PFA_SUPERADMIN_PASSWORD --password2 $PFA_SUPERADMIN_PASSWORD --superadmin -# adding sudo to postfixadmin container -docker-compose exec -T postfixadmin apk add sudo -# giving pfexec user a specific sudo perm ONLY for launching the bind-mounted mailbox-postdeletion script -docker-compose exec -T postfixadmin bash -c 'echo "" >> /etc/sudoers && echo "#pfexec single command perm" >> /etc/sudoers && echo "pfexec ALL=(root) NOPASSWD: /usr/local/bin/postfixadmin-mailbox-postdeletion.sh" >> /etc/sudoers' - # Adding domains to postfix is done by docker exec instead of docker-compose exec on purpose. Reason: with compose the loop aborts after the first item for an unknown reason echo "Adding domains to Postfix" # The password_expiry parameter is only a workaround, and does not have any effect @@ -87,7 +90,7 @@ docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailb # display DKIM DNS setup info/instructions to the user echo -e "\n\n\n" echo -e "Please add the following records to your domain's DNS configuration:\n" -find /mnt/repo-base/volumes/mail/dkim/ -maxdepth 1 -mindepth 1 -type d | while read line; do DOMAIN=$(basename $line); echo " - DKIM record (TXT) for $DOMAIN:" && cat $line/public.key; done +find /mnt/repo-base/volumes/mail/dkim/ -maxdepth 1 -mindepth 1 -type d | while read line; do DOMAIN=$(basename $line); echo " - DKIM record (TXT) for $DOMAIN:" && cat $line/mail.public.key; done echo "=================================================================================================================================" echo "=================================================================================================================================" -- GitLab From a4efee2de5ee95b645fa279f18c26a43011c7402 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 6 Oct 2021 14:15:00 +0530 Subject: [PATCH 04/72] Updated nc to custom image; update welcome to latest --- .../docker-compose/docker-compose-base.yml | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 9e87c01..93d20d1 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -47,13 +47,20 @@ services: - serverbase environment: - DBPASS=${DBPASS} + - DBHOST=mariadb #point to NC1 on wg network - DOMAIN=${DOMAIN} + - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} - - SMTPHOST=${SMTP_HOST} - - ADMIN_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} + - POSTFIXADMIN_DB_TYPE=mysqli + - POSTFIXADMIN_DB_HOST=mariadb + - POSTFIXADMIN_DB_USER=postfix + - POSTFIXADMIN_DB_NAME=postfix + - POSTFIXADMIN_DB_PASSWORD=${DBPASS} + - POSTFIXADMIN_SMTP_SERVER=mailserver + - POSTFIXADMIN_SMTP_PORT=587 + volumes: - /mnt/repo-base/volumes/mail:/var/mail - - /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh depends_on: - mailserver - mariadb @@ -102,6 +109,10 @@ services: - SMTP_HOST=${SMTP_HOST} - SMTP_FROM=${SMTP_FROM} - SMTP_PW=${SMTP_PW} + - SMTP_PORT=587 + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} restart: always networks: @@ -112,11 +123,22 @@ services: - mariadb nextcloud: - image: nextcloud:20.0.12-fpm + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:master container_name: nextcloud restart: always networks: - serverbase + environment: + - MYSQL_DATABASE=${MYSQL_DATABASE_NC} + - MYSQL_USER=${MYSQL_USER_NC} + - MYSQL_PASSWORD=${MYSQL_PASSWORD_NC} + - MYSQL_HOST=mariadb + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + # below ENV disabled so NC container do not start install + # - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - OVERWRITEPROTOCOL=https + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + volumes: - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ -- GitLab From 07c1d671451fa4003d9378522c475edf7bef20f6 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 6 Oct 2021 14:15:35 +0530 Subject: [PATCH 05/72] Added recovery app secret to questionnaire --- deployment/questionnaire/questionnaire.dat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/questionnaire/questionnaire.dat b/deployment/questionnaire/questionnaire.dat index 74dff8e..acbca2c 100644 --- a/deployment/questionnaire/questionnaire.dat +++ b/deployment/questionnaire/questionnaire.dat @@ -26,6 +26,8 @@ PFA_SUPERADMIN_PASSWORD=1@@@generate@@@:16@2 WELCOME_SECRET=@@@generate@@@:20@ +NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=@@@generate@@@:15@ + # fixed defaults ENABLE_POP3=false;default DISABLE_RATELIMITING=false;default -- GitLab From d38c2b01e54c89a19662acd777a024e53ebac0da Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 6 Oct 2021 16:34:38 +0530 Subject: [PATCH 06/72] Updated rainloop domain config correctly --- templates/rainloop/domain-config.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/rainloop/domain-config.ini b/templates/rainloop/domain-config.ini index 06299c9..179cbc3 100644 --- a/templates/rainloop/domain-config.ini +++ b/templates/rainloop/domain-config.ini @@ -1,13 +1,13 @@ -imap_host = "eelomailserver" +imap_host = "mailserver" imap_port = 993 imap_secure = "SSL" imap_short_login = Off sieve_use = Off sieve_allow_raw = Off -sieve_host = "eelomailserver" +sieve_host = "mailserver" sieve_port = 4190 sieve_secure = "TLS" -smtp_host = "eelomailserver" +smtp_host = "mailserver" smtp_port = 587 smtp_secure = "TLS" smtp_short_login = Off -- GitLab From a252ae58d073774da3874843e467da68e4cdb3fb Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 6 Oct 2021 16:35:18 +0530 Subject: [PATCH 07/72] Disabled first run wizard --- scripts/postinstall.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 2949c56..2b6aa6c 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -37,6 +37,7 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enabl docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud_drop_account docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud-theme-helper docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud-launcher +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:disable firstrunwizard docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks -- GitLab From d62fee51ea42aeb7e2f910e59c93b27651680d2b Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 13:12:43 +0530 Subject: [PATCH 08/72] Added freshclam.conf and updated drop_account --- config-static/mail/clamav/freshclam.conf | 33 +++++++++++++++++++ scripts/postinstall.sh | 2 +- .../docker-compose/docker-compose-base.yml | 3 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 config-static/mail/clamav/freshclam.conf diff --git a/config-static/mail/clamav/freshclam.conf b/config-static/mail/clamav/freshclam.conf new file mode 100644 index 0000000..6b53852 --- /dev/null +++ b/config-static/mail/clamav/freshclam.conf @@ -0,0 +1,33 @@ +# Automatically created by the clamav-freshclam postinst +# Comments will get lost when you reconfigure the clamav-freshclam package + +DatabaseOwner clamav +UpdateLogFile /var/log/clamav/freshclam.log +LogVerbose false +LogSyslog true +LogFacility LOG_MAIL +LogFileMaxSize 0 +LogRotate true +LogTime true +Foreground true +Debug false +MaxAttempts 3 +DatabaseDirectory /var/lib/clamav +DNSDatabaseInfo current.cvd.clamav.net +ConnectTimeout 30 +ReceiveTimeout 0 +TestDatabases yes +ScriptedUpdates yes +CompressLocalDatabase no +SafeBrowsing false +Bytecode true +NotifyClamd /etc/clamav/clamd.conf +# Check for new database 24 times a day +Checks 4 +DatabaseMirror db.local.clamav.net +DatabaseMirror switch.clamav.net +DatabaseMirror clamav.easynet.fr +DatabaseMirror clamav.begi.net +DatabaseMirror clamav.univ-nantes.fr +DatabaseMirror db.fr.clamav.net + diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 2b6aa6c..aaf6a40 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -42,7 +42,7 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:ap docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install photos -git clone --single-branch https://framagit.org/tcit/drop_user.git volumes/nextcloud/custom_apps/drop_account +git clone --single-branch https://framagit.org/framasoft/nextcloud/drop_account.git volumes/nextcloud/custom_apps/drop_account docker-compose exec -T --user www-data nextcloud php occ app:enable drop_account echo "Installing custom ecloud drop account plugin" diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 93d20d1..3136c04 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -35,6 +35,7 @@ services: - /mnt/repo-base/config-static/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf - /mnt/repo-base/config-static/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map - /mnt/repo-base/config-static/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf + - /mnt/repo-base/config-static/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf depends_on: - mariadb - redis @@ -47,7 +48,7 @@ services: - serverbase environment: - DBPASS=${DBPASS} - - DBHOST=mariadb #point to NC1 on wg network + - DBHOST=mariadb - DOMAIN=${DOMAIN} - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} -- GitLab From 7844899a0ca86cd58a26558dc0062213a29f8367 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 13:45:09 +0530 Subject: [PATCH 09/72] Updated nginx nextcloud.conf template --- config-static/nginx/params/proxy_params | 2 + templates/nginx/sites-enabled/nextcloud.conf | 81 ++++++++++++++------ 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/config-static/nginx/params/proxy_params b/config-static/nginx/params/proxy_params index 7a40d4d..8107da6 100644 --- a/config-static/nginx/params/proxy_params +++ b/config-static/nginx/params/proxy_params @@ -7,6 +7,8 @@ proxy_redirect off; proxy_headers_hash_max_size 1024; proxy_headers_hash_bucket_size 128; +proxy_http_version 1.1; +proxy_set_header Connection ""; # No compression for json to avoid BREACH attack. gzip on; diff --git a/templates/nginx/sites-enabled/nextcloud.conf b/templates/nginx/sites-enabled/nextcloud.conf index f8aebf5..e360d1f 100644 --- a/templates/nginx/sites-enabled/nextcloud.conf +++ b/templates/nginx/sites-enabled/nextcloud.conf @@ -17,25 +17,39 @@ server { listen 4430 ssl http2; server_name @@@DOMAIN@@@; + root /var/www/html; + ssl_certificate /certs/live/@@@DOMAIN@@@/fullchain.pem; ssl_certificate_key /certs/live/@@@DOMAIN@@@/privkey.pem; include /etc/nginx/params/ssl_params; # We include these headers directly because some are already set by Nextcloud. #include /etc/nginx/params/headers_params; - add_header X-Content-Type-Options nosniff; + add_header X-Content-Type-Options "nosniff" always; add_header X-XSS-Protection "1; mode=block"; add_header Strict-Transport-Security "max-age=15768000"; - add_header Referrer-Policy "same-origin"; - add_header X-Robots-Tag none; - add_header X-Download-Options noopen; - add_header X-Permitted-Cross-Domain-Policies none; + add_header Referrer-Policy "no-referrer" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Robots-Tag "none" always; + add_header X-Download-Options "noopen" always; + add_header X-Permitted-Cross-Domain-Policies "none" always; fastcgi_hide_header X-Powered-By; server_tokens off; + location = /.well-known/carddav { + return 301 $scheme://$host/remote.php/dav; + } + location = /.well-known/caldav { + return 301 $scheme://$host/remote.php/dav; + } + client_max_body_size 4096M; fastcgi_buffers 64 4K; + fastcgi_connect_timeout 60; + fastcgi_send_timeout 600; + fastcgi_read_timeout 600; + # Enable gzip but do not remove ETag headers gzip on; gzip_vary on; @@ -45,9 +59,8 @@ server { # Enable compression for JS/CSS/HTML bundle, for improved client load times. # It might be nice to compress JSON, but leaving that out to protect against potential # compression+encryption information leak attacks like BREACH. - gzip_types application/atom+xml application/javascript application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy; + gzip_types application/atom+xml application/javascript application/json application/ld+json application/manifest+json application/rss+xml application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt text/x-component text/x-cross-domain-policy; - root /var/www/html; location / { rewrite ^ /index.php; @@ -60,13 +73,14 @@ server { deny all; } - location ~ ^/(?:index|remote|public|cron|core/ajax/update|status|ocs/v[12]|updater/.+|ocs-provider/.+)\.php(?:$|/) { - fastcgi_split_path_info ^(.+\.php)(/.*)$; + location ~ ^/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+|.+\/richdocumentscode\/proxy)\.php(?:$|\/) { + fastcgi_split_path_info ^(.+?\.php)(\/.*|)$; + set $path_info $fastcgi_path_info; try_files $fastcgi_script_name =404; include fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; - fastcgi_param PATH_INFO $fastcgi_path_info; - # fastcgi_param HTTPS on; + fastcgi_param PATH_INFO $path_info; + fastcgi_param HTTPS on; #Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; fastcgi_param front_controller_active true; @@ -75,21 +89,42 @@ server { fastcgi_request_buffering off; } - location = /.well-known/carddav { - return 301 $scheme://$host/remote.php/dav; - } - location = /.well-known/caldav { - return 301 $scheme://$host/remote.php/dav; + location ~ ^\/(?:updater|oc[ms]-provider)(?:$|\/) { + try_files $uri/ =404; + index index.php; } - location ~ (\.(?:css|js|woff2?|svg|gif|png|jpg|ico)$|^/core/img/background.png$) { + + # Adding the cache control header for js, css and map files + # Make sure it is BELOW the PHP block + location ~ \.(?:css|js|woff2?|svg|gif|map)$ { try_files $uri /index.php$request_uri; add_header Cache-Control "public, max-age=15778463"; - add_header X-Content-Type-Options nosniff; - add_header X-XSS-Protection "1; mode=block"; - add_header X-Robots-Tag none; - add_header X-Download-Options noopen; - add_header X-Permitted-Cross-Domain-Policies none; - add_header Referrer-Policy no-referrer; + # Add headers to serve security related headers (It is intended to + # have those duplicated to the ones above) + # Before enabling Strict-Transport-Security headers please read into + # this topic first. + #add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload;" always; + # + # WARNING: Only add the preload option once you read about + # the consequences in https://hstspreload.org/. This option + # will add the domain to a hardcoded list that is shipped + # in all major browsers and getting removed from this list + # could take several months. + add_header Referrer-Policy "no-referrer" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Download-Options "noopen" always; + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Permitted-Cross-Domain-Policies "none" always; + add_header X-Robots-Tag "none" always; + add_header X-XSS-Protection "1; mode=block" always; + + # Optional: Don't log access to assets + access_log off; + } + + location ~ \.(?:png|html|ttf|ico|jpg|jpeg|bcmap|mp4|webm)$ { + try_files $uri /index.php$request_uri; + # Optional: Don't log access to other assets access_log off; } } -- GitLab From eca389b74a4e6f53da8e1dc8c8992e643cea2818 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 15:09:01 +0530 Subject: [PATCH 10/72] Added empty skeleton directory --- config-static/nextcloud/skeleton/.keep | 0 templates/nextcloud/config.php | 1 + 2 files changed, 1 insertion(+) create mode 100644 config-static/nextcloud/skeleton/.keep diff --git a/config-static/nextcloud/skeleton/.keep b/config-static/nextcloud/skeleton/.keep new file mode 100644 index 0000000..e69de29 diff --git a/templates/nextcloud/config.php b/templates/nextcloud/config.php index c3311a7..f70e02f 100644 --- a/templates/nextcloud/config.php +++ b/templates/nextcloud/config.php @@ -69,5 +69,6 @@ $CONFIG = array ( 'loglevel' => 2, 'preview_max_x' => 1024, 'preview_max_y' => 1024, + 'skeletondirectory' => '/var/www/skeleton/', ); ?> -- GitLab From 21db18c7a9c5cacf673b8b06c61b8ee16491bd0a Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 15:09:58 +0530 Subject: [PATCH 11/72] Added correct drop_account version --- scripts/postinstall.sh | 9 ++++++++- templates/docker-compose/docker-compose-base.yml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index aaf6a40..1faeb46 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -42,7 +42,14 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:ap docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install photos -git clone --single-branch https://framagit.org/framasoft/nextcloud/drop_account.git volumes/nextcloud/custom_apps/drop_account + +DROP_ACCOUNT_RELEASE_LINK='https://framagit.org/framasoft/nextcloud/drop_account/uploads/f4bba3da2e1db522eb41297b7fd2c001/drop_account.tar.gz' + +curl -fsSL -o drop_account.tar.gz \ +"$DROP_ACCOUNT_RELEASE_LINK" && \ +tar -xf drop_account.tar.gz -C /mnt/repo-base/volumes/nextcloud/custom_apps/ && \ +rm drop_account.tar.gz; + docker-compose exec -T --user www-data nextcloud php occ app:enable drop_account echo "Installing custom ecloud drop account plugin" diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 3136c04..62741a8 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -139,7 +139,6 @@ services: # - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - OVERWRITEPROTOCOL=https - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} - volumes: - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ @@ -148,6 +147,7 @@ services: - /mnt/repo-base/config-dynamic/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - /mnt/repo-base/config-dynamic/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ + - /mnt/repo-base/config-static/nextcloud/skeleton:/var/www/skeleton/ depends_on: - mariadb -- GitLab From d7e1e61ac1c629b183668d2a549aef4a88bdf866 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 16:09:59 +0530 Subject: [PATCH 12/72] Fixed postfixadmin port --- scripts/postinstall.sh | 1 - templates/nginx/sites-enabled/postfixadmin.conf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 1faeb46..2b6b110 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -41,7 +41,6 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:disab docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install photos DROP_ACCOUNT_RELEASE_LINK='https://framagit.org/framasoft/nextcloud/drop_account/uploads/f4bba3da2e1db522eb41297b7fd2c001/drop_account.tar.gz' diff --git a/templates/nginx/sites-enabled/postfixadmin.conf b/templates/nginx/sites-enabled/postfixadmin.conf index 714bef3..5e99cc6 100644 --- a/templates/nginx/sites-enabled/postfixadmin.conf +++ b/templates/nginx/sites-enabled/postfixadmin.conf @@ -26,7 +26,7 @@ server { #auth_basic_user_file /passwds/.htpasswd; location / { - proxy_pass http://postfixadmin:8888; + proxy_pass http://postfixadmin:80; include /etc/nginx/params/proxy_params; } } -- GitLab From f11907ea7c8dab7ce4b67c7d3968d6ac8cdb4362 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 16:16:35 +0530 Subject: [PATCH 13/72] Moved bootstrap scripts into repository --- scripts/bootstrap-commons.sh | 81 ++++++++++++++++++++++++++++++++++++ scripts/bootstrap.sh | 34 +++++++++++++++ scripts/init-repo.sh | 2 +- 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 scripts/bootstrap-commons.sh create mode 100644 scripts/bootstrap.sh diff --git a/scripts/bootstrap-commons.sh b/scripts/bootstrap-commons.sh new file mode 100644 index 0000000..43de4e8 --- /dev/null +++ b/scripts/bootstrap-commons.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +function getRandomString { + LENGTH=$1 + cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LENGTH | head -n 1 +} + +function replaceTokensWithRandomStrings { + INPUT="$1" + TOBEREPLACED=$(echo "$INPUT" | grep -o '@@@generate@@@:[0-9]\+@') + REPLACEMENT_LENGTH=$(echo "$TOBEREPLACED" | awk -F: '{ print $NF }' | sed 's/@$//g') + RANDOMPART=$(getRandomString $REPLACEMENT_LENGTH) + echo "$INPUT" | sed "s/$TOBEREPLACED/$RANDOMPART/g" +} + +function doReplacementIfNecessary { + VALUE="$1" + echo "$VALUE" | grep -q "@@@generate@@@" && replaceTokensWithRandomStrings "$VALUE" || echo "$VALUE" +} + +function getValidationPattern { + INPUT="$1" + echo "$INPUT" | sed 's/^.*||||//g' | sed 's/;;;;.*$//g' +} + +function getInputDirectionMessage { + INPUT="$1" + echo "$INPUT" | sed 's/^.*;;;;//g' +} + +function getQuestion { + INPUT="$1" + echo "$INPUT" | sed 's/||||.*$//g' +} + +function generateEnvFile { + QUESTFILE="$1" + ANSWERFILE="$2" + ENVFILE="$3" + while read KEY VALUE; do + PREVVALUE="$VALUE" + VALUE=$(doReplacementIfNecessary "$VALUE") + if [[ "$PREVVALUE" = "$VALUE" ]] + then + if [[ "$#" -ne 1 ]] + then + DEFVAL=$(echo "$PREVVALUE" | grep -q ";default$" && echo "$PREVVALUE" | sed 's/;default$//g' || echo 0) + if [[ "$DEFVAL" = "0" ]] + then + PATTERN=$(getValidationPattern "$VALUE") + INPUT_DIRECTION=$(getInputDirectionMessage "$VALUE") + QUESTION=$(getQuestion "$VALUE") + echo "$QUESTION" + if [[ "$PATTERN$INPUT_DIRECTION" = "$VALUE$VALUE" ]] + then + read INPUT < /dev/tty + else + FIRST=1 + while [[ $(echo "$INPUT" | grep -q -P "$PATTERN" && echo ok || echo nok) = "nok" ]]; do + if [[ ${FIRST} != "1" ]] + then + echo "$INPUT_DIRECTION" + else + FIRST=0 + fi + read INPUT < /dev/tty + :; done + fi + echo "$KEY=$INPUT" >> "$ENVFILE" + else + echo "$KEY=$DEFVAL" >> "$ENVFILE" + fi + else + VALUE=$(grep "^$KEY=" "$ANSWERFILE" | awk -F= '{ print $NF }') + echo "$KEY=$VALUE" >> "$ENVFILE" + fi + else + echo "$KEY=$VALUE" >> "$ENVFILE" + fi + :;done <<< "$(grep -v \# ${QUESTFILE} | sed '/^$/d'| sed 's/=/ /g')" +} diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100644 index 0000000..f45f73c --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +if [[ $# -lt 1 ]] +then + echo "Usage $0 [branch name] [environment]" + exit 1 +fi +REPO=$1 +BRANCH="$2" +ENVIRONMENT="$3" +if [[ "$BRANCH" != "" ]] +then + BRANCH="--branch $BRANCH" +fi +################################################################################ +apt-get update && apt install -y --asume-yes true git salt-minion +################################################################################ + + +# Clone repo +echo "Cloning repo .." +git -C /mnt clone ${REPO} ${BRANCH} repo-base + + +# Init salt-minion (masterless) +cp /mnt/repo-base/deployment/salt/init-config/masterless.conf /etc/salt/minion.d/ + +# Run repo init (might run a few minutes) +echo "System update and packages installation .." +salt-call state.apply docker-compose + + +# init repo +bash /mnt/repo-base/scripts/init-repo.sh $ENVIRONMENT diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index d5ca69d..9d5c49e 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -6,7 +6,7 @@ function validateDomains { (INPUT=$(echo "$INPUT"| sed 's@;@,@g' | sed 's@ @,@g'); IFS=','; for DOMAIN in $INPUT; do echo "$DOMAIN" | xargs; done) | while read line; do echo "$line"; done | sort -u | while read line; do echo $line | grep -P '(?=^.{4,253}$)(^(?:[a-zA-Z0-9](?:(?:[a-zA-Z0-9\-]){0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$)'; done | tr "\n" "," | sed 's@,$@@g' } -source <(curl -s https://gitlab.e.foundation/e/infra/bootstrap/raw/master/bootstrap-commons.sh) +source bootstrap-commons.sh cd "/mnt/repo-base/" ENVFILE="/mnt/repo-base/.env" -- GitLab From f2c6d55951c86faf8a295ca3a2a25534a228e623 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 16:22:53 +0530 Subject: [PATCH 14/72] Corrected path of bootstrap-commons --- scripts/init-repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 9d5c49e..e93c6f2 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -6,7 +6,7 @@ function validateDomains { (INPUT=$(echo "$INPUT"| sed 's@;@,@g' | sed 's@ @,@g'); IFS=','; for DOMAIN in $INPUT; do echo "$DOMAIN" | xargs; done) | while read line; do echo "$line"; done | sort -u | while read line; do echo $line | grep -P '(?=^.{4,253}$)(^(?:[a-zA-Z0-9](?:(?:[a-zA-Z0-9\-]){0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$)'; done | tr "\n" "," | sed 's@,$@@g' } -source bootstrap-commons.sh +source /mnt/repo-base/scripts/bootstrap-commons.sh cd "/mnt/repo-base/" ENVFILE="/mnt/repo-base/.env" -- GitLab From 5d0b85f8bf5f311626a84a24a08298cfd20e7420 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 17:03:22 +0530 Subject: [PATCH 15/72] Removed fastcgi https on --- templates/nginx/sites-enabled/nextcloud.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/nginx/sites-enabled/nextcloud.conf b/templates/nginx/sites-enabled/nextcloud.conf index e360d1f..9d01aab 100644 --- a/templates/nginx/sites-enabled/nextcloud.conf +++ b/templates/nginx/sites-enabled/nextcloud.conf @@ -80,7 +80,6 @@ server { include fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $path_info; - fastcgi_param HTTPS on; #Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; fastcgi_param front_controller_active true; -- GitLab From c832f0327f144e464f6a8d0b75ee8bdfa536f712 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 8 Oct 2021 18:46:34 +0530 Subject: [PATCH 16/72] Fixed custom image tag; Removed extra config and custom_apps folders; fixed drop_account installation --- scripts/init-repo.sh | 4 ++-- scripts/postinstall.sh | 11 +---------- templates/docker-compose/docker-compose-base.yml | 4 +--- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index e93c6f2..aa48a53 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -178,11 +178,11 @@ else fi # create nextcloud config -mkdir -p "/mnt/repo-base/volumes/nextcloud/config/" +mkdir -p "/mnt/repo-base/volumes/nextcloud/html/config/" cat /mnt/repo-base/templates/nextcloud/config.php | sed "s/@@@DOMAIN@@@/$DOMAIN/g" | \ sed "s/@@@DRIVE_SMTP_PASSWORD@@@/$DRIVE_SMTP_PASSWORD/g" | sed "s/@@@PFDB_DB@@@/$PFDB_DB/g" | \ sed "s/@@@PFDB_USR@@@/$PFDB_USR/g" | sed "s/@@@DBPASS@@@/$PFDB_DBPASS/g" > \ - "/mnt/repo-base/volumes/nextcloud/config/config.php" + "/mnt/repo-base/volumes/nextcloud/html/config/config.php" chown www-data:www-data "/mnt/repo-base/volumes/nextcloud/" -R docker-compose up -d diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 2b6b110..69bee43 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -46,7 +46,7 @@ DROP_ACCOUNT_RELEASE_LINK='https://framagit.org/framasoft/nextcloud/drop_account curl -fsSL -o drop_account.tar.gz \ "$DROP_ACCOUNT_RELEASE_LINK" && \ -tar -xf drop_account.tar.gz -C /mnt/repo-base/volumes/nextcloud/custom_apps/ && \ +tar -xf drop_account.tar.gz -C /mnt/repo-base/volumes/nextcloud/html/custom_apps/ && \ rm drop_account.tar.gz; docker-compose exec -T --user www-data nextcloud php occ app:enable drop_account @@ -57,15 +57,6 @@ docker-compose exec -T --user www-data nextcloud php occ config:system:set e_wel # Add VHOST_ACCOUNTS from .env file as a system config value, to be used by our ecloud_drop_account plugin docker-compose exec -T --user www-data nextcloud php occ config:system:set e_welcome_domain --value="welcome.$DOMAIN" - -echo "Installing Nextcloud theme" -wget "https://gitlab.e.foundation/api/v4/projects/315/repository/archive.tar.gz" -O "/tmp/nextcloud-theme.tar.gz" -tar -xzf "/tmp/nextcloud-theme.tar.gz" -C "volumes/nextcloud/html/themes/" --strip-components=1 -chown www-data:www-data "volumes/nextcloud/html/themes/" -R -rm "/tmp/nextcloud-theme.tar.gz" - -docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:system:set theme --value eelo - docker-compose exec -T --user www-data nextcloud php occ maintenance:mode --off echo "Restarting Nextcloud container" diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 62741a8..70132db 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -124,7 +124,7 @@ services: - mariadb nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:master + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 container_name: nextcloud restart: always networks: @@ -141,8 +141,6 @@ services: - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} volumes: - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ - - /mnt/repo-base/volumes/nextcloud/config:/var/www/html/config/ - /mnt/repo-base/volumes/nextcloud/data:/var/www/html/data/ - /mnt/repo-base/config-dynamic/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - /mnt/repo-base/config-dynamic/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini -- GitLab From cddfa3adf422c31044c86847e15657529ea84829 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:11:24 +0530 Subject: [PATCH 17/72] Added ':l' lowercase option to questionnaire --- deployment/questionnaire/questionnaire.dat | 2 +- scripts/bootstrap-commons.sh | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/deployment/questionnaire/questionnaire.dat b/deployment/questionnaire/questionnaire.dat index acbca2c..0190e01 100644 --- a/deployment/questionnaire/questionnaire.dat +++ b/deployment/questionnaire/questionnaire.dat @@ -5,7 +5,7 @@ INSTALL_ONLYOFFICE=Do you want to install OnlyOffice? [y/n]||||^[yY|nN]$;;;;Plea # Generate and display RSPAMD_PASSWORD=@@@generate@@@:20@ -NEXTCLOUD_ADMIN_USER=ncadmin_@@@generate@@@:4@ +NEXTCLOUD_ADMIN_USER=ncadmin_@@@generate@@@:l:4@ NEXTCLOUD_ADMIN_PASSWORD=@@@generate@@@:20@ diff --git a/scripts/bootstrap-commons.sh b/scripts/bootstrap-commons.sh index 43de4e8..69950ac 100644 --- a/scripts/bootstrap-commons.sh +++ b/scripts/bootstrap-commons.sh @@ -2,14 +2,25 @@ function getRandomString { LENGTH=$1 - cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $LENGTH | head -n 1 + CHARACTERS=$2 + cat /dev/urandom | tr -dc $CHARACTERS | fold -w $LENGTH | head -n 1 } function replaceTokensWithRandomStrings { INPUT="$1" - TOBEREPLACED=$(echo "$INPUT" | grep -o '@@@generate@@@:[0-9]\+@') + TOBEREPLACED=$(echo "$INPUT" | grep -o -e '@@@generate@@@:[u|l]:[0-9]\+@' -e '@@@generate@@@:[0-9]\+@') + LOWERCASED=$(echo "$TOBEREPLACED" | grep -o ':l') + UPPERCASED=$(echo "$TOBEREPLACED" | grep -o ':u') REPLACEMENT_LENGTH=$(echo "$TOBEREPLACED" | awk -F: '{ print $NF }' | sed 's/@$//g') - RANDOMPART=$(getRandomString $REPLACEMENT_LENGTH) + if [ ! -z "$LOWERCASED" ] + then + RANDOMPART=$(getRandomString $REPLACEMENT_LENGTH 'a-z0-9') + elif [ ! -z "$UPPERCASED" ] + then + RANDOMPART=$(getRandomString $REPLACEMENT_LENGTH 'A-Z0-9') + else + RANDOMPART=$(getRandomString $REPLACEMENT_LENGTH 'a-zA-Z0-9') + fi echo "$INPUT" | sed "s/$TOBEREPLACED/$RANDOMPART/g" } -- GitLab From d796c4056ec429b19c09f6dc8988caab635cb3b5 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:22:44 +0530 Subject: [PATCH 18/72] Removed config-static and config-dynamic --- {config-dynamic => config}/automx/.keep | 0 .../autorenew => config/letsencrypt}/.keep | 0 .../nextcloud => config/mail}/.keep | 0 .../mail/clamav/freshclam.conf | 0 .../mail/dovecot/10-mail.conf | 0 .../mail/dovecot/90-quota.conf | 0 .../mail/dovecot/90-sieve.conf | 0 .../mail/rspamd/multimap.conf | 0 .../mail/rspamd/ratelimit.conf | 0 .../mail/rspamd/whitelist.sender.domain.map | 0 .../nextcloud/skeleton}/.keep | 0 .../nextcloud/skeleton => config/nginx}/.keep | 0 .../nginx/params/headers_params | 0 .../nginx/params/proxy_params | 0 .../nginx/params/ssl_params | 0 scripts/init-repo.sh | 32 +++++++++---------- scripts/show-info.sh | 6 ++-- scripts/ssl-renew.sh | 6 ++-- .../docker-compose/docker-compose-base.yml | 26 +++++++-------- .../docker-compose-networks.yml | 10 +++--- templates/nextcloud/config.php | 2 +- 21 files changed, 41 insertions(+), 41 deletions(-) rename {config-dynamic => config}/automx/.keep (100%) rename {config-dynamic/letsencrypt/autorenew => config/letsencrypt}/.keep (100%) rename {config-dynamic/nextcloud => config/mail}/.keep (100%) rename {config-static => config}/mail/clamav/freshclam.conf (100%) rename {config-static => config}/mail/dovecot/10-mail.conf (100%) rename {config-static => config}/mail/dovecot/90-quota.conf (100%) rename {config-static => config}/mail/dovecot/90-sieve.conf (100%) rename {config-static => config}/mail/rspamd/multimap.conf (100%) rename {config-static => config}/mail/rspamd/ratelimit.conf (100%) rename {config-static => config}/mail/rspamd/whitelist.sender.domain.map (100%) rename {config-dynamic/nginx/sites-enabled => config/nextcloud/skeleton}/.keep (100%) rename {config-static/nextcloud/skeleton => config/nginx}/.keep (100%) rename {config-static => config}/nginx/params/headers_params (100%) rename {config-static => config}/nginx/params/proxy_params (100%) rename {config-static => config}/nginx/params/ssl_params (100%) diff --git a/config-dynamic/automx/.keep b/config/automx/.keep similarity index 100% rename from config-dynamic/automx/.keep rename to config/automx/.keep diff --git a/config-dynamic/letsencrypt/autorenew/.keep b/config/letsencrypt/.keep similarity index 100% rename from config-dynamic/letsencrypt/autorenew/.keep rename to config/letsencrypt/.keep diff --git a/config-dynamic/nextcloud/.keep b/config/mail/.keep similarity index 100% rename from config-dynamic/nextcloud/.keep rename to config/mail/.keep diff --git a/config-static/mail/clamav/freshclam.conf b/config/mail/clamav/freshclam.conf similarity index 100% rename from config-static/mail/clamav/freshclam.conf rename to config/mail/clamav/freshclam.conf diff --git a/config-static/mail/dovecot/10-mail.conf b/config/mail/dovecot/10-mail.conf similarity index 100% rename from config-static/mail/dovecot/10-mail.conf rename to config/mail/dovecot/10-mail.conf diff --git a/config-static/mail/dovecot/90-quota.conf b/config/mail/dovecot/90-quota.conf similarity index 100% rename from config-static/mail/dovecot/90-quota.conf rename to config/mail/dovecot/90-quota.conf diff --git a/config-static/mail/dovecot/90-sieve.conf b/config/mail/dovecot/90-sieve.conf similarity index 100% rename from config-static/mail/dovecot/90-sieve.conf rename to config/mail/dovecot/90-sieve.conf diff --git a/config-static/mail/rspamd/multimap.conf b/config/mail/rspamd/multimap.conf similarity index 100% rename from config-static/mail/rspamd/multimap.conf rename to config/mail/rspamd/multimap.conf diff --git a/config-static/mail/rspamd/ratelimit.conf b/config/mail/rspamd/ratelimit.conf similarity index 100% rename from config-static/mail/rspamd/ratelimit.conf rename to config/mail/rspamd/ratelimit.conf diff --git a/config-static/mail/rspamd/whitelist.sender.domain.map b/config/mail/rspamd/whitelist.sender.domain.map similarity index 100% rename from config-static/mail/rspamd/whitelist.sender.domain.map rename to config/mail/rspamd/whitelist.sender.domain.map diff --git a/config-dynamic/nginx/sites-enabled/.keep b/config/nextcloud/skeleton/.keep similarity index 100% rename from config-dynamic/nginx/sites-enabled/.keep rename to config/nextcloud/skeleton/.keep diff --git a/config-static/nextcloud/skeleton/.keep b/config/nginx/.keep similarity index 100% rename from config-static/nextcloud/skeleton/.keep rename to config/nginx/.keep diff --git a/config-static/nginx/params/headers_params b/config/nginx/params/headers_params similarity index 100% rename from config-static/nginx/params/headers_params rename to config/nginx/params/headers_params diff --git a/config-static/nginx/params/proxy_params b/config/nginx/params/proxy_params similarity index 100% rename from config-static/nginx/params/proxy_params rename to config/nginx/params/proxy_params diff --git a/config-static/nginx/params/ssl_params b/config/nginx/params/ssl_params similarity index 100% rename from config-static/nginx/params/ssl_params rename to config/nginx/params/ssl_params diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index aa48a53..b6c6d3d 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -60,9 +60,9 @@ DC_DIR="templates/docker-compose/" case $INSTALL_ONLYOFFICE in [Yy]* ) cat "${DC_DIR}docker-compose-base.yml" "${DC_DIR}docker-compose-onlyoffice.yml" "${DC_DIR}docker-compose-networks.yml" > docker-compose.yml; - cat "templates/nginx/sites-enabled/onlyoffice.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/nginx/sites-enabled/onlyoffice.conf" + cat "templates/nginx/sites-enabled/onlyoffice.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/onlyoffice.conf" OFFICE_DOMAIN=",office.$DOMAIN" - OFFICE_LETSENCRYPT_KEY="config-dynamic/letsencrypt/certstore/live/office.$DOMAIN/privkey.pem" + OFFICE_LETSENCRYPT_KEY="config/letsencrypt/certstore/live/office.$DOMAIN/privkey.pem" NUM_CERTIFICATES="4" ;; [Nn]* ) @@ -83,32 +83,32 @@ echo "VIRTUAL_HOST=$VIRTUAL_HOST" >> "$ENVFILE" # finished .env file generation # fill autorenew config -rm -f "/mnt/repo-base/config-dynamic/letsencrypt/autorenew/ssl-domains.dat" +rm -f "/mnt/repo-base/config/letsencrypt/autorenew/ssl-domains.dat" echo "$DOMAIN,$VIRTUAL_HOST,mail.$DOMAIN,spam.$DOMAIN,welcome.$DOMAIN$OFFICE_DOMAIN" | tr "," "\n" | while read CURDOMAIN; do - echo "$CURDOMAIN" >> config-dynamic/letsencrypt/autorenew/ssl-domains.dat + echo "$CURDOMAIN" >> config/letsencrypt/autorenew/ssl-domains.dat :; done # Configure automx -cat templates/automx/automx.conf | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/automx/automx.conf" -chown www-data:www-data "config-dynamic/automx/automx.conf" +cat templates/automx/automx.conf | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/automx/automx.conf" +chown www-data:www-data "config/automx/automx.conf" # Configure nginx vhost # automx echo "$DOMAIN,$ADD_DOMAINS" | tr "," "\n" | while read CURDOMAIN; do - cat "templates/nginx/sites-enabled/autoconfig.conf" | sed "s/@@@DOMAIN@@@/$CURDOMAIN/g" | sed "s/@@@SERVICE@@@/autoconfig/g" > "config-dynamic/nginx/sites-enabled/autoconfig.$CURDOMAIN.conf" - cat "templates/nginx/sites-enabled/autoconfig.conf" | sed "s/@@@DOMAIN@@@/$CURDOMAIN/g" | sed "s/@@@SERVICE@@@/autodiscover/g" > "config-dynamic/nginx/sites-enabled/autodiscover.$CURDOMAIN.conf" + cat "templates/nginx/sites-enabled/autoconfig.conf" | sed "s/@@@DOMAIN@@@/$CURDOMAIN/g" | sed "s/@@@SERVICE@@@/autoconfig/g" > "config/nginx/sites-enabled/autoconfig.$CURDOMAIN.conf" + cat "templates/nginx/sites-enabled/autoconfig.conf" | sed "s/@@@DOMAIN@@@/$CURDOMAIN/g" | sed "s/@@@SERVICE@@@/autodiscover/g" > "config/nginx/sites-enabled/autodiscover.$CURDOMAIN.conf" :; done # other hosts -cat "templates/nginx/sites-enabled/nextcloud.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/nginx/sites-enabled/nextcloud.conf" -cat "templates/nginx/sites-enabled/postfixadmin.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/nginx/sites-enabled/postfixadmin.conf" -cat "templates/nginx/sites-enabled/rspamd.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/nginx/sites-enabled/rspamd.conf" -cat "templates/nginx/sites-enabled/welcome.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config-dynamic/nginx/sites-enabled/welcome.conf" +cat "templates/nginx/sites-enabled/nextcloud.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/nextcloud.conf" +cat "templates/nginx/sites-enabled/postfixadmin.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/postfixadmin.conf" +cat "templates/nginx/sites-enabled/rspamd.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/rspamd.conf" +cat "templates/nginx/sites-enabled/welcome.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/welcome.conf" -cp "templates/nextcloud/x-fpm-overloads.conf" "config-dynamic/nextcloud/x-fpm-overloads.conf" -cp "templates/nextcloud/x-php-overloads.ini" "config-dynamic/nextcloud/x-php-overloads.ini" +cp "templates/nextcloud/x-fpm-overloads.conf" "config/nextcloud/x-fpm-overloads.conf" +cp "templates/nextcloud/x-php-overloads.ini" "config/nextcloud/x-php-overloads.ini" # confirm DNS is ready echo "" @@ -164,8 +164,8 @@ fi bash scripts/ssl-renew.sh # verify LE status -CTR_LE=$(find config-dynamic/letsencrypt/certstore/live/mail.$DOMAIN/privkey.pem config-dynamic/letsencrypt/certstore/live/spam.$DOMAIN/privkey.pem config-dynamic/letsencrypt/certstore/live/welcome.$DOMAIN/privkey.pem $OFFICE_LETSENCRYPT_KEY 2>/dev/null| wc -l) -CTR_AC_LE=$(echo "$VIRTUAL_HOST" | tr "," "\n" | while read CURDOMAIN; do find config-dynamic/letsencrypt/certstore/live/$CURDOMAIN/privkey.pem 2>/dev/null | grep $CURDOMAIN && echo found || echo missing; done | grep missing | wc -l) +CTR_LE=$(find config/letsencrypt/certstore/live/mail.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/spam.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/welcome.$DOMAIN/privkey.pem $OFFICE_LETSENCRYPT_KEY 2>/dev/null| wc -l) +CTR_AC_LE=$(echo "$VIRTUAL_HOST" | tr "," "\n" | while read CURDOMAIN; do find config/letsencrypt/certstore/live/$CURDOMAIN/privkey.pem 2>/dev/null | grep $CURDOMAIN && echo found || echo missing; done | grep missing | wc -l) if [ "$CTR_LE$CTR_AC_LE" = "${NUM_CERTIFICATES}0" ] then diff --git a/scripts/show-info.sh b/scripts/show-info.sh index e25eb5f..59e9ac4 100755 --- a/scripts/show-info.sh +++ b/scripts/show-info.sh @@ -3,14 +3,14 @@ set -e source /mnt/repo-base/scripts/base.sh -SPAM_UI=$(grep server_name $(grep -l mailserver:11334 /mnt/repo-base/config-dynamic/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') +SPAM_UI=$(grep server_name $(grep -l mailserver:11334 /mnt/repo-base/config/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') RSPAMD_PASSWORD=$(grep ^RSPAMD_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') -NEXTCLOUD_UI=$(grep server_name $(grep -l nextcloud:9000 /mnt/repo-base/config-dynamic/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') +NEXTCLOUD_UI=$(grep server_name $(grep -l nextcloud:9000 /mnt/repo-base/config/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') NEXTCLOUD_ADMIN_USER=$(grep ^NEXTCLOUD_ADMIN_USER= "$ENVFILE" | awk -F= '{ print $NF }') NEXTCLOUD_ADMIN_PASSWORD=$(grep ^NEXTCLOUD_ADMIN_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') -POSTFIX_UI=$(grep server_name $(grep -l postfixadmin:8888 /mnt/repo-base/config-dynamic/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') +POSTFIX_UI=$(grep server_name $(grep -l postfixadmin:8888 /mnt/repo-base/config/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') POSTFIX_USER=$(grep ALT_EMAIL= "$ENVFILE" | awk -F= '{ print $NF }') POSTFIX_PASSWORD=$(grep PFA_SUPERADMIN_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') diff --git a/scripts/ssl-renew.sh b/scripts/ssl-renew.sh index 61c3a6b..2fd4cab 100755 --- a/scripts/ssl-renew.sh +++ b/scripts/ssl-renew.sh @@ -3,8 +3,8 @@ source /mnt/repo-base/scripts/base.sh -CONFIG=/mnt/repo-base/config-dynamic/letsencrypt/autorenew/ssl-domains.dat -CONFIG_DIR=/mnt/repo-base/config-dynamic/letsencrypt/certstore +CONFIG=/mnt/repo-base/config/letsencrypt/autorenew/ssl-domains.dat +CONFIG_DIR=/mnt/repo-base/config/letsencrypt/certstore LIVE_DIR=$CONFIG_DIR/live cat "$CONFIG" | while read DOMAIN; do @@ -16,7 +16,7 @@ cat "$CONFIG" | while read DOMAIN; do else CERT_UPDATED_FILE="$LIVE_DIR/$DOMAIN/cert-updated" certbot certonly -d "$DOMAIN" --non-interactive -m "$ALT_EMAIL" --agree-tos \ - --webroot --webroot-path='/mnt/repo-base/config-dynamic/letsencrypt/acme-challenge/' \ + --webroot --webroot-path='/mnt/repo-base/config/letsencrypt/acme-challenge/' \ --config-dir="$CONFIG_DIR" \ --deploy-hook "touch $CERT_UPDATED_FILE" # add the following parameters to test renewal (will install invalid certificates) diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 70132db..6945872 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -28,14 +28,14 @@ services: # Full list of options: https://github.com/hardware/mailserver#environment-variables volumes: - /mnt/repo-base/volumes/mail:/var/mail - - /mnt/repo-base/config-dynamic/letsencrypt/certstore:/etc/letsencrypt - - /mnt/repo-base/config-static/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf - - /mnt/repo-base/config-static/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf - - /mnt/repo-base/config-static/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf - - /mnt/repo-base/config-static/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf - - /mnt/repo-base/config-static/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map - - /mnt/repo-base/config-static/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf - - /mnt/repo-base/config-static/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf + - /mnt/repo-base/config/letsencrypt/certstore:/etc/letsencrypt + - /mnt/repo-base/config/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf + - /mnt/repo-base/config/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf + - /mnt/repo-base/config/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf + - /mnt/repo-base/config/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf + - /mnt/repo-base/config/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map + - /mnt/repo-base/config/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf + - /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf depends_on: - mariadb - redis @@ -80,7 +80,7 @@ services: - MYSQL_PASSWORD=${DBPASS} volumes: - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql - - /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d + - /mnt/repo-base/config/nextcloud/database:/docker-entrypoint-initdb.d redis: image: redis:6.0-alpine @@ -142,10 +142,10 @@ services: volumes: - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - /mnt/repo-base/volumes/nextcloud/data:/var/www/html/data/ - - /mnt/repo-base/config-dynamic/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - - /mnt/repo-base/config-dynamic/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini + - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf + - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ - - /mnt/repo-base/config-static/nextcloud/skeleton:/var/www/skeleton/ + - /mnt/repo-base/config/nextcloud/skeleton:/var/www/skeleton/ depends_on: - mariadb @@ -161,5 +161,5 @@ services: networks: - serverbase volumes: - - /mnt/repo-base/config-dynamic/automx/automx.conf:/etc/automx.conf + - /mnt/repo-base/config/automx/automx.conf:/etc/automx.conf diff --git a/templates/docker-compose/docker-compose-networks.yml b/templates/docker-compose/docker-compose-networks.yml index 099ccc1..597375e 100644 --- a/templates/docker-compose/docker-compose-networks.yml +++ b/templates/docker-compose/docker-compose-networks.yml @@ -9,11 +9,11 @@ - "80:8000" - "443:4430" volumes: - - /mnt/repo-base/config-dynamic/nginx/sites-enabled:/etc/nginx/conf.d/ - - /mnt/repo-base/config-static/nginx/params:/etc/nginx/params/ - - /mnt/repo-base/config-dynamic/letsencrypt/certstore:/certs - - /mnt/repo-base/config-dynamic/nginx/passwds:/passwds - - /mnt/repo-base/config-dynamic/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge + - /mnt/repo-base/config/nginx/sites-enabled:/etc/nginx/conf.d/ + - /mnt/repo-base/config/nginx/params:/etc/nginx/params/ + - /mnt/repo-base/config/letsencrypt/certstore:/certs + - /mnt/repo-base/config/nginx/passwds:/passwds + - /mnt/repo-base/config/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge - /mnt/repo-base/volumes/nextcloud/html:/var/www/html - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ depends_on: diff --git a/templates/nextcloud/config.php b/templates/nextcloud/config.php index f70e02f..4bad40e 100644 --- a/templates/nextcloud/config.php +++ b/templates/nextcloud/config.php @@ -65,7 +65,7 @@ $CONFIG = array ( ), 'hash_algorithm_for_new_passwords' => 'sha512', ), - 'theme' => 'eelo', + 'theme' => 'eCloud', 'loglevel' => 2, 'preview_max_x' => 1024, 'preview_max_y' => 1024, -- GitLab From 3a2ee6f80f6a5342f0b228b485ed941beff849bb Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:24:18 +0530 Subject: [PATCH 19/72] Removed git clone step from bootstrap.sh --- scripts/bootstrap.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index f45f73c..3389eb3 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -17,11 +17,6 @@ apt-get update && apt install -y --asume-yes true git salt-minion ################################################################################ -# Clone repo -echo "Cloning repo .." -git -C /mnt clone ${REPO} ${BRANCH} repo-base - - # Init salt-minion (masterless) cp /mnt/repo-base/deployment/salt/init-config/masterless.conf /etc/salt/minion.d/ -- GitLab From de34bfd00bbf393cac31ff00ad0acba9e6a4c42f Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:28:29 +0530 Subject: [PATCH 20/72] Removed arguments for bootstrap.sh --- scripts/bootstrap.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 3389eb3..e6a4ac1 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,17 +1,5 @@ #!/bin/bash -if [[ $# -lt 1 ]] -then - echo "Usage $0 [branch name] [environment]" - exit 1 -fi -REPO=$1 -BRANCH="$2" -ENVIRONMENT="$3" -if [[ "$BRANCH" != "" ]] -then - BRANCH="--branch $BRANCH" -fi ################################################################################ apt-get update && apt install -y --asume-yes true git salt-minion ################################################################################ -- GitLab From 513026558fe3a668c421ab281e904b44c5b525dc Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:30:32 +0530 Subject: [PATCH 21/72] Added folders in autorenew and nginx config --- config/letsencrypt/{ => autorenew}/.keep | 0 config/nginx/{ => sites-enabled}/.keep | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename config/letsencrypt/{ => autorenew}/.keep (100%) rename config/nginx/{ => sites-enabled}/.keep (100%) diff --git a/config/letsencrypt/.keep b/config/letsencrypt/autorenew/.keep similarity index 100% rename from config/letsencrypt/.keep rename to config/letsencrypt/autorenew/.keep diff --git a/config/nginx/.keep b/config/nginx/sites-enabled/.keep similarity index 100% rename from config/nginx/.keep rename to config/nginx/sites-enabled/.keep -- GitLab From 588bee77665662837d3d12f0dccab872a02bd99d Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 12:59:51 +0530 Subject: [PATCH 22/72] Removed serverbase network, replaced with default --- templates/docker-compose/docker-compose-base.yml | 14 +++++++------- .../docker-compose/docker-compose-networks.yml | 6 +----- .../docker-compose/docker-compose-onlyoffice.yml | 6 +++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 6945872..2a06951 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -8,7 +8,7 @@ services: hostname: mail restart: always networks: - - serverbase + - default ports: - "25:25" # SMTP - Required - "110:110" # POP3 STARTTLS - Optional - For webmails/desktop clients @@ -45,7 +45,7 @@ services: container_name: postfixadmin restart: always networks: - - serverbase + - default environment: - DBPASS=${DBPASS} - DBHOST=mariadb @@ -71,7 +71,7 @@ services: container_name: mariadb restart: always networks: - - serverbase + - default environment: # Note: These variables are only used for the first start. Later changes are ignored. - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} @@ -87,7 +87,7 @@ services: container_name: redis restart: always networks: - - serverbase + - default command: redis-server --appendonly yes volumes: - /mnt/repo-base/volumes/redis/db:/data @@ -117,7 +117,7 @@ services: - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} restart: always networks: - - serverbase + - default volumes: - /mnt/repo-base/volumes/accounts:/var/accounts depends_on: @@ -128,7 +128,7 @@ services: container_name: nextcloud restart: always networks: - - serverbase + - default environment: - MYSQL_DATABASE=${MYSQL_DATABASE_NC} - MYSQL_USER=${MYSQL_USER_NC} @@ -159,7 +159,7 @@ services: - HOSTNAME=automx restart: always networks: - - serverbase + - default volumes: - /mnt/repo-base/config/automx/automx.conf:/etc/automx.conf diff --git a/templates/docker-compose/docker-compose-networks.yml b/templates/docker-compose/docker-compose-networks.yml index 597375e..901503f 100644 --- a/templates/docker-compose/docker-compose-networks.yml +++ b/templates/docker-compose/docker-compose-networks.yml @@ -4,7 +4,7 @@ container_name: nginx restart: unless-stopped networks: - - serverbase + - default ports: - "80:8000" - "443:4430" @@ -15,7 +15,6 @@ - /mnt/repo-base/config/nginx/passwds:/passwds - /mnt/repo-base/config/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge - /mnt/repo-base/volumes/nextcloud/html:/var/www/html - - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ depends_on: - nextcloud - automx @@ -24,6 +23,3 @@ - mailserver #- onlyoffice-community-server -networks: - serverbase: - driver: 'bridge' diff --git a/templates/docker-compose/docker-compose-onlyoffice.yml b/templates/docker-compose/docker-compose-onlyoffice.yml index 81b83fc..7ca917f 100644 --- a/templates/docker-compose/docker-compose-onlyoffice.yml +++ b/templates/docker-compose/docker-compose-onlyoffice.yml @@ -5,7 +5,7 @@ stdin_open: true restart: always networks: - - serverbase + - default volumes: - /mnt/repo-base/volumes/onlyoffice/DocumentServer/data:/var/www/onlyoffice/Data - /mnt/repo-base/volumes/onlyoffice/DocumentServer/logs:/var/log/onlyoffice @@ -16,7 +16,7 @@ hostname: onlyoffice.${DOMAIN} restart: always networks: - - serverbase + - default volumes: - /mnt/repo-base/volumes/onlyoffice/MailServer/data:/var/vmail - /mnt/repo-base/volumes/onlyoffice/MailServer/data/certs:/etc/pki/tls/mailserver @@ -27,7 +27,7 @@ container_name: onlyoffice-community-server restart: always networks: - - serverbase + - default ports: - 5222:5222 environment: -- GitLab From 17db6c714ae0ed80ce98541c1a459171544e74a9 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 13:05:16 +0530 Subject: [PATCH 23/72] Added line to remove .keep file from skeleton dir --- scripts/init-repo.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index b6c6d3d..0f57ad5 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -184,6 +184,7 @@ cat /mnt/repo-base/templates/nextcloud/config.php | sed "s/@@@DOMAIN@@@/$DOMAIN/ sed "s/@@@PFDB_USR@@@/$PFDB_USR/g" | sed "s/@@@DBPASS@@@/$PFDB_DBPASS/g" > \ "/mnt/repo-base/volumes/nextcloud/html/config/config.php" chown www-data:www-data "/mnt/repo-base/volumes/nextcloud/" -R +rm -f /mnt/repo-base/config/nextcloud/skeleton/.keep docker-compose up -d -- GitLab From 2ea87bdc7c08d99f2dba2bdd4f42711f326169be Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 11 Oct 2021 18:27:52 +0530 Subject: [PATCH 24/72] Fixed show-info.sh with correct url; Added jq package installation --- scripts/bootstrap.sh | 2 +- scripts/show-info.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index e6a4ac1..40595fb 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash ################################################################################ -apt-get update && apt install -y --asume-yes true git salt-minion +apt-get update && apt install -y --asume-yes true git salt-minion && apt-get install -y jq ################################################################################ diff --git a/scripts/show-info.sh b/scripts/show-info.sh index 59e9ac4..8a64243 100755 --- a/scripts/show-info.sh +++ b/scripts/show-info.sh @@ -10,7 +10,7 @@ NEXTCLOUD_UI=$(grep server_name $(grep -l nextcloud:9000 /mnt/repo-base/config/n NEXTCLOUD_ADMIN_USER=$(grep ^NEXTCLOUD_ADMIN_USER= "$ENVFILE" | awk -F= '{ print $NF }') NEXTCLOUD_ADMIN_PASSWORD=$(grep ^NEXTCLOUD_ADMIN_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') -POSTFIX_UI=$(grep server_name $(grep -l postfixadmin:8888 /mnt/repo-base/config/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') +POSTFIX_UI=$(grep server_name $(grep -l postfixadmin /mnt/repo-base/config/nginx/sites-enabled/*.conf) | sort -u | head -n1 | awk '{ print $2 }' | sed 's/;$//g') POSTFIX_USER=$(grep ALT_EMAIL= "$ENVFILE" | awk -F= '{ print $NF }') POSTFIX_PASSWORD=$(grep PFA_SUPERADMIN_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') -- GitLab From 21065abfa12fde2898561d9c0d9f6171778c5ea4 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 22:41:11 +0530 Subject: [PATCH 25/72] Removed skeleton dir --- config/nextcloud/{skeleton => }/.keep | 0 scripts/bootstrap.sh | 2 +- templates/docker-compose/docker-compose-base.yml | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) rename config/nextcloud/{skeleton => }/.keep (100%) diff --git a/config/nextcloud/skeleton/.keep b/config/nextcloud/.keep similarity index 100% rename from config/nextcloud/skeleton/.keep rename to config/nextcloud/.keep diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 40595fb..e6a4ac1 100644 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash ################################################################################ -apt-get update && apt install -y --asume-yes true git salt-minion && apt-get install -y jq +apt-get update && apt install -y --asume-yes true git salt-minion ################################################################################ diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index 2a06951..f1465ed 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -145,7 +145,6 @@ services: - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ - - /mnt/repo-base/config/nextcloud/skeleton:/var/www/skeleton/ depends_on: - mariadb -- GitLab From c68b78809469bc0a23412262a81e1a3bd6f82ced Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 22:56:31 +0530 Subject: [PATCH 26/72] Removed OnlyOffice and added mariadb conf --- config/mariadb/ecloud.cnf | 7 +++ deployment/questionnaire/questionnaire.dat | 1 - scripts/base.sh | 1 - scripts/delete-account.sh | 1 - scripts/init-repo.sh | 13 ------ .../docker-compose/docker-compose-base.yml | 3 +- .../docker-compose-onlyoffice.yml | 43 ------------------- 7 files changed, 8 insertions(+), 61 deletions(-) create mode 100644 config/mariadb/ecloud.cnf delete mode 100644 templates/docker-compose/docker-compose-onlyoffice.yml diff --git a/config/mariadb/ecloud.cnf b/config/mariadb/ecloud.cnf new file mode 100644 index 0000000..9e16248 --- /dev/null +++ b/config/mariadb/ecloud.cnf @@ -0,0 +1,7 @@ +[mysqld] +transaction_isolation = READ-COMMITTED +binlog_format = ROW + + +innodb_buffer_pool_size=512M +innodb_io_capacity=4000 \ No newline at end of file diff --git a/deployment/questionnaire/questionnaire.dat b/deployment/questionnaire/questionnaire.dat index 0190e01..b448f58 100644 --- a/deployment/questionnaire/questionnaire.dat +++ b/deployment/questionnaire/questionnaire.dat @@ -1,7 +1,6 @@ DOMAIN=Enter your mailserver (management) domain (e.g. domainA.com): ADD_DOMAINS=Optionally enter additional domain(s) (comma separated, no white spaces) to handle mail for (e.g. domainB.com,domainC.com) or just press enter if you need none: ALT_EMAIL=Enter alternative email: -INSTALL_ONLYOFFICE=Do you want to install OnlyOffice? [y/n]||||^[yY|nN]$;;;;Please enter 'y' or 'n' # Generate and display RSPAMD_PASSWORD=@@@generate@@@:20@ diff --git a/scripts/base.sh b/scripts/base.sh index be318b2..3675432 100755 --- a/scripts/base.sh +++ b/scripts/base.sh @@ -21,7 +21,6 @@ MYSQL_USER_NC=$(grep ^MYSQL_USER_NC= "$ENVFILE" | awk -F= '{ print $NF }') MYSQL_PASSWORD_NC=$(grep ^MYSQL_PASSWORD_NC= "$ENVFILE" | awk -F= '{ print $NF }') MYSQL_ROOT_PASSWORD=$(grep ^MYSQL_ROOT_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') -INSTALL_ONLYOFFICE=$(grep ^INSTALL_ONLYOFFICE= "$ENVFILE" | awk -F= '{ print $NF }') DRIVE_SMTP_PASSWORD=$(grep ^DRIVE_SMTP_PASSWORD= "$ENVFILE" | awk -F= '{ print $NF }') diff --git a/scripts/delete-account.sh b/scripts/delete-account.sh index c4d42af..4787587 100755 --- a/scripts/delete-account.sh +++ b/scripts/delete-account.sh @@ -19,5 +19,4 @@ if [[ $response =~ ^([yY][eE][sS]|[yY])$ ]]; then echo "Deleting email account" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox delete "$ACCOUNT" - # TODO: delete onlyoffice account??? fi diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 0f57ad5..63f6511 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -57,19 +57,6 @@ echo "WEBSITE_SECRET=not_defined" >> "$ENVFILE" source /mnt/repo-base/scripts/base.sh DC_DIR="templates/docker-compose/" -case $INSTALL_ONLYOFFICE in - [Yy]* ) - cat "${DC_DIR}docker-compose-base.yml" "${DC_DIR}docker-compose-onlyoffice.yml" "${DC_DIR}docker-compose-networks.yml" > docker-compose.yml; - cat "templates/nginx/sites-enabled/onlyoffice.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/onlyoffice.conf" - OFFICE_DOMAIN=",office.$DOMAIN" - OFFICE_LETSENCRYPT_KEY="config/letsencrypt/certstore/live/office.$DOMAIN/privkey.pem" - NUM_CERTIFICATES="4" - ;; - [Nn]* ) - cat "${DC_DIR}docker-compose-base.yml" "${DC_DIR}docker-compose-networks.yml" > docker-compose.yml - NUM_CERTIFICATES="3" - ;; -esac # To be constructed repo specific echo "VHOSTS_ACCOUNTS=welcome.$DOMAIN" >> "$ENVFILE" diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index f1465ed..ddd921d 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -80,8 +80,7 @@ services: - MYSQL_PASSWORD=${DBPASS} volumes: - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql - - /mnt/repo-base/config/nextcloud/database:/docker-entrypoint-initdb.d - + - /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro redis: image: redis:6.0-alpine container_name: redis diff --git a/templates/docker-compose/docker-compose-onlyoffice.yml b/templates/docker-compose/docker-compose-onlyoffice.yml deleted file mode 100644 index 7ca917f..0000000 --- a/templates/docker-compose/docker-compose-onlyoffice.yml +++ /dev/null @@ -1,43 +0,0 @@ - - onlyoffice-documentserver: - image: onlyoffice/documentserver:5.2.6.3 - container_name: onlyoffice-documentserver - stdin_open: true - restart: always - networks: - - default - volumes: - - /mnt/repo-base/volumes/onlyoffice/DocumentServer/data:/var/www/onlyoffice/Data - - /mnt/repo-base/volumes/onlyoffice/DocumentServer/logs:/var/log/onlyoffice - onlyoffice-mail-server: - image: onlyoffice/mailserver:1.6.35 - container_name: onlyoffice-mail-server - stdin_open: true - hostname: onlyoffice.${DOMAIN} - restart: always - networks: - - default - volumes: - - /mnt/repo-base/volumes/onlyoffice/MailServer/data:/var/vmail - - /mnt/repo-base/volumes/onlyoffice/MailServer/data/certs:/etc/pki/tls/mailserver - - /mnt/repo-base/volumes/onlyoffice/MailServer/logs:/var/log - - /mnt/repo-base/volumes/onlyoffice/MailServer/mysql:/var/lib/mysql - onlyoffice-community-server: - image: onlyoffice/communityserver:9.6.5.771 - container_name: onlyoffice-community-server - restart: always - networks: - - default - ports: - - 5222:5222 - environment: - - DOCUMENT_SERVER_PORT_80_TCP_ADDR=onlyoffice-document-server - - MAIL_SERVER_DB_HOST=onlyoffice-mail-server - volumes: - - /mnt/repo-base/volumes/onlyoffice/CommunityServer/data:/var/www/onlyoffice/Data - - /mnt/repo-base/volumes/onlyoffice/CommunityServer/mysql:/var/lib/mysql - - /mnt/repo-base/volumes/onlyoffice/CommunityServer/logs:/var/log/onlyoffice - - /mnt/repo-base/volumes/onlyoffice/DocumentServer/data:/var/www/onlyoffice/DocumentServerData - depends_on: - - onlyoffice-documentserver - - onlyoffice-mail-server -- GitLab From 7b2dddd464a94e9e3f8327bca62f93e2e0d9650e Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:00:18 +0530 Subject: [PATCH 27/72] Move fpm conf files to config folder --- config/nextcloud/.keep | 0 {templates => config}/nextcloud/x-fpm-overloads.conf | 0 config/nextcloud/x-php-overloads.ini | 8 ++++++++ templates/nextcloud/x-php-overloads.ini | 2 -- 4 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 config/nextcloud/.keep rename {templates => config}/nextcloud/x-fpm-overloads.conf (100%) create mode 100644 config/nextcloud/x-php-overloads.ini delete mode 100644 templates/nextcloud/x-php-overloads.ini diff --git a/config/nextcloud/.keep b/config/nextcloud/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/templates/nextcloud/x-fpm-overloads.conf b/config/nextcloud/x-fpm-overloads.conf similarity index 100% rename from templates/nextcloud/x-fpm-overloads.conf rename to config/nextcloud/x-fpm-overloads.conf diff --git a/config/nextcloud/x-php-overloads.ini b/config/nextcloud/x-php-overloads.ini new file mode 100644 index 0000000..459f5e8 --- /dev/null +++ b/config/nextcloud/x-php-overloads.ini @@ -0,0 +1,8 @@ +[PHP] +opcache.enable=1 +opcache.interned_strings_buffer=8 +opcache.max_accelerated_files=10000 +opcache.memory_consumption=128 +opcache.save_comments=1 +opcache.revalidate_freq=1 +memory_limit = 512M diff --git a/templates/nextcloud/x-php-overloads.ini b/templates/nextcloud/x-php-overloads.ini deleted file mode 100644 index e1ff110..0000000 --- a/templates/nextcloud/x-php-overloads.ini +++ /dev/null @@ -1,2 +0,0 @@ -[PHP] -memory_limit = 512M -- GitLab From 5e92705b43e7a7a2523fa6275e8fcf3568dc3356 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:01:15 +0530 Subject: [PATCH 28/72] Removed copy of fpm overload conf --- scripts/init-repo.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 63f6511..2a0b09e 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -94,9 +94,6 @@ cat "templates/nginx/sites-enabled/postfixadmin.conf" | sed "s/@@@DOMAIN@@@/$DOM cat "templates/nginx/sites-enabled/rspamd.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/rspamd.conf" cat "templates/nginx/sites-enabled/welcome.conf" | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/nginx/sites-enabled/welcome.conf" -cp "templates/nextcloud/x-fpm-overloads.conf" "config/nextcloud/x-fpm-overloads.conf" -cp "templates/nextcloud/x-php-overloads.ini" "config/nextcloud/x-php-overloads.ini" - # confirm DNS is ready echo "" echo "" -- GitLab From d6febff56ee8ed593f71c2ebc0cb8096670c16b6 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:10:23 +0530 Subject: [PATCH 29/72] Added suggested config.php improvements for caching and for externalized log and data dirs --- templates/docker-compose/docker-compose-base.yml | 4 +++- templates/nextcloud/config.php | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml index ddd921d..efa2ba9 100644 --- a/templates/docker-compose/docker-compose-base.yml +++ b/templates/docker-compose/docker-compose-base.yml @@ -140,10 +140,12 @@ services: - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} volumes: - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - - /mnt/repo-base/volumes/nextcloud/data:/var/www/html/data/ + - /mnt/repo-base/volumes/nextcloud/data:/var/www/data/ - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ + - /mnt/repo-base/volumes/nextcloud/log:/var/www/log/ + - /mnt/repo-base/volumes/redis/db:/tmp/redis depends_on: - mariadb diff --git a/templates/nextcloud/config.php b/templates/nextcloud/config.php index 4bad40e..15b45ef 100644 --- a/templates/nextcloud/config.php +++ b/templates/nextcloud/config.php @@ -3,6 +3,7 @@ $CONFIG = array ( 'lost_password_link' => 'https://mail.@@@DOMAIN@@@/users/password-recover.php', 'htaccess.RewriteBase' => '/', 'memcache.local' => '\OC\Memcache\APCu', + 'memcache.distributed' => '\OC\Memcache\Redis', 'memcache.locking' => '\OC\Memcache\Redis', 'redis' => [ 'host' => 'redis', @@ -67,6 +68,9 @@ $CONFIG = array ( ), 'theme' => 'eCloud', 'loglevel' => 2, + 'logfile' => '/var/www/log/nextcloud.log', + 'filelocking.enabled' => true, + 'cron_log' => true, 'preview_max_x' => 1024, 'preview_max_y' => 1024, 'skeletondirectory' => '/var/www/skeleton/', -- GitLab From 9c79562f99c0622f472403a105ba2ad092fc5ae2 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:17:34 +0530 Subject: [PATCH 30/72] Updated docker-compose to single yml --- .../docker-compose/docker-compose-base.yml | 165 ------------------ .../docker-compose-networks.yml | 25 --- 2 files changed, 190 deletions(-) delete mode 100644 templates/docker-compose/docker-compose-base.yml delete mode 100644 templates/docker-compose/docker-compose-networks.yml diff --git a/templates/docker-compose/docker-compose-base.yml b/templates/docker-compose/docker-compose-base.yml deleted file mode 100644 index efa2ba9..0000000 --- a/templates/docker-compose/docker-compose-base.yml +++ /dev/null @@ -1,165 +0,0 @@ -version: '3' - -services: - mailserver: - image: mailserver2/mailserver:1.1.4 - container_name: mailserver - domainname: ${DOMAIN} # Mail server A/MX/FQDN & reverse PTR = mail.${DOMAIN}. - hostname: mail - restart: always - networks: - - default - ports: - - "25:25" # SMTP - Required - - "110:110" # POP3 STARTTLS - Optional - For webmails/desktop clients - - "143:143" # IMAP STARTTLS - Optional - For webmails/desktop clients - # - "465:465" # SMTPS SSL/TLS - Optional - Enabled for compatibility reason, otherwise disabled - - "587:587" # Submission STARTTLS - Optional - For webmails/desktop clients - - "993:993" # IMAPS SSL/TLS - Optional - For webmails/desktop clients - - "995:995" # POP3S SSL/TLS - Optional - For webmails/desktop clients - - "4190:4190" # SIEVE STARTTLS - Optional - Recommended for mail filtering - environment: - - DBPASS=${DBPASS} - - RSPAMD_PASSWORD=${RSPAMD_PASSWORD} - - ADD_DOMAINS=${ADD_DOMAINS} - - ENABLE_POP3=${ENABLE_POP3} - - DISABLE_RATELIMITING=${DISABLE_RATELIMITING} - - RELAY_NETWORKS=172.16.0.0/12 - # Full list of options: https://github.com/hardware/mailserver#environment-variables - volumes: - - /mnt/repo-base/volumes/mail:/var/mail - - /mnt/repo-base/config/letsencrypt/certstore:/etc/letsencrypt - - /mnt/repo-base/config/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf - - /mnt/repo-base/config/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf - - /mnt/repo-base/config/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf - - /mnt/repo-base/config/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf - - /mnt/repo-base/config/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map - - /mnt/repo-base/config/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf - - /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf - depends_on: - - mariadb - - redis - - postfixadmin: - image: registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1 - container_name: postfixadmin - restart: always - networks: - - default - environment: - - DBPASS=${DBPASS} - - DBHOST=mariadb - - DOMAIN=${DOMAIN} - - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} - - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} - - POSTFIXADMIN_DB_TYPE=mysqli - - POSTFIXADMIN_DB_HOST=mariadb - - POSTFIXADMIN_DB_USER=postfix - - POSTFIXADMIN_DB_NAME=postfix - - POSTFIXADMIN_DB_PASSWORD=${DBPASS} - - POSTFIXADMIN_SMTP_SERVER=mailserver - - POSTFIXADMIN_SMTP_PORT=587 - - volumes: - - /mnt/repo-base/volumes/mail:/var/mail - depends_on: - - mailserver - - mariadb - - mariadb: - image: mariadb:10.3 - container_name: mariadb - restart: always - networks: - - default - environment: - # Note: These variables are only used for the first start. Later changes are ignored. - - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - - MYSQL_DATABASE=${PFDB_DB} - - MYSQL_USER=${PFDB_USR} - - MYSQL_PASSWORD=${DBPASS} - volumes: - - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql - - /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro - redis: - image: redis:6.0-alpine - container_name: redis - restart: always - networks: - - default - command: redis-server --appendonly yes - volumes: - - /mnt/repo-base/volumes/redis/db:/data - - /mnt/repo-base/volumes/redis/tmp:/tmp/redis - - welcome: - image: registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2 - container_name: welcome - environment: - - DOMAINS=${VHOSTS_ACCOUNTS} - - DOMAIN=${DOMAIN} - - IS_WELCOME=true - - PFDB_HOST=mariadb - - PFDB_DB=${PFDB_DB} - - PFDB_USR=${PFDB_USR} - - PFDB_PW=${DBPASS} - - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} - - WELCOME_SECRET_SHA=${WELCOME_SECRET_SHA} - - WEBSITE_SECRET=${WEBSITE_SECRET} - - SMTP_HOST=${SMTP_HOST} - - SMTP_FROM=${SMTP_FROM} - - SMTP_PW=${SMTP_PW} - - SMTP_PORT=587 - - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} - - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} - - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} - restart: always - networks: - - default - volumes: - - /mnt/repo-base/volumes/accounts:/var/accounts - depends_on: - - mariadb - - nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 - container_name: nextcloud - restart: always - networks: - - default - environment: - - MYSQL_DATABASE=${MYSQL_DATABASE_NC} - - MYSQL_USER=${MYSQL_USER_NC} - - MYSQL_PASSWORD=${MYSQL_PASSWORD_NC} - - MYSQL_HOST=mariadb - - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} - # below ENV disabled so NC container do not start install - # - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - - OVERWRITEPROTOCOL=https - - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} - volumes: - - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ - - /mnt/repo-base/volumes/nextcloud/data:/var/www/data/ - - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ - - /mnt/repo-base/volumes/nextcloud/log:/var/www/log/ - - /mnt/repo-base/volumes/redis/db:/tmp/redis - depends_on: - - mariadb - - automx: - image: registry.gitlab.e.foundation/e/infra/docker-mailstack:automx-0.1.0 - container_name: automx - hostname: automx - environment: - - VIRTUAL_HOST=${VIRTUAL_HOST} - - DOMAIN=${DOMAIN} - - HOSTNAME=automx - restart: always - networks: - - default - volumes: - - /mnt/repo-base/config/automx/automx.conf:/etc/automx.conf - diff --git a/templates/docker-compose/docker-compose-networks.yml b/templates/docker-compose/docker-compose-networks.yml deleted file mode 100644 index 901503f..0000000 --- a/templates/docker-compose/docker-compose-networks.yml +++ /dev/null @@ -1,25 +0,0 @@ - - nginx: - image: nginx:1.19-alpine - container_name: nginx - restart: unless-stopped - networks: - - default - ports: - - "80:8000" - - "443:4430" - volumes: - - /mnt/repo-base/config/nginx/sites-enabled:/etc/nginx/conf.d/ - - /mnt/repo-base/config/nginx/params:/etc/nginx/params/ - - /mnt/repo-base/config/letsencrypt/certstore:/certs - - /mnt/repo-base/config/nginx/passwds:/passwds - - /mnt/repo-base/config/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge - - /mnt/repo-base/volumes/nextcloud/html:/var/www/html - depends_on: - - nextcloud - - automx - - postfixadmin - - welcome - - mailserver - #- onlyoffice-community-server - -- GitLab From bba1cbd643bd57d1bea8299657bc2c7c364c67ee Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:21:35 +0530 Subject: [PATCH 31/72] Updated README.md for bootstrap changes --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 118713f..20f3f8d 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,15 @@ In the following text, `$DOMAIN` refers to the domain (`youdomain.com`) that you Login to the server via ssh as root (on Linux/macOS the ssh client is available out of the box, on Windows you need to use an ssh client like [Putty](https://www.putty.org/) for example). +- Please note that for Ubuntu 20.04+, you will have to add the repository for "SaltStack" using the [instructions](https://repo.saltproject.io/#ubuntu) + Execute this command and follow its on-screen instructions: ``` $ ssh root@$DOMAIN -# wget https://gitlab.e.foundation/e/infra/bootstrap/raw/master/bootstrap-generic.sh -# bash bootstrap-generic.sh https://gitlab.e.foundation/e/infra/ecloud-selfhosting +# git clone https://gitlab.e.foundation/e/infra/ecloud-selfhosting.git --single-branch --branch master /mnt/repo-base +# cd /mnt/repo-base +#bash scripts/bootstrap-generic.sh https://gitlab.e.foundation/e/infra/ecloud-selfhosting ``` The setup script will ask you to input some details of your setup (like your domain name) and to setup additional DNS records (the two A records plus the PTR record were set already above). -- GitLab From d52ea712e1dbe7833b156f935d6f19853a747014 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:24:27 +0530 Subject: [PATCH 32/72] Removed remaining OnlyOffice references --- README.md | 14 +----- templates/nginx/sites-enabled/onlyoffice.conf | 46 ------------------- 2 files changed, 1 insertion(+), 59 deletions(-) delete mode 100644 templates/nginx/sites-enabled/onlyoffice.conf diff --git a/README.md b/README.md index 20f3f8d..ea37759 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This way, a user can use [/e/OS](https://e.foundation/products/) on a smartphone 5. tasks 6. device configuration... -The setup, which is relying on NextCloud, OnlyOffice, Postfix, and other open source components, is very close to the one used on [ecloud.global](https://ecloud.global). +The setup, which is relying on NextCloud, Postfix, and other open source components, is very close to the one used on [ecloud.global](https://ecloud.global). Important note: this project is currently in beta. You should have some experience with Linux server administration if you want to use it. The current setup makes updates difficult, @@ -25,12 +25,6 @@ For the full setup, the following server hardware is recommended: - 4 GB RAM - 20 GB disk space -For the setup without OnlyOffice, requirements are a bit lower: - -- 1 core CPU (x86/x86-64 only, ARM not supported yet) -- 2 GB RAM -- 15 GB disk space - Disk space only refers to the basic installation. You will need additional space for any emails, documents and files you store on the server. @@ -106,8 +100,6 @@ Optionally enter additional domain(s) (comma separated, no white spaces) to hand Enter alternative email: someone@example.org -Do you want to install OnlyOffice? [y/n] -n Your management domain is: yourdomain.com Your additional domains are: [N/A] Is this correct? (yes or no) yes @@ -142,8 +134,6 @@ A few services can't be configured automatically and need manual account creatio To change it, visit `https://$DOMAIN/apps/rainloop/app/?admin` and enter username: `admin` and password: `12345`. Go to the security tab to change the password. -**OnlyOffice**: Open `office.$DOMAIN`, then follow the instructions to add a new admin user. This -is only necessary if you chose to install OnlyOffice. ## Available Services @@ -161,8 +151,6 @@ Your credentials for postfix admin (https://mail.yourdomain.com) are: user/pass [rainloop](https://www.rainloop.net/) - `welcome.$DOMAIN`: Allows users to sign up for a new account (you can create signup links with `bash /mnt/repo-base/scripts/generate-signup-link.sh`, account creation with this "self service" is only possible when such a link is generated) -- `office.$DOMAIN`: Create and edit office documents ([OnlyOffice](https://www.onlyoffice.com/)) - (only when you answered yes to the question "Install OnlyOffice?" during setup obviously) ## Administration diff --git a/templates/nginx/sites-enabled/onlyoffice.conf b/templates/nginx/sites-enabled/onlyoffice.conf deleted file mode 100644 index 25d7059..0000000 --- a/templates/nginx/sites-enabled/onlyoffice.conf +++ /dev/null @@ -1,46 +0,0 @@ -server { - listen 8000; - server_name office.@@@DOMAIN@@@; - location /.well-known/acme-challenge/ { - alias /etc/letsencrypt/acme-challenge/.well-known/acme-challenge/; - } - location / { - return 301 https://$host$request_uri; - } -} - -server { - listen 4430 ssl http2; - server_name office.@@@DOMAIN@@@; - - ssl_certificate /certs/live/office.@@@DOMAIN@@@/fullchain.pem; - ssl_certificate_key /certs/live/office.@@@DOMAIN@@@/privkey.pem; - - include /etc/nginx/params/ssl_params; - include /etc/nginx/params/headers_params; - - add_header Strict-Transport-Security "max-age=31536000;"; - client_max_body_size 1024M; - - #auth_basic "Who's this?"; - #auth_basic_user_file /passwds/.htpasswd; - - location / { - # Fix OnlyOffice mixed content problem with ChunkedUploader - add_header Content-Security-Policy upgrade-insecure-requests always; - add_header "Access-Control-Allow-Origin" "*" always; - add_header "Access-Control-Allow-Methods" "POST, GET, OPTIONS" always; - add_header "Access-Control-Allow-Headers" "Content-Type, Accept, Authorization, Origin, Content-Range, Content-Disposition" always; - proxy_hide_header Access-Control-Allow-Origin; - proxy_read_timeout 300s; - client_max_body_size 4096M; - if ($request_method = "OPTIONS") { - return 204; - } - proxy_pass http://onlyoffice-community-server:80; - include /etc/nginx/params/proxy_params; - - # hide this version header for better security - proxy_set_header x-aspnet-verion ""; - } -} -- GitLab From e57d60a2fa8944a489827465b15353b6cf8184ed Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:24:49 +0530 Subject: [PATCH 33/72] Removed OO refs from docs also --- docs/delete-account.md | 5 ---- docs/env_file.md | 1 - docs/update_onlyoffice.md | 53 --------------------------------------- 3 files changed, 59 deletions(-) delete mode 100644 docs/update_onlyoffice.md diff --git a/docs/delete-account.md b/docs/delete-account.md index 21cddbd..39056e8 100644 --- a/docs/delete-account.md +++ b/docs/delete-account.md @@ -5,8 +5,3 @@ Delete User Cccount - `ssh user@$DOMAIN` - `cd /mnt/repo-base/` - `sudo ./scripts/delete-account $USER` - -2. onlyoffice data - - go to [https://office.$DOMAIN/products/people/#sortorder=ascending](https://office.$DOMAIN/products/people/#sortorder=ascending) - - search for the username - - click on the arrow at the far right and press "Delete profile" diff --git a/docs/env_file.md b/docs/env_file.md index 3d87655..171f828 100644 --- a/docs/env_file.md +++ b/docs/env_file.md @@ -8,7 +8,6 @@ selfhosting (ie, the domain you entered during setup). DOMAIN=example.com # the main domain for your installation ADD_DOMAINS=example.com, example2.com # one or more domains that are used for email ALT_EMAIL=myname@some-other-domain.com # admin email address -INSTALL_ONLYOFFICE=n # y or n, whether Onlyoffice is installed ``` ## Nextcloud diff --git a/docs/update_onlyoffice.md b/docs/update_onlyoffice.md deleted file mode 100644 index b3bf922..0000000 --- a/docs/update_onlyoffice.md +++ /dev/null @@ -1,53 +0,0 @@ -# UPDATE PROCEDURE (expect downtime) - -```shell -# this is knowingly not using compose functionality to stop/rm/pull - -# Stop containers -docker stop onlyoffice-community-server -docker stop onlyoffice-document-server -docker stop onlyoffice-mail-server - -#Create backup copy of files -cp -pR /mnt/docker/onlyoffice{,.bck} - -# Save image IDs of old images to a file -docker images | grep office > /somewhere/a-file.txt - - -docker rm onlyoffice-community-server -docker rm onlyoffice-document-server -docker rm onlyoffice-mail-server - -docker pull onlyoffice/documentserver -docker pull onlyoffice/communityserver -docker pull onlyoffice/mailserver - -# Start again -cd /mnt/docker/compose -docker-compose up -d -``` - -# ROLLBACK IN CASE OF ISSUE (expect downtime) - -```shell -# Stop and delete containers as above - -# Delete new images -docker rmi onlyoffice/documentserver -docker rmi onlyoffice/communityserver -docker rmi onlyoffice/mailserver - -# Retag the previous images version (see a-file.txt) IMAGE iDs to the correct name, e.g.: -docker tag 9a77d093202e onlyoffice/documentserver -docker tag 0e667b917252 onlyoffice/communityserver -dockr tag 6b2398f473ea onlyoffice/mailserver - -# Move current files to yet another location and move previous backup into original location -mv /mnt/docker/onlyoffice /mnt/docker/onlyoffice.bck.rolledback -mv /mnt/docker/onlyoffice.bck /mnt/docker/onlyoffice - -# Start again -cd /mnt/docker/compose -docker-compose up -d -``` \ No newline at end of file -- GitLab From 1a27b50fa038408c933171333c8fed3679c83d9b Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 12 Oct 2021 23:26:47 +0530 Subject: [PATCH 34/72] Updated README for bootstrap.sh --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ea37759..6bc3d17 100644 --- a/README.md +++ b/README.md @@ -82,13 +82,13 @@ Execute this command and follow its on-screen instructions: $ ssh root@$DOMAIN # git clone https://gitlab.e.foundation/e/infra/ecloud-selfhosting.git --single-branch --branch master /mnt/repo-base # cd /mnt/repo-base -#bash scripts/bootstrap-generic.sh https://gitlab.e.foundation/e/infra/ecloud-selfhosting +# bash scripts/bootstrap.sh ``` The setup script will ask you to input some details of your setup (like your domain name) and to setup additional DNS records (the two A records plus the PTR record were set already above). Example session for yourdomain.com: ``` -bash bootstrap-generic.sh https://gitlab.e.foundation/e/infra/ecloud-selfhosting +bash bootstrap-generic.sh [...] Resolving deltas: 100% (681/681), done. System update and packages installation .. -- GitLab From 3576b51f94758d0ca51e02000cafc7cb2bc09fe6 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 14:46:47 +0530 Subject: [PATCH 35/72] Re-added docker-compose.yml pasting from templates --- scripts/init-repo.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 2a0b09e..a6edaf5 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -57,6 +57,8 @@ echo "WEBSITE_SECRET=not_defined" >> "$ENVFILE" source /mnt/repo-base/scripts/base.sh DC_DIR="templates/docker-compose/" +cat "${DC_DIR}docker-compose.yml" > docker-compose.yml +NUM_CERTIFICATES="3" # To be constructed repo specific echo "VHOSTS_ACCOUNTS=welcome.$DOMAIN" >> "$ENVFILE" -- GitLab From c42a58203001f2130dce133bcfadbfbfd6c1887e Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 15:13:54 +0530 Subject: [PATCH 36/72] Don't ignore docker-compose.yml in templaets --- .gitignore | 2 +- templates/docker-compose/docker-compose.yml | 187 ++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 templates/docker-compose/docker-compose.yml diff --git a/.gitignore b/.gitignore index c393c8d..de762f8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.iml # docker config files -docker-compose.yml +./docker-compose.yml .env # data for the local installation diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml new file mode 100644 index 0000000..44647d2 --- /dev/null +++ b/templates/docker-compose/docker-compose.yml @@ -0,0 +1,187 @@ +version: '3' + +services: + mailserver: + image: mailserver2/mailserver:1.1.4 + container_name: mailserver + domainname: ${DOMAIN} # Mail server A/MX/FQDN & reverse PTR = mail.${DOMAIN}. + hostname: mail + restart: always + networks: + - default + ports: + - "25:25" # SMTP - Required + - "110:110" # POP3 STARTTLS - Optional - For webmails/desktop clients + - "143:143" # IMAP STARTTLS - Optional - For webmails/desktop clients + # - "465:465" # SMTPS SSL/TLS - Optional - Enabled for compatibility reason, otherwise disabled + - "587:587" # Submission STARTTLS - Optional - For webmails/desktop clients + - "993:993" # IMAPS SSL/TLS - Optional - For webmails/desktop clients + - "995:995" # POP3S SSL/TLS - Optional - For webmails/desktop clients + - "4190:4190" # SIEVE STARTTLS - Optional - Recommended for mail filtering + environment: + - DBPASS=${DBPASS} + - RSPAMD_PASSWORD=${RSPAMD_PASSWORD} + - ADD_DOMAINS=${ADD_DOMAINS} + - ENABLE_POP3=${ENABLE_POP3} + - DISABLE_RATELIMITING=${DISABLE_RATELIMITING} + - RELAY_NETWORKS=172.16.0.0/12 + # Full list of options: https://github.com/hardware/mailserver#environment-variables + volumes: + - /mnt/repo-base/volumes/mail:/var/mail + - /mnt/repo-base/config/letsencrypt/certstore:/etc/letsencrypt + - /mnt/repo-base/config/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf + - /mnt/repo-base/config/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf + - /mnt/repo-base/config/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf + - /mnt/repo-base/config/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf + - /mnt/repo-base/config/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map + - /mnt/repo-base/config/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf + - /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf + depends_on: + - mariadb + - redis + + postfixadmin: + image: registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1 + container_name: postfixadmin + restart: always + networks: + - default + environment: + - DBPASS=${DBPASS} + - DBHOST=mariadb + - DOMAIN=${DOMAIN} + - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} + - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} + - POSTFIXADMIN_DB_TYPE=mysqli + - POSTFIXADMIN_DB_HOST=mariadb + - POSTFIXADMIN_DB_USER=postfix + - POSTFIXADMIN_DB_NAME=postfix + - POSTFIXADMIN_DB_PASSWORD=${DBPASS} + - POSTFIXADMIN_SMTP_SERVER=mailserver + - POSTFIXADMIN_SMTP_PORT=587 + volumes: + - /mnt/repo-base/volumes/mail:/var/mail + depends_on: + - mailserver + - mariadb + + mariadb: + image: mariadb:10.3 + container_name: mariadb + restart: always + networks: + - default + environment: + # Note: These variables are only used for the first start. Later changes are ignored. + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${PFDB_DB} + - MYSQL_USER=${PFDB_USR} + - MYSQL_PASSWORD=${DBPASS} + volumes: + - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql + - /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro + redis: + image: redis:6.0-alpine + container_name: redis + restart: always + networks: + - default + command: redis-server --appendonly yes + volumes: + - /mnt/repo-base/volumes/redis/db:/data + - /mnt/repo-base/volumes/redis/tmp:/tmp/redis + + welcome: + image: registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2 + container_name: welcome + environment: + - DOMAINS=${VHOSTS_ACCOUNTS} + - DOMAIN=${DOMAIN} + - IS_WELCOME=true + - PFDB_HOST=mariadb + - PFDB_DB=${PFDB_DB} + - PFDB_USR=${PFDB_USR} + - PFDB_PW=${DBPASS} + - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} + - WELCOME_SECRET_SHA=${WELCOME_SECRET_SHA} + - WEBSITE_SECRET=${WEBSITE_SECRET} + - SMTP_HOST=${SMTP_HOST} + - SMTP_FROM=${SMTP_FROM} + - SMTP_PW=${SMTP_PW} + - SMTP_PORT=587 + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} + restart: always + networks: + - default + volumes: + - /mnt/repo-base/volumes/accounts:/var/accounts + depends_on: + - mariadb + + nextcloud: + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 + container_name: nextcloud + restart: always + networks: + - default + environment: + - MYSQL_DATABASE=${MYSQL_DATABASE_NC} + - MYSQL_USER=${MYSQL_USER_NC} + - MYSQL_PASSWORD=${MYSQL_PASSWORD_NC} + - MYSQL_HOST=mariadb + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + # below ENV disabled so NC container do not start install + # - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - OVERWRITEPROTOCOL=https + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + volumes: + - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ + - /mnt/repo-base/volumes/nextcloud/data:/var/www/data/ + - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf + - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini + - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ + - /mnt/repo-base/volumes/nextcloud/log:/var/www/log/ + - /mnt/repo-base/volumes/redis/db:/tmp/redis + depends_on: + - mariadb + + automx: + image: registry.gitlab.e.foundation/e/infra/docker-mailstack:automx-0.1.0 + container_name: automx + hostname: automx + environment: + - VIRTUAL_HOST=${VIRTUAL_HOST} + - DOMAIN=${DOMAIN} + - HOSTNAME=automx + restart: always + networks: + - default + volumes: + - /mnt/repo-base/config/automx/automx.conf:/etc/automx.conf + + nginx: + image: nginx:1.19-alpine + container_name: nginx + restart: unless-stopped + networks: + - default + ports: + - "80:8000" + - "443:4430" + volumes: + - /mnt/repo-base/config/nginx/sites-enabled:/etc/nginx/conf.d/ + - /mnt/repo-base/config/nginx/params:/etc/nginx/params/ + - /mnt/repo-base/config/letsencrypt/certstore:/certs + - /mnt/repo-base/config/nginx/passwds:/passwds + - /mnt/repo-base/config/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge + - /mnt/repo-base/volumes/nextcloud/html:/var/www/html + depends_on: + - nextcloud + - automx + - postfixadmin + - welcome + - mailserver + -- GitLab From 6fa01a24a3ac73ec36c73f42bb9730d74c7006c3 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 15:22:06 +0530 Subject: [PATCH 37/72] Removed office domain from domains list --- scripts/init-repo.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index a6edaf5..e1c0824 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -73,7 +73,7 @@ echo "VIRTUAL_HOST=$VIRTUAL_HOST" >> "$ENVFILE" # fill autorenew config rm -f "/mnt/repo-base/config/letsencrypt/autorenew/ssl-domains.dat" -echo "$DOMAIN,$VIRTUAL_HOST,mail.$DOMAIN,spam.$DOMAIN,welcome.$DOMAIN$OFFICE_DOMAIN" | tr "," "\n" | while read CURDOMAIN; do +echo "$DOMAIN,$VIRTUAL_HOST,mail.$DOMAIN,spam.$DOMAIN,welcome.$DOMAIN" | tr "," "\n" | while read CURDOMAIN; do echo "$CURDOMAIN" >> config/letsencrypt/autorenew/ssl-domains.dat :; done @@ -114,7 +114,7 @@ echo "$ADD_DOMAINS" | tr "," "\n" | while read CURDOMAIN; do :; done echo "PTR(For reverse DNS),|,,|,mail.$DOMAIN,|,-" >> "$tempfile" echo "" -echo "$VIRTUAL_HOST,spam.$DOMAIN,welcome.$DOMAIN$OFFICE_DOMAIN" | tr "," "\n" | while read CURDOMAIN; do +echo "$VIRTUAL_HOST,spam.$DOMAIN,welcome.$DOMAIN" | tr "," "\n" | while read CURDOMAIN; do echo "CNAME,|,$CURDOMAIN,|,mail.$DOMAIN,|,-" >> "$tempfile" :; done column "$tempfile" -t -s "," @@ -150,7 +150,7 @@ fi bash scripts/ssl-renew.sh # verify LE status -CTR_LE=$(find config/letsencrypt/certstore/live/mail.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/spam.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/welcome.$DOMAIN/privkey.pem $OFFICE_LETSENCRYPT_KEY 2>/dev/null| wc -l) +CTR_LE=$(find config/letsencrypt/certstore/live/mail.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/spam.$DOMAIN/privkey.pem config/letsencrypt/certstore/live/welcome.$DOMAIN/privkey.pem 2>/dev/null| wc -l) CTR_AC_LE=$(echo "$VIRTUAL_HOST" | tr "," "\n" | while read CURDOMAIN; do find config/letsencrypt/certstore/live/$CURDOMAIN/privkey.pem 2>/dev/null | grep $CURDOMAIN && echo found || echo missing; done | grep missing | wc -l) if [ "$CTR_LE$CTR_AC_LE" = "${NUM_CERTIFICATES}0" ] -- GitLab From b7da59a4e81d9e6bbb70cfa44b5eea30edf90938 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 15:53:48 +0530 Subject: [PATCH 38/72] Removed duplicate repository --- scripts/init-repo.sh | 1 - templates/docker-compose/docker-compose.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index e1c0824..4541ea2 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -170,7 +170,6 @@ cat /mnt/repo-base/templates/nextcloud/config.php | sed "s/@@@DOMAIN@@@/$DOMAIN/ sed "s/@@@PFDB_USR@@@/$PFDB_USR/g" | sed "s/@@@DBPASS@@@/$PFDB_DBPASS/g" > \ "/mnt/repo-base/volumes/nextcloud/html/config/config.php" chown www-data:www-data "/mnt/repo-base/volumes/nextcloud/" -R -rm -f /mnt/repo-base/config/nextcloud/skeleton/.keep docker-compose up -d diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index 44647d2..c0eafe4 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -142,7 +142,6 @@ services: - /mnt/repo-base/volumes/nextcloud/data:/var/www/data/ - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini - - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ - /mnt/repo-base/volumes/nextcloud/log:/var/www/log/ - /mnt/repo-base/volumes/redis/db:/tmp/redis depends_on: -- GitLab From fde7c84a385c2b68912c183ef66a2bd64735358e Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 17:49:30 +0530 Subject: [PATCH 39/72] Installing drop_account from app store --- scripts/postinstall.sh | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 69bee43..439dd4a 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -39,17 +39,8 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enabl docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable ecloud-launcher docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:disable firstrunwizard docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 - docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks - -DROP_ACCOUNT_RELEASE_LINK='https://framagit.org/framasoft/nextcloud/drop_account/uploads/f4bba3da2e1db522eb41297b7fd2c001/drop_account.tar.gz' - -curl -fsSL -o drop_account.tar.gz \ -"$DROP_ACCOUNT_RELEASE_LINK" && \ -tar -xf drop_account.tar.gz -C /mnt/repo-base/volumes/nextcloud/html/custom_apps/ && \ -rm drop_account.tar.gz; - -docker-compose exec -T --user www-data nextcloud php occ app:enable drop_account +docker-compose exec -T --user www-data nextcloud php occ app:install drop_account echo "Installing custom ecloud drop account plugin" # Add WELCOME_SECRET from .env file as a system config value, to be used by our ecloud_drop_account plugin -- GitLab From f520ff5fbd23096175c272548eb0a535fd61be4c Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 17:50:18 +0530 Subject: [PATCH 40/72] Fixed occ path --- scripts/postinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 439dd4a..942869b 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -40,7 +40,7 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enabl docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:disable firstrunwizard docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:app:set rainloop rainloop-autologin --value 1 docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks -docker-compose exec -T --user www-data nextcloud php occ app:install drop_account +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install drop_account echo "Installing custom ecloud drop account plugin" # Add WELCOME_SECRET from .env file as a system config value, to be used by our ecloud_drop_account plugin -- GitLab From 2a6573fcfce530730d1fc2601cbeb3e1f1c9461e Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 13 Oct 2021 17:59:05 +0530 Subject: [PATCH 41/72] Point nc to nc-21 image --- templates/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index c0eafe4..3e05fb4 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -122,7 +122,7 @@ services: - mariadb nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:nc-21 container_name: nextcloud restart: always networks: -- GitLab From 17037a4e0dd529ba5f9ada718203068cebb7c2a7 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 14 Oct 2021 17:27:37 +0530 Subject: [PATCH 42/72] Fixed datadirectory --- README.md | 4 ++++ templates/nextcloud/config.php | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6bc3d17..23cf8af 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ A few services can't be configured automatically and need manual account creatio To change it, visit `https://$DOMAIN/apps/rainloop/app/?admin` and enter username: `admin` and password: `12345`. Go to the security tab to change the password. +## Cron configuration + +- Many background jobs(e.g. jobs run when an account is deleted) need cron to be setup correctly +- Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to setup cron on your server ## Available Services diff --git a/templates/nextcloud/config.php b/templates/nextcloud/config.php index 15b45ef..34f53b0 100644 --- a/templates/nextcloud/config.php +++ b/templates/nextcloud/config.php @@ -28,7 +28,7 @@ $CONFIG = array ( array ( 0 => '@@@DOMAIN@@@', ), - 'datadirectory' => '/var/www/html/data', + 'datadirectory' => '/var/www/data', 'overwrite.cli.url' => 'https://@@@DOMAIN@@@', 'overwriteprotocol' => 'https', 'mysql.utf8mb4' => true, -- GitLab From 8d51593d5994d60175bff9fa0e9184ae627a0d7b Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 14 Oct 2021 19:06:39 +0530 Subject: [PATCH 43/72] Fixed nc directory creation --- scripts/init-repo.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 4541ea2..afa8a17 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -164,12 +164,13 @@ else fi # create nextcloud config -mkdir -p "/mnt/repo-base/volumes/nextcloud/html/config/" +mkdir -p /mnt/repo-base/volumes/nextcloud/{html,data,log} +mkdir "/mnt/repo-base/volumes/nextcloud/html/config/" cat /mnt/repo-base/templates/nextcloud/config.php | sed "s/@@@DOMAIN@@@/$DOMAIN/g" | \ sed "s/@@@DRIVE_SMTP_PASSWORD@@@/$DRIVE_SMTP_PASSWORD/g" | sed "s/@@@PFDB_DB@@@/$PFDB_DB/g" | \ sed "s/@@@PFDB_USR@@@/$PFDB_USR/g" | sed "s/@@@DBPASS@@@/$PFDB_DBPASS/g" > \ "/mnt/repo-base/volumes/nextcloud/html/config/config.php" -chown www-data:www-data "/mnt/repo-base/volumes/nextcloud/" -R + chown -R www-data: "/mnt/repo-base/volumes/nextcloud/" docker-compose up -d -- GitLab From e3bd003551a6d89db4a6b200e86544c9c1b83cb1 Mon Sep 17 00:00:00 2001 From: akhil Date: Fri, 15 Oct 2021 19:21:38 +0530 Subject: [PATCH 44/72] Added sed to replace the invite template domain --- scripts/postinstall.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 942869b..1655ddf 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -76,6 +76,9 @@ echo "Adding email accounts used by system senders (drive, ...)" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add drive@$DOMAIN --password $DRIVE_SMTP_PASSWORD --password2 $DRIVE_SMTP_PASSWORD --name "drive" --email-other $ALT_EMAIL docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add $SMTP_FROM --password $SMTP_PW --password2 $SMTP_PW --name "welcome" --email-other $ALT_EMAIL +echo "Setting the right domain in welcome email templates" +docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec sed -i "s/ecloud.global/$DOMAIN/g" {} \; + # display DKIM DNS setup info/instructions to the user echo -e "\n\n\n" echo -e "Please add the following records to your domain's DNS configuration:\n" -- GitLab From 27e5febb46c608622a64ddb3e23abb710607370e Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 18 Oct 2021 13:13:49 +0530 Subject: [PATCH 45/72] Removed lost password link in config.php --- templates/nextcloud/config.php | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/nextcloud/config.php b/templates/nextcloud/config.php index 34f53b0..70d5689 100644 --- a/templates/nextcloud/config.php +++ b/templates/nextcloud/config.php @@ -1,6 +1,5 @@ 'https://mail.@@@DOMAIN@@@/users/password-recover.php', 'htaccess.RewriteBase' => '/', 'memcache.local' => '\OC\Memcache\APCu', 'memcache.distributed' => '\OC\Memcache\Redis', -- GitLab From ed443edc4185e84d40c391d09cafe70e14ff3531 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 18 Oct 2021 14:25:37 +0530 Subject: [PATCH 46/72] Added webfinger and nodeinfo rules to nc nginx conf --- templates/nginx/sites-enabled/nextcloud.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/templates/nginx/sites-enabled/nextcloud.conf b/templates/nginx/sites-enabled/nextcloud.conf index 9d01aab..ff18ff7 100644 --- a/templates/nginx/sites-enabled/nextcloud.conf +++ b/templates/nginx/sites-enabled/nextcloud.conf @@ -42,6 +42,12 @@ server { location = /.well-known/caldav { return 301 $scheme://$host/remote.php/dav; } + location = /.well-known/webfinger { + return 301 $scheme://$host/index.php$uri; + } + location = /.well-known/nodeinfo { + return 301 $scheme://$host/index.php$uri; + } client_max_body_size 4096M; fastcgi_buffers 64 4K; -- GitLab From be7ec0c1255e4115e3dca31fa7b640c423a0b214 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 19 Oct 2021 17:49:44 +0530 Subject: [PATCH 47/72] Removed http2; Added maintenance:theme:update --- scripts/postinstall.sh | 3 +++ templates/nginx/sites-enabled/autoconfig.conf | 2 +- templates/nginx/sites-enabled/nextcloud.conf | 2 +- templates/nginx/sites-enabled/postfixadmin.conf | 2 +- templates/nginx/sites-enabled/rspamd.conf | 2 +- templates/nginx/sites-enabled/welcome.conf | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 1655ddf..2b9081b 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -25,6 +25,9 @@ docker-compose exec -T --user www-data nextcloud php occ db:convert-filecache-bi docker-compose exec -T --user www-data nextcloud php occ config:system:set trusted_domains 0 --value="$DOMAIN" docker-compose exec -T --user www-data nextcloud php occ app:disable theming +# Update theme +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ maintenance:theme:update + echo "Enabling nextcloud apps" docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable calendar docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:enable notes diff --git a/templates/nginx/sites-enabled/autoconfig.conf b/templates/nginx/sites-enabled/autoconfig.conf index b203a0b..9be5652 100644 --- a/templates/nginx/sites-enabled/autoconfig.conf +++ b/templates/nginx/sites-enabled/autoconfig.conf @@ -10,7 +10,7 @@ server { } server { - listen 4430 ssl http2; + listen 4430 ssl; server_name @@@SERVICE@@@.@@@DOMAIN@@@; ssl_certificate /certs/live/@@@SERVICE@@@.@@@DOMAIN@@@/fullchain.pem; diff --git a/templates/nginx/sites-enabled/nextcloud.conf b/templates/nginx/sites-enabled/nextcloud.conf index ff18ff7..3ba52cc 100644 --- a/templates/nginx/sites-enabled/nextcloud.conf +++ b/templates/nginx/sites-enabled/nextcloud.conf @@ -14,7 +14,7 @@ server { } server { - listen 4430 ssl http2; + listen 4430 ssl; server_name @@@DOMAIN@@@; root /var/www/html; diff --git a/templates/nginx/sites-enabled/postfixadmin.conf b/templates/nginx/sites-enabled/postfixadmin.conf index 5e99cc6..ae73561 100644 --- a/templates/nginx/sites-enabled/postfixadmin.conf +++ b/templates/nginx/sites-enabled/postfixadmin.conf @@ -10,7 +10,7 @@ server { } server { - listen 4430 ssl http2; + listen 4430 ssl; server_name mail.@@@DOMAIN@@@; ssl_certificate /certs/live/mail.@@@DOMAIN@@@/fullchain.pem; diff --git a/templates/nginx/sites-enabled/rspamd.conf b/templates/nginx/sites-enabled/rspamd.conf index fc722b9..69e6dd7 100644 --- a/templates/nginx/sites-enabled/rspamd.conf +++ b/templates/nginx/sites-enabled/rspamd.conf @@ -10,7 +10,7 @@ server { } server { - listen 4430 ssl http2; + listen 4430 ssl; server_name spam.@@@DOMAIN@@@; ssl_certificate /certs/live/spam.@@@DOMAIN@@@/fullchain.pem; diff --git a/templates/nginx/sites-enabled/welcome.conf b/templates/nginx/sites-enabled/welcome.conf index 25baf3f..bcbd5f3 100644 --- a/templates/nginx/sites-enabled/welcome.conf +++ b/templates/nginx/sites-enabled/welcome.conf @@ -10,7 +10,7 @@ server { } server { - listen 4430 ssl http2; + listen 4430 ssl; server_name welcome.@@@DOMAIN@@@; ssl_certificate /certs/live/welcome.@@@DOMAIN@@@/fullchain.pem; -- GitLab From 0d69773132b1ac8705d9eec0e8b169b916ff82f3 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 11:19:15 +0530 Subject: [PATCH 48/72] Set background jobs to use system cron by default --- README.md | 7 ++++--- scripts/postinstall.sh | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 23cf8af..78135de 100644 --- a/README.md +++ b/README.md @@ -134,10 +134,11 @@ A few services can't be configured automatically and need manual account creatio To change it, visit `https://$DOMAIN/apps/rainloop/app/?admin` and enter username: `admin` and password: `12345`. Go to the security tab to change the password. -## Cron configuration +## Background job configuration -- Many background jobs(e.g. jobs run when an account is deleted) need cron to be setup correctly -- Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to setup cron on your server +- Many background jobs(e.g. jobs run when an account is deleted) need to run for eCloud to work correctly +- In this installation, background jobs are set to use system cron and crontab for this is added +- Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to change background job configuration ## Available Services diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 2b9081b..1edc388 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -25,6 +25,15 @@ docker-compose exec -T --user www-data nextcloud php occ db:convert-filecache-bi docker-compose exec -T --user www-data nextcloud php occ config:system:set trusted_domains 0 --value="$DOMAIN" docker-compose exec -T --user www-data nextcloud php occ app:disable theming +# Set background jobs to use system cron +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ background:cron + +# add crontab on the server to run cron.php every 5 minutes +crontab -l | { + cat + echo "*/5 * * * * cd /mnt/repo-base && /usr/bin/docker-compose exec -T -u www-data nextcloud php -f /var/www/html/cron.php 2>&1 | /usr/bin/logger -t NC_CRON" +} | crontab - + # Update theme docker-compose exec -T --user www-data nextcloud php /var/www/html/occ maintenance:theme:update @@ -64,7 +73,7 @@ done chown www-data:www-data /mnt/repo-base/volumes/nextcloud/ -R echo "Creating postfix database schema" -curl --silent -L https://mail.$DOMAIN/setup.php > /dev/null +curl --silent -L https://mail.$DOMAIN/setup.php >/dev/null echo "Adding Postfix admin superadmin account" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli admin add $ALT_EMAIL --password $PFA_SUPERADMIN_PASSWORD --password2 $PFA_SUPERADMIN_PASSWORD --superadmin @@ -85,7 +94,10 @@ docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec # display DKIM DNS setup info/instructions to the user echo -e "\n\n\n" echo -e "Please add the following records to your domain's DNS configuration:\n" -find /mnt/repo-base/volumes/mail/dkim/ -maxdepth 1 -mindepth 1 -type d | while read line; do DOMAIN=$(basename $line); echo " - DKIM record (TXT) for $DOMAIN:" && cat $line/mail.public.key; done +find /mnt/repo-base/volumes/mail/dkim/ -maxdepth 1 -mindepth 1 -type d | while read line; do + DOMAIN=$(basename $line) + echo " - DKIM record (TXT) for $DOMAIN:" && cat $line/mail.public.key +done echo "=================================================================================================================================" echo "=================================================================================================================================" -- GitLab From f436d346403cfff97ef8a12e547b274d0bc8ca0c Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 11:45:03 +0530 Subject: [PATCH 49/72] Changed nc version to 20.0.13 --- templates/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index 3e05fb4..eea84fb 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -122,7 +122,7 @@ services: - mariadb nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:nc-21 + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:20.0.13 container_name: nextcloud restart: always networks: -- GitLab From 52cb560c6c7118a5040f400747472d48cb25d727 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 11:48:23 +0530 Subject: [PATCH 50/72] Corrected nc image tag --- templates/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index eea84fb..c8f4f57 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -122,7 +122,7 @@ services: - mariadb nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:20.0.13 + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:20-0-13 container_name: nextcloud restart: always networks: -- GitLab From 9e5e7687be39d8025499f03ed78fff25c9711874 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 12:43:44 +0530 Subject: [PATCH 51/72] Added hosts entry for welcome in postinstall.sh --- README.md | 4 ++-- scripts/postinstall.sh | 5 +++++ templates/nginx/sites-enabled/nextcloud.conf | 6 ------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 78135de..a4dddf4 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Login to the server via ssh as root (on Linux/macOS the ssh client is available - Please note that for Ubuntu 20.04+, you will have to add the repository for "SaltStack" using the [instructions](https://repo.saltproject.io/#ubuntu) -Execute this command and follow its on-screen instructions: +Execute these commands and follow the on-screen instructions: ``` $ ssh root@$DOMAIN @@ -88,7 +88,7 @@ The setup script will ask you to input some details of your setup (like your dom Example session for yourdomain.com: ``` -bash bootstrap-generic.sh +bash bootstrap.sh [...] Resolving deltas: 100% (681/681), done. System update and packages installation .. diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 1edc388..ce488d3 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -88,6 +88,11 @@ echo "Adding email accounts used by system senders (drive, ...)" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add drive@$DOMAIN --password $DRIVE_SMTP_PASSWORD --password2 $DRIVE_SMTP_PASSWORD --name "drive" --email-other $ALT_EMAIL docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add $SMTP_FROM --password $SMTP_PW --password2 $SMTP_PW --name "welcome" --email-other $ALT_EMAIL +# Add server IP and domain to welcome's /etc/hosts +IP=$(dig mail.$DOMAIN| grep mail.$DOMAIN | grep -v '^;' | awk '{ print $NF }') +HOSTS_ENTRY="$IP $DOMAIN" +docker-compose exec -T welcome bash -c "echo $HOSTS_ENTRY >> /etc/hosts" + echo "Setting the right domain in welcome email templates" docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec sed -i "s/ecloud.global/$DOMAIN/g" {} \; diff --git a/templates/nginx/sites-enabled/nextcloud.conf b/templates/nginx/sites-enabled/nextcloud.conf index 3ba52cc..0f40745 100644 --- a/templates/nginx/sites-enabled/nextcloud.conf +++ b/templates/nginx/sites-enabled/nextcloud.conf @@ -42,12 +42,6 @@ server { location = /.well-known/caldav { return 301 $scheme://$host/remote.php/dav; } - location = /.well-known/webfinger { - return 301 $scheme://$host/index.php$uri; - } - location = /.well-known/nodeinfo { - return 301 $scheme://$host/index.php$uri; - } client_max_body_size 4096M; fastcgi_buffers 64 4K; -- GitLab From 72629ebe4ebee349e14500d9f8c15ec571845fa4 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 13:09:11 +0530 Subject: [PATCH 52/72] Updated tag to stable --- templates/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index c8f4f57..c0eafe4 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -122,7 +122,7 @@ services: - mariadb nextcloud: - image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:20-0-13 + image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 container_name: nextcloud restart: always networks: -- GitLab From b4d041858dc0e44fc7705dad5382d43c3647ebad Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 14:22:22 +0530 Subject: [PATCH 53/72] Added data-dir option in maintenance:install --- scripts/postinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index ce488d3..52bcd15 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -18,7 +18,7 @@ docker-compose exec -T --user www-data nextcloud php occ maintenance:install \ --admin-user="$NEXTCLOUD_ADMIN_USER" --admin-pass="$NEXTCLOUD_ADMIN_PASSWORD" \ --admin-email="$ALT_EMAIL" --database="mysql" --database-pass="$MYSQL_PASSWORD_NC" \ --database-name="$MYSQL_DATABASE_NC" --database-host="mariadb" --database-user="$MYSQL_USER_NC" \ - --database-port="3306" + --database-port="3306" --data-dir="/var/www/data" docker-compose exec -T --user www-data nextcloud php occ db:convert-filecache-bigint --no-interaction # Nextcloud resets trusted_domains to localhost during installation, so we have to set it again -- GitLab From f8ac06687e5b095b05ee3bb430e2e29bc1981f9f Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 14:53:28 +0530 Subject: [PATCH 54/72] Added extra host with NC host IP to welcome container --- deployment/questionnaire/questionnaire.dat | 1 + scripts/init-repo.sh | 3 +++ templates/docker-compose/docker-compose.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/deployment/questionnaire/questionnaire.dat b/deployment/questionnaire/questionnaire.dat index b448f58..9e57c61 100644 --- a/deployment/questionnaire/questionnaire.dat +++ b/deployment/questionnaire/questionnaire.dat @@ -35,3 +35,4 @@ DISABLE_RATELIMITING=false;default #SMTP_FROM=welcome@domainA.com #VIRTUAL_HOST (for each domain two subdomains autoconfig/autodiscover) #VHOSTS_ACCOUNTS=welcome.domainA.com +#NC_HOST_IP \ No newline at end of file diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index afa8a17..3cc89d1 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -146,6 +146,9 @@ then exit 1 fi +# Add NC_HOST_IP env variable to .env +echo "NC_HOST_IP=$IP" >> "$ENVFILE" + # Run LE cert request bash scripts/ssl-renew.sh diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index c0eafe4..8eb51cb 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -120,6 +120,8 @@ services: - /mnt/repo-base/volumes/accounts:/var/accounts depends_on: - mariadb + extra_hosts: + - "${DOMAIN}:${NC_HOST_IP}" nextcloud: image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 -- GitLab From cd98258acba95f4e0a032e2ba89b2ed011ed9cc8 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 15:02:47 +0530 Subject: [PATCH 55/72] Removed explicitly setting IP in welcome /etc/hosts --- scripts/postinstall.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 52bcd15..81d1810 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -88,10 +88,6 @@ echo "Adding email accounts used by system senders (drive, ...)" docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add drive@$DOMAIN --password $DRIVE_SMTP_PASSWORD --password2 $DRIVE_SMTP_PASSWORD --name "drive" --email-other $ALT_EMAIL docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add $SMTP_FROM --password $SMTP_PW --password2 $SMTP_PW --name "welcome" --email-other $ALT_EMAIL -# Add server IP and domain to welcome's /etc/hosts -IP=$(dig mail.$DOMAIN| grep mail.$DOMAIN | grep -v '^;' | awk '{ print $NF }') -HOSTS_ENTRY="$IP $DOMAIN" -docker-compose exec -T welcome bash -c "echo $HOSTS_ENTRY >> /etc/hosts" echo "Setting the right domain in welcome email templates" docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec sed -i "s/ecloud.global/$DOMAIN/g" {} \; -- GitLab From 34d76dca4c8301887706831af2fdbb89d2748929 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 17:21:04 +0530 Subject: [PATCH 56/72] Updated README.md --- README.md | 2 +- scripts/init-repo.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a4dddf4..7a20eeb 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Note about TLS certificates: a certificate will be added automatically during se ### Create an Ubuntu server instance -The project should work with any Ubuntu server (Virtual Private Server (VPS), dedicated server...) versions 18.04 (should work on 16.04). +The project should work with any Ubuntu server (Virtual Private Server (VPS), dedicated server...) versions 20.04 (should work on 18.04). Debian server should work as well, though it has not been tested yet. diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 3cc89d1..1f3b88e 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -173,7 +173,7 @@ cat /mnt/repo-base/templates/nextcloud/config.php | sed "s/@@@DOMAIN@@@/$DOMAIN/ sed "s/@@@DRIVE_SMTP_PASSWORD@@@/$DRIVE_SMTP_PASSWORD/g" | sed "s/@@@PFDB_DB@@@/$PFDB_DB/g" | \ sed "s/@@@PFDB_USR@@@/$PFDB_USR/g" | sed "s/@@@DBPASS@@@/$PFDB_DBPASS/g" > \ "/mnt/repo-base/volumes/nextcloud/html/config/config.php" - chown -R www-data: "/mnt/repo-base/volumes/nextcloud/" +chown -R www-data: "/mnt/repo-base/volumes/nextcloud/" docker-compose up -d -- GitLab From d9b1b6a8cb77e1575cff76277a7017b7b59df7fe Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 18:24:46 +0530 Subject: [PATCH 57/72] Added sed to replace domain in docker-welcome html templates also --- scripts/postinstall.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 81d1810..500a205 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -54,6 +54,8 @@ docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:ap docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install tasks docker-compose exec -T --user www-data nextcloud php /var/www/html/occ app:install drop_account +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ config:system:set integrity.check.disabled --value='true' --type=boolean + echo "Installing custom ecloud drop account plugin" # Add WELCOME_SECRET from .env file as a system config value, to be used by our ecloud_drop_account plugin docker-compose exec -T --user www-data nextcloud php occ config:system:set e_welcome_secret --value="$WELCOME_SECRET" @@ -89,8 +91,10 @@ docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailb docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli mailbox add $SMTP_FROM --password $SMTP_PW --password2 $SMTP_PW --name "welcome" --email-other $ALT_EMAIL -echo "Setting the right domain in welcome email templates" +echo "Setting the right domain in welcome templates" docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec sed -i "s/ecloud.global/$DOMAIN/g" {} \; +docker-compose exec -T welcome find /var/www/html/invite_template/ -type f -exec sed -i "s/e.email/$DOMAIN/g" {} \; +docker-compose exec -T welcome find /var/www/html/ -type f -name '*.html' -exec sed -i "s/e.email/$DOMAIN/g" {} \; # display DKIM DNS setup info/instructions to the user echo -e "\n\n\n" -- GitLab From 3bafe5b59c39feb366008332477ad0e9ae74e34f Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 18:30:12 +0530 Subject: [PATCH 58/72] Added admin overview message in README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7a20eeb..49ecf2f 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,10 @@ Go to the security tab to change the password. - In this installation, background jobs are set to use system cron and crontab for this is added - Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to change background job configuration +## Admin Overview + +- Post installation, please navigate to https://$DOMAIN/settings/admin/overview to check if there are any configuration warnings related to your installation + ## Available Services You can find login information for these services by running `bash /mnt/repo-base/scripts/showInfo.sh`. -- GitLab From 817e4650d3f8acbf677ef684a379037665145a1e Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 19:01:27 +0530 Subject: [PATCH 59/72] Added comments for http2 and freshclam.conf --- README.md | 3 ++- config/mail/clamav/freshclam.conf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 49ecf2f..ccf8cba 100644 --- a/README.md +++ b/README.md @@ -140,9 +140,10 @@ Go to the security tab to change the password. - In this installation, background jobs are set to use system cron and crontab for this is added - Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to change background job configuration -## Admin Overview +## Administration - Post installation, please navigate to https://$DOMAIN/settings/admin/overview to check if there are any configuration warnings related to your installation +- Please note that we have not used http2 protocol in nginx because of nextcloud's notably [slow performance](https://github.com/nextcloud/server/issues/25297) with http2 ## Available Services diff --git a/config/mail/clamav/freshclam.conf b/config/mail/clamav/freshclam.conf index 6b53852..f37ff27 100644 --- a/config/mail/clamav/freshclam.conf +++ b/config/mail/clamav/freshclam.conf @@ -22,7 +22,8 @@ CompressLocalDatabase no SafeBrowsing false Bytecode true NotifyClamd /etc/clamav/clamd.conf -# Check for new database 24 times a day +# Check for new database 4 times a day +# Spawns too many processes otherwise and causes server to run out of memory Checks 4 DatabaseMirror db.local.clamav.net DatabaseMirror switch.clamav.net -- GitLab From 77ff182efd793bb0164766a0f9c699462a8e5524 Mon Sep 17 00:00:00 2001 From: akhil Date: Wed, 20 Oct 2021 19:03:11 +0530 Subject: [PATCH 60/72] Single heading for administration in README --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ccf8cba..5a53c4f 100644 --- a/README.md +++ b/README.md @@ -140,11 +140,6 @@ Go to the security tab to change the password. - In this installation, background jobs are set to use system cron and crontab for this is added - Please follow the instructions [here](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/background_jobs_configuration.html) to change background job configuration -## Administration - -- Post installation, please navigate to https://$DOMAIN/settings/admin/overview to check if there are any configuration warnings related to your installation -- Please note that we have not used http2 protocol in nginx because of nextcloud's notably [slow performance](https://github.com/nextcloud/server/issues/25297) with http2 - ## Available Services You can find login information for these services by running `bash /mnt/repo-base/scripts/showInfo.sh`. @@ -166,6 +161,8 @@ Your credentials for postfix admin (https://mail.yourdomain.com) are: user/pass - `spam.$DOMAIN`: Email spam filter ([rspamd](https://www.rspamd.com/)) - `mail.$DOMAIN`: Administrate email and create accounts ([postfixadmin](http://postfixadmin.sourceforge.net/)) when not using the "self service" `welcome.$DOMAIN` - this requires you to set a intermediate password during account creation. +- Post installation, please navigate to https://$DOMAIN/settings/admin/overview to check if there are any configuration warnings related to your installation +- Please note that we have not used http2 protocol in nginx because of nextcloud's notably [slow performance](https://github.com/nextcloud/server/issues/25297) with http2 ## Setting up /e/ OS with /e/ selfhosting -- GitLab From 44d3626a4133fbc6dd2f2791e403883ab722f8e4 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 13:09:35 +0530 Subject: [PATCH 61/72] Added basic migration guide --- docs/delete-account.md | 2 +- docs/migration_guide.md | 66 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 docs/migration_guide.md diff --git a/docs/delete-account.md b/docs/delete-account.md index 39056e8..4727e5a 100644 --- a/docs/delete-account.md +++ b/docs/delete-account.md @@ -1,4 +1,4 @@ -Delete User Cccount +Delete User Account ------------------- 1. Delete email account, Nextcloud account and all data diff --git a/docs/migration_guide.md b/docs/migration_guide.md new file mode 100644 index 0000000..31ba516 --- /dev/null +++ b/docs/migration_guide.md @@ -0,0 +1,66 @@ +## To migrate from old selfhost installation to the latest + +### Update your docker-compose.yml file +1. Networking + 1. Remove the `serverbase` network entry as we move to using the `default` docker network` + 1. Replace `serverbase` with `default` in the `networks` entry for each service +1. `eelomailserver` + 1. Update image from `hardware/mailserver:1.1-stable` to `mailserver2/mailserver:1.1.4` + 1. Add freshclam configuration + - Add "freshclam.conf" from "config/mail/clamav" to "config-static/mail/clamav/" + - Mount the file through an entry in "volumes" like `- /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf` +1. `postfixadmin` + 1. Update image to `registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1` + 1. Ensure that the `DRIVE_SMTP_PASSWORD` and the `DBPASS` environment variables are set in the .env file + 1. Add the following environment variables to the service + ``` + - DBHOST=mariadb + - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} + - POSTFIXADMIN_DB_TYPE=mysqli + - POSTFIXADMIN_DB_HOST=mariadb + - POSTFIXADMIN_DB_USER=postfix + - POSTFIXADMIN_DB_NAME=postfix + - POSTFIXADMIN_DB_PASSWORD=${DBPASS} + - POSTFIXADMIN_SMTP_SERVER=eelomailserver + - POSTFIXADMIN_SMTP_PORT=587 + ``` + 1. Remove the volume entry `- /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh` as it is bundled in the image +1. `mariadb` + 1. Update image to `mariadb:10.3` + 1. Remove the volume entry `- /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d` + 1. Add the volume entry `- /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro` +1. `redis` + 1. Update image to `redis:6.0-alpine` +1. `welcome` + 1. Update image to `registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2` + 1. Ensure that the `NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET` environment variable is set correctly in the .env file + 1. Ensure that the `NC_HOST_IP` environment variable is set to point to your server's IP address in the .env file + 1. Add the following environment variables: + ``` + - SMTP_PORT=587 + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + ``` + 1. Add the following `extra_hosts` entry: + ``` + extra_hosts: + - "${DOMAIN}:${NC_HOST_IP}" + ``` +1. `nextcloud` + 1. Update image to `registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0` + 1. Add the following environment variables: + ``` + - MYSQL_DATABASE=${MYSQL_DATABASE_NC} + - MYSQL_USER=${MYSQL_USER_NC} + - MYSQL_PASSWORD=${MYSQL_PASSWORD_NC} + - MYSQL_HOST=mariadb + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} + - OVERWRITEPROTOCOL=https + - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + ``` +1. `create-account`: Remove the `create-account` service as it is no longer used +1. `nginx` + 1. Update image to `nginx:1.19-alpine` + 1. Set `restart` value to `unless-stopped` + 1. Remove `create-account` from the `depends_on` entry -- GitLab From 76f603c49b19c7e50183d334ff1a34d45b5088b1 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 13:35:26 +0530 Subject: [PATCH 62/72] Added instructions to edit config files --- docs/migration_guide.md | 57 ++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/docs/migration_guide.md b/docs/migration_guide.md index 31ba516..5695f62 100644 --- a/docs/migration_guide.md +++ b/docs/migration_guide.md @@ -1,18 +1,22 @@ ## To migrate from old selfhost installation to the latest -### Update your docker-compose.yml file +### Update your docker-compose.yml file and configuration files for your services +1. Run `docker-compose down` to stop all services before upgrading 1. Networking - 1. Remove the `serverbase` network entry as we move to using the `default` docker network` - 1. Replace `serverbase` with `default` in the `networks` entry for each service + - Remove the `serverbase` network entry as we move to using the `default` docker network` + - Replace `serverbase` with `default` in the `networks` entry for each service 1. `eelomailserver` - 1. Update image from `hardware/mailserver:1.1-stable` to `mailserver2/mailserver:1.1.4` - 1. Add freshclam configuration + - Update image from `hardware/mailserver:1.1-stable` to `mailserver2/mailserver:1.1.4` + - Add freshclam configuration - Add "freshclam.conf" from "config/mail/clamav" to "config-static/mail/clamav/" - Mount the file through an entry in "volumes" like `- /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf` + - Update the `.conf` files in `config-static/mail/dovecot` directory according to the `.conf` files in `config/mail/dovecot` in the repository + - Update the `.conf` files in `config-static/mail/rspamd` directory according to the `.conf` files in `config/mail/rspamd` in the repository + 1. `postfixadmin` - 1. Update image to `registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1` - 1. Ensure that the `DRIVE_SMTP_PASSWORD` and the `DBPASS` environment variables are set in the .env file - 1. Add the following environment variables to the service + - Update image to `registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1` + - Ensure that the `DRIVE_SMTP_PASSWORD` and the `DBPASS` environment variables are set in the .env file + - Add the following environment variables to the service ``` - DBHOST=mariadb - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} @@ -24,32 +28,33 @@ - POSTFIXADMIN_SMTP_SERVER=eelomailserver - POSTFIXADMIN_SMTP_PORT=587 ``` - 1. Remove the volume entry `- /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh` as it is bundled in the image + - Remove the volume entry `- /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh` as it is bundled in the image 1. `mariadb` - 1. Update image to `mariadb:10.3` - 1. Remove the volume entry `- /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d` - 1. Add the volume entry `- /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro` + - Update image to `mariadb:10.3` + - Remove the volume entry `- /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d` + - Add the ecloud.cnf file from `config/mariadb/` in repository to `config-static/mariadb/` on your server + - Add the volume entry `- /mnt/repo-base/config-static/mariadb/:/etc/mysql/conf.d/:ro` 1. `redis` - 1. Update image to `redis:6.0-alpine` + - Update image to `redis:6.0-alpine` 1. `welcome` - 1. Update image to `registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2` - 1. Ensure that the `NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET` environment variable is set correctly in the .env file - 1. Ensure that the `NC_HOST_IP` environment variable is set to point to your server's IP address in the .env file - 1. Add the following environment variables: + - Update image to `registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2` + - Ensure that the `NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET` environment variable is set correctly in the .env file + - Ensure that the `NC_HOST_IP` environment variable is set to point to your server's IP address in the .env file + - Add the following environment variables: ``` - SMTP_PORT=587 - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} ``` - 1. Add the following `extra_hosts` entry: + - Add the following `extra_hosts` entry: ``` extra_hosts: - "${DOMAIN}:${NC_HOST_IP}" ``` 1. `nextcloud` - 1. Update image to `registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0` - 1. Add the following environment variables: + - Update image to `registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0` + - Add the following environment variables: ``` - MYSQL_DATABASE=${MYSQL_DATABASE_NC} - MYSQL_USER=${MYSQL_USER_NC} @@ -59,8 +64,14 @@ - OVERWRITEPROTOCOL=https - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} ``` + - Update the `x-fpm-overloads.conf` and `x-php-overloads.ini` files in `config-static/nextcloud/` using the files in `config/nextcloud/` in repository as reference 1. `create-account`: Remove the `create-account` service as it is no longer used 1. `nginx` - 1. Update image to `nginx:1.19-alpine` - 1. Set `restart` value to `unless-stopped` - 1. Remove `create-account` from the `depends_on` entry + - Update image to `nginx:1.19-alpine` + - Set `restart` value to `unless-stopped` + - Remove `create-account` from the `depends_on` entry + - Update the files in `config-static/nginx/params` using the files in `config/nginx/params` in repository for reference + - Update the configs in `config-dynamic/nginx/sites-enabled/` using the configs in `templates/nginx/sites-enabled/` in repository for reference +1. Pull and update the latest versions + - Run `docker-compose pull` + - Run `docker-compose up --force-recreate -d` \ No newline at end of file -- GitLab From 5a688f4024c7228a057acdd8c32d8b33c2e0355e Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 14:58:57 +0530 Subject: [PATCH 63/72] Added diff for users upgrading --- docs/update-from-old-selfhost.diff | 241 ++++++++++++++++++ ...md => upgrade-to-latest-selfhost-guide.md} | 20 +- 2 files changed, 253 insertions(+), 8 deletions(-) create mode 100644 docs/update-from-old-selfhost.diff rename docs/{migration_guide.md => upgrade-to-latest-selfhost-guide.md} (77%) diff --git a/docs/update-from-old-selfhost.diff b/docs/update-from-old-selfhost.diff new file mode 100644 index 0000000..803d287 --- /dev/null +++ b/docs/update-from-old-selfhost.diff @@ -0,0 +1,241 @@ +diff --git a/docker-compose-old.yml b/docker-compose.yml +index 279ec78..8eb51cb 100644 +--- a/docker-compose-old.yml ++++ b/docker-compose.yml +@@ -1,14 +1,14 @@ +-version: '2.1' ++version: '3' + + services: +- eelomailserver: +- image: hardware/mailserver:1.1-stable +- container_name: eelomailserver ++ mailserver: ++ image: mailserver2/mailserver:1.1.4 ++ container_name: mailserver + domainname: ${DOMAIN} # Mail server A/MX/FQDN & reverse PTR = mail.${DOMAIN}. + hostname: mail + restart: always + networks: +- - serverbase ++ - default + ports: + - "25:25" # SMTP - Required + - "110:110" # POP3 STARTTLS - Optional - For webmails/desktop clients +@@ -28,42 +28,49 @@ services: + # Full list of options: https://github.com/hardware/mailserver#environment-variables + volumes: + - /mnt/repo-base/volumes/mail:/var/mail +- - /mnt/repo-base/config-dynamic/letsencrypt/certstore:/etc/letsencrypt +- - /mnt/repo-base/config-static/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf +- - /mnt/repo-base/config-static/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf +- - /mnt/repo-base/config-static/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf +- - /mnt/repo-base/config-static/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf +- - /mnt/repo-base/config-static/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map +- - /mnt/repo-base/config-static/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf ++ - /mnt/repo-base/config/letsencrypt/certstore:/etc/letsencrypt ++ - /mnt/repo-base/config/mail/dovecot/10-mail.conf:/etc/dovecot/conf.d/10-mail.conf ++ - /mnt/repo-base/config/mail/dovecot/90-quota.conf:/etc/dovecot/conf.d/90-quota.conf ++ - /mnt/repo-base/config/mail/dovecot/90-sieve.conf:/etc/dovecot/conf.d/90-sieve.conf ++ - /mnt/repo-base/config/mail/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf ++ - /mnt/repo-base/config/mail/rspamd/whitelist.sender.domain.map:/etc/rspamd/local.d/whitelist.sender.domain.map ++ - /mnt/repo-base/config/mail/rspamd/ratelimit.conf:/etc/rspamd/local.d/ratelimit.conf ++ - /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf + depends_on: + - mariadb + - redis + + postfixadmin: +- image: registry.gitlab.e.foundation:5000/e/infra/docker-postfixadmin:1.0.0 ++ image: registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1 + container_name: postfixadmin + restart: always + networks: +- - serverbase ++ - default + environment: + - DBPASS=${DBPASS} ++ - DBHOST=mariadb + - DOMAIN=${DOMAIN} ++ - DRIVE_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} + - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} +- - SMTPHOST=${SMTP_HOST} +- - ADMIN_SMTP_PASSWORD=${DRIVE_SMTP_PASSWORD} ++ - POSTFIXADMIN_DB_TYPE=mysqli ++ - POSTFIXADMIN_DB_HOST=mariadb ++ - POSTFIXADMIN_DB_USER=postfix ++ - POSTFIXADMIN_DB_NAME=postfix ++ - POSTFIXADMIN_DB_PASSWORD=${DBPASS} ++ - POSTFIXADMIN_SMTP_SERVER=mailserver ++ - POSTFIXADMIN_SMTP_PORT=587 + volumes: + - /mnt/repo-base/volumes/mail:/var/mail +- - /mnt/repo-base/scripts/postfixadmin-mailbox-postdeletion.sh:/usr/local/bin/postfixadmin-mailbox-postdeletion.sh + depends_on: +- - eelomailserver ++ - mailserver + - mariadb + + mariadb: +- image: mariadb:10.3.17 ++ image: mariadb:10.3 + container_name: mariadb + restart: always + networks: +- - serverbase ++ - default + environment: + # Note: These variables are only used for the first start. Later changes are ignored. + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} +@@ -71,22 +78,21 @@ services: + - MYSQL_USER=${PFDB_USR} + - MYSQL_PASSWORD=${DBPASS} + volumes: +- - /mnt/repo-base/volumes/mysql/db:/var/lib/mysql +- - /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d +- ++ - /mnt/repo-base/volumes/mysql/db/data:/var/lib/mysql ++ - /mnt/repo-base/config/mariadb/:/etc/mysql/conf.d/:ro + redis: +- image: redis:5.0-alpine ++ image: redis:6.0-alpine + container_name: redis + restart: always + networks: +- - serverbase ++ - default + command: redis-server --appendonly yes + volumes: + - /mnt/repo-base/volumes/redis/db:/data + - /mnt/repo-base/volumes/redis/tmp:/tmp/redis + + welcome: +- image: registry.gitlab.e.foundation:5000/e/infra/docker-welcome:1.1.0 ++ image: registry.gitlab.e.foundation/e/infra/docker-welcome:2.1.2 + container_name: welcome + environment: + - DOMAINS=${VHOSTS_ACCOUNTS} +@@ -102,34 +108,49 @@ services: + - SMTP_HOST=${SMTP_HOST} + - SMTP_FROM=${SMTP_FROM} + - SMTP_PW=${SMTP_PW} ++ - SMTP_PORT=587 ++ - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} ++ - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} ++ - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} + restart: always + networks: +- - serverbase ++ - default + volumes: + - /mnt/repo-base/volumes/accounts:/var/accounts + depends_on: + - mariadb ++ extra_hosts: ++ - "${DOMAIN}:${NC_HOST_IP}" + + nextcloud: +- image: nextcloud:16.0.5-fpm ++ image: registry.gitlab.e.foundation/e/infra/ecloud/nextcloud:316db4e0 + container_name: nextcloud + restart: always + networks: +- - serverbase ++ - default ++ environment: ++ - MYSQL_DATABASE=${MYSQL_DATABASE_NC} ++ - MYSQL_USER=${MYSQL_USER_NC} ++ - MYSQL_PASSWORD=${MYSQL_PASSWORD_NC} ++ - MYSQL_HOST=mariadb ++ - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} ++ # below ENV disabled so NC container do not start install ++ # - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} ++ - OVERWRITEPROTOCOL=https ++ - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} + volumes: + - /mnt/repo-base/volumes/nextcloud/html:/var/www/html/ +- - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ +- - /mnt/repo-base/volumes/nextcloud/config:/var/www/html/config/ +- - /mnt/repo-base/volumes/nextcloud/data:/var/www/html/data/ +- - /mnt/repo-base/config-dynamic/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf +- - /mnt/repo-base/config-dynamic/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini +- - /mnt/repo-base/volumes/redis/tmp:/tmp/redis/ ++ - /mnt/repo-base/volumes/nextcloud/data:/var/www/data/ ++ - /mnt/repo-base/config/nextcloud/x-fpm-overloads.conf:/usr/local/etc/php-fpm.d/x-fpm-overloads.conf ++ - /mnt/repo-base/config/nextcloud/x-php-overloads.ini:/usr/local/etc/php/conf.d/x-php-overloads.ini ++ - /mnt/repo-base/volumes/nextcloud/log:/var/www/log/ ++ - /mnt/repo-base/volumes/redis/db:/tmp/redis + depends_on: + - mariadb + + automx: +- image: registry.gitlab.e.foundation:5000/e/infra/docker-mailstack:automx-0.1.0 ++ image: registry.gitlab.e.foundation/e/infra/docker-mailstack:automx-0.1.0 + container_name: automx + hostname: automx + environment: +@@ -138,52 +159,30 @@ services: + - HOSTNAME=automx + restart: always + networks: +- - serverbase ++ - default + volumes: +- - /mnt/repo-base/config-dynamic/automx/automx.conf:/etc/automx.conf +- +- create-account: +- image: registry.gitlab.e.foundation:5000/e/infra/docker-create-account:1.0.1 +- container_name: create-account +- restart: always +- environment: +- - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER} +- - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} +- - POSTFIXADMIN_SSH_PASSWORD=${POSTFIXADMIN_SSH_PASSWORD} +- - DOMAIN=${DOMAIN} +- - CREATE_ACCOUNT_PASSWORD=${CREATE_ACCOUNT_PASSWORD} +- networks: +- - serverbase +- depends_on: +- - nextcloud +- - postfixadmin ++ - /mnt/repo-base/config/automx/automx.conf:/etc/automx.conf + + nginx: +- image: registry.gitlab.e.foundation:5000/e/infra/docker-nginx:1.17 ++ image: nginx:1.19-alpine + container_name: nginx +- restart: always ++ restart: unless-stopped + networks: +- - serverbase ++ - default + ports: + - "80:8000" + - "443:4430" + volumes: +- - /mnt/repo-base/config-dynamic/nginx/sites-enabled:/etc/nginx/conf.d/ +- - /mnt/repo-base/config-static/nginx/params:/etc/nginx/params/ +- - /mnt/repo-base/config-dynamic/letsencrypt/certstore:/certs +- - /mnt/repo-base/config-dynamic/nginx/passwds:/passwds +- - /mnt/repo-base/config-dynamic/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge ++ - /mnt/repo-base/config/nginx/sites-enabled:/etc/nginx/conf.d/ ++ - /mnt/repo-base/config/nginx/params:/etc/nginx/params/ ++ - /mnt/repo-base/config/letsencrypt/certstore:/certs ++ - /mnt/repo-base/config/nginx/passwds:/passwds ++ - /mnt/repo-base/config/letsencrypt/acme-challenge:/etc/letsencrypt/acme-challenge + - /mnt/repo-base/volumes/nextcloud/html:/var/www/html +- - /mnt/repo-base/volumes/nextcloud/custom_apps:/var/www/html/custom_apps/ + depends_on: + - nextcloud +- - create-account + - automx + - postfixadmin + - welcome +- - eelomailserver +- #- onlyoffice-community-server ++ - mailserver + +-networks: +- serverbase: +- driver: 'bridge' diff --git a/docs/migration_guide.md b/docs/upgrade-to-latest-selfhost-guide.md similarity index 77% rename from docs/migration_guide.md rename to docs/upgrade-to-latest-selfhost-guide.md index 5695f62..28c9446 100644 --- a/docs/migration_guide.md +++ b/docs/upgrade-to-latest-selfhost-guide.md @@ -1,17 +1,21 @@ ## To migrate from old selfhost installation to the latest +- You can use the [diff](update-from-old-selfhost.diff) to compare and update the `docker-compose.yml` configuration +- You can also follow the steps given below to update the configuration to the latest(Note that volume locations and service names are not changed but old ones are used) + ### Update your docker-compose.yml file and configuration files for your services 1. Run `docker-compose down` to stop all services before upgrading +1. Update `version` to '3' 1. Networking - Remove the `serverbase` network entry as we move to using the `default` docker network` - Replace `serverbase` with `default` in the `networks` entry for each service 1. `eelomailserver` - Update image from `hardware/mailserver:1.1-stable` to `mailserver2/mailserver:1.1.4` - Add freshclam configuration - - Add "freshclam.conf" from "config/mail/clamav" to "config-static/mail/clamav/" - - Mount the file through an entry in "volumes" like `- /mnt/repo-base/config/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf` - - Update the `.conf` files in `config-static/mail/dovecot` directory according to the `.conf` files in `config/mail/dovecot` in the repository - - Update the `.conf` files in `config-static/mail/rspamd` directory according to the `.conf` files in `config/mail/rspamd` in the repository + - Add [freshclam.conf](../config/mail/clamav/freshclam.conf) to "config-static/mail/clamav/" + - Mount the file through an entry in "volumes" like `- /mnt/repo-base/config-static/mail/clamav/freshclam.conf:/etc/clamav/freshclam.conf` + - Update the `.conf` files in `config-static/mail/dovecot` directory according to the `.conf` files in the [repository](../config/mail/dovecot/) + - Update the `.conf` files in `config-static/mail/rspamd` directory according to the `.conf` files in the [repository](../config/mail/rspamd/) 1. `postfixadmin` - Update image to `registry.gitlab.e.foundation/e/infra/docker-postfixadmin:2.0.1` @@ -32,7 +36,7 @@ 1. `mariadb` - Update image to `mariadb:10.3` - Remove the volume entry `- /mnt/repo-base/config-dynamic/nextcloud/database:/docker-entrypoint-initdb.d` - - Add the ecloud.cnf file from `config/mariadb/` in repository to `config-static/mariadb/` on your server + - Add the [ecloud.cnf](../config/mariadb/ecloud.cnf) to `config-static/mariadb/` on your server - Add the volume entry `- /mnt/repo-base/config-static/mariadb/:/etc/mysql/conf.d/:ro` 1. `redis` - Update image to `redis:6.0-alpine` @@ -64,14 +68,14 @@ - OVERWRITEPROTOCOL=https - NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET=${NEXTCLOUD_EMAIL_RECOVERY_APP_SECRET} ``` - - Update the `x-fpm-overloads.conf` and `x-php-overloads.ini` files in `config-static/nextcloud/` using the files in `config/nextcloud/` in repository as reference + - Update the `x-fpm-overloads.conf` and `x-php-overloads.ini` files in `config-static/nextcloud/` using the files in [config/nextcloud](../config/nextcloud/) as reference 1. `create-account`: Remove the `create-account` service as it is no longer used 1. `nginx` - Update image to `nginx:1.19-alpine` - Set `restart` value to `unless-stopped` - Remove `create-account` from the `depends_on` entry - - Update the files in `config-static/nginx/params` using the files in `config/nginx/params` in repository for reference - - Update the configs in `config-dynamic/nginx/sites-enabled/` using the configs in `templates/nginx/sites-enabled/` in repository for reference + - Update the files in `config-static/nginx/params` using the files in [config/nginx/params](../config/nginx/params/) for reference + - Update the configs in `config-dynamic/nginx/sites-enabled/` using the configs in [templates/nginx/sites-enabled](../templates/nginx/sites-enabled/) in repository for reference 1. Pull and update the latest versions - Run `docker-compose pull` - Run `docker-compose up --force-recreate -d` \ No newline at end of file -- GitLab From 8e53cd102e98920a2e203027425ec2983382e4e5 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 15:01:57 +0530 Subject: [PATCH 64/72] Minor typo correction --- docs/upgrade-to-latest-selfhost-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade-to-latest-selfhost-guide.md b/docs/upgrade-to-latest-selfhost-guide.md index 28c9446..f6d80cd 100644 --- a/docs/upgrade-to-latest-selfhost-guide.md +++ b/docs/upgrade-to-latest-selfhost-guide.md @@ -1,7 +1,7 @@ ## To migrate from old selfhost installation to the latest - You can use the [diff](update-from-old-selfhost.diff) to compare and update the `docker-compose.yml` configuration -- You can also follow the steps given below to update the configuration to the latest(Note that volume locations and service names are not changed but old ones are used) +- You can also follow the steps given below to update the configuration to the latest(Note that volume locations and service names are not changed but old ones are used in the steps given below) ### Update your docker-compose.yml file and configuration files for your services 1. Run `docker-compose down` to stop all services before upgrading -- GitLab From 3f3360b996f675af65e762c7a6b04cad23f7ce01 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 15:53:00 +0530 Subject: [PATCH 65/72] Removed ubuntu v 18.04 from README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a53c4f..784bd20 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Note about TLS certificates: a certificate will be added automatically during se ### Create an Ubuntu server instance -The project should work with any Ubuntu server (Virtual Private Server (VPS), dedicated server...) versions 20.04 (should work on 18.04). +The project should work with any Ubuntu server (Virtual Private Server (VPS), dedicated server...) version 20.04 Debian server should work as well, though it has not been tested yet. -- GitLab From 3f069dc833a66f0918c751c221af7b1c2fc26eb2 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 16:31:43 +0530 Subject: [PATCH 66/72] Set innodb_buffer_pool_size to 1G --- config/mariadb/ecloud.cnf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/mariadb/ecloud.cnf b/config/mariadb/ecloud.cnf index 9e16248..cf98153 100644 --- a/config/mariadb/ecloud.cnf +++ b/config/mariadb/ecloud.cnf @@ -3,5 +3,5 @@ transaction_isolation = READ-COMMITTED binlog_format = ROW -innodb_buffer_pool_size=512M +innodb_buffer_pool_size=1G innodb_io_capacity=4000 \ No newline at end of file -- GitLab From ed7da47bbc04c6b493f5c5b84564064aa10d75e9 Mon Sep 17 00:00:00 2001 From: akhil Date: Thu, 21 Oct 2021 18:12:55 +0530 Subject: [PATCH 67/72] Update ecloud.cnf --- config/mariadb/ecloud.cnf | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/config/mariadb/ecloud.cnf b/config/mariadb/ecloud.cnf index cf98153..e1b9b82 100644 --- a/config/mariadb/ecloud.cnf +++ b/config/mariadb/ecloud.cnf @@ -1,7 +1,26 @@ +[server] +performance_schema = on +skip_name_resolve = 1 + +innodb_buffer_pool_size=1G +innodb_buffer_pool_instances = 1 +innodb_buffer_pool_chunk_size = 1G +innodb_flush_log_at_trx_commit = 2 +innodb_log_buffer_size = 256M +innodb_max_dirty_pages_pct = 90 +innodb_io_capacity=4000 + +query_cache_type = 1 +query_cache_limit = 2M +query_cache_min_res_unit = 2k +query_cache_size = 64M +tmp_table_size= 64M +max_heap_table_size= 64M + + [mysqld] +character_set_server = utf8mb4 +collation_server = utf8mb4_general_ci +innodb_file_per_table=1 transaction_isolation = READ-COMMITTED binlog_format = ROW - - -innodb_buffer_pool_size=1G -innodb_io_capacity=4000 \ No newline at end of file -- GitLab From b1f40532b1dd0cce6e645cc38a5fb84e7472f14f Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 25 Oct 2021 12:32:29 +0530 Subject: [PATCH 68/72] Added apt update and apt upgrade instructions before installation --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 784bd20..4541433 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ Hosting at home is also possible in principle, but you will probably have proble First, create your hosting server. Please follow your hoster documentation to create your server or VPS. +- Please run the following commands and then reboot your server before installation: + - `apt update` + - `apt upgrade` + ### Set your server with proper DNS settings 1. point your domain DNS entries to your server -- GitLab From 73fbc7fa2b71d90b3e9a4b90d5bd54c0e9e888a2 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 25 Oct 2021 12:49:27 +0530 Subject: [PATCH 69/72] Moved sender whitelist to templates and made it use domain --- config/mail/.keep | 0 config/mail/rspamd/whitelist.sender.domain.map | 1 - scripts/init-repo.sh | 3 +++ templates/mail/rspamd/whitelist.sender.domain.map | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 config/mail/.keep delete mode 100644 config/mail/rspamd/whitelist.sender.domain.map create mode 100644 templates/mail/rspamd/whitelist.sender.domain.map diff --git a/config/mail/.keep b/config/mail/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/config/mail/rspamd/whitelist.sender.domain.map b/config/mail/rspamd/whitelist.sender.domain.map deleted file mode 100644 index 883cb0f..0000000 --- a/config/mail/rspamd/whitelist.sender.domain.map +++ /dev/null @@ -1 +0,0 @@ -e.email diff --git a/scripts/init-repo.sh b/scripts/init-repo.sh index 1f3b88e..1e9571e 100755 --- a/scripts/init-repo.sh +++ b/scripts/init-repo.sh @@ -82,6 +82,9 @@ echo "$DOMAIN,$VIRTUAL_HOST,mail.$DOMAIN,spam.$DOMAIN,welcome.$DOMAIN" | tr "," cat templates/automx/automx.conf | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/automx/automx.conf" chown www-data:www-data "config/automx/automx.conf" +# Configure rspamd whitelist +cat templates/mail/rspamd/whitelist.sender.domain.map | sed "s/@@@DOMAIN@@@/$DOMAIN/g" > "config/mail/rspamd/whitelist.sender.domain.map" + # Configure nginx vhost # automx diff --git a/templates/mail/rspamd/whitelist.sender.domain.map b/templates/mail/rspamd/whitelist.sender.domain.map new file mode 100644 index 0000000..2eb784d --- /dev/null +++ b/templates/mail/rspamd/whitelist.sender.domain.map @@ -0,0 +1 @@ +@@@DOMAIN@@@ \ No newline at end of file -- GitLab From 6a16cf5818a8a4d8ffdf0921acd67aa84d2e07d5 Mon Sep 17 00:00:00 2001 From: akhil Date: Mon, 25 Oct 2021 20:10:48 +0530 Subject: [PATCH 70/72] Added db:add-missing-indices occ command --- scripts/postinstall.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 500a205..05570ba 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -62,6 +62,9 @@ docker-compose exec -T --user www-data nextcloud php occ config:system:set e_wel # Add VHOST_ACCOUNTS from .env file as a system config value, to be used by our ecloud_drop_account plugin docker-compose exec -T --user www-data nextcloud php occ config:system:set e_welcome_domain --value="welcome.$DOMAIN" +# Add missing indices +docker-compose exec -T --user www-data nextcloud php /var/www/html/occ db:add-missing-indices + docker-compose exec -T --user www-data nextcloud php occ maintenance:mode --off echo "Restarting Nextcloud container" -- GitLab From 33e62719113c8e607e5cb5edcff7f23e610a4d1c Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 26 Oct 2021 10:50:59 +0530 Subject: [PATCH 71/72] Corrected superadmin for postfixadmin --- scripts/postinstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/postinstall.sh b/scripts/postinstall.sh index 05570ba..3c3370a 100755 --- a/scripts/postinstall.sh +++ b/scripts/postinstall.sh @@ -81,7 +81,7 @@ echo "Creating postfix database schema" curl --silent -L https://mail.$DOMAIN/setup.php >/dev/null echo "Adding Postfix admin superadmin account" -docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli admin add $ALT_EMAIL --password $PFA_SUPERADMIN_PASSWORD --password2 $PFA_SUPERADMIN_PASSWORD --superadmin +docker-compose exec -T postfixadmin /postfixadmin/scripts/postfixadmin-cli admin add $ALT_EMAIL --password $PFA_SUPERADMIN_PASSWORD --password2 $PFA_SUPERADMIN_PASSWORD --superadmin 1 # Adding domains to postfix is done by docker exec instead of docker-compose exec on purpose. Reason: with compose the loop aborts after the first item for an unknown reason echo "Adding domains to Postfix" -- GitLab From 2f210951736a0fdd49dd734c587162f46aa4fbf2 Mon Sep 17 00:00:00 2001 From: akhil Date: Tue, 26 Oct 2021 13:55:40 +0530 Subject: [PATCH 72/72] Updated PF_SMTP_SERVER var to mail. --- templates/docker-compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index 8eb51cb..7a03729 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -57,7 +57,7 @@ services: - POSTFIXADMIN_DB_USER=postfix - POSTFIXADMIN_DB_NAME=postfix - POSTFIXADMIN_DB_PASSWORD=${DBPASS} - - POSTFIXADMIN_SMTP_SERVER=mailserver + - POSTFIXADMIN_SMTP_SERVER=mail.${DOMAIN} - POSTFIXADMIN_SMTP_PORT=587 volumes: - /mnt/repo-base/volumes/mail:/var/mail -- GitLab