From df602c0e9d01e110b331952d8ba2ba2ab980b9ed Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 8 Jun 2021 14:33:43 +0200 Subject: [PATCH 01/20] prepare a production ready branch from wg-binding-and-LB-eeo-staging and master (spot 1.0.0) --- .env | 8 +++--- .gitlab-ci.yml | 63 +++++++++++++++++++++++++++++++++------------- docker-compose.yml | 37 ++++++++++++++++++++++----- 3 files changed, 81 insertions(+), 27 deletions(-) diff --git a/.env b/.env index 61ecbf8c1..0bfc8dfa2 100644 --- a/.env +++ b/.env @@ -2,10 +2,12 @@ SPOT_HOSTNAME=spot.ecloud.global SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_TAG=latest -SEARX_MORTY_URL=http://localhost:8089 +SEARX_MORTY_URL=https://localhost:8089 SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo -SEARX_PROXY_HTTP=socks5://tor:9050 -SEARX_PROXY_HTTPS=socks5://tor:9050 +SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 +SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 +WIREGUARD_IP=127.0.0.1 +REPO_BASE=/mnt/repo-base/staging-spot \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a1d861dc..f6526ac95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,9 @@ stages: - test - deploy +services: + - docker:20.10-dind + python: stage: check before_script: @@ -28,8 +31,6 @@ build:web: image: docker:git tags: - generic_privileged - services: - - docker:18-dind before_script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY script: @@ -64,36 +65,62 @@ test:unit: .deploy:template: stage: deploy before_script: - - eval $(ssh-agent -s) - - echo "$PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh - chmod 700 ~/.ssh - - echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - ssh $DOCKER_HOST "mkdir -p $(dirname $FILTRON_RULES) && echo '$(cat ./etc/filtron/rules.json)' > $FILTRON_RULES" + - echo "$SSH_PRIVATE_KEY_ED" > $HOME/.ssh/id_ed25519 + - echo "$SSH_PUBKEY_ED" > $HOME/.ssh/id_ed25519.pub + - echo "$SSH_KNOWN_HOSTS" > $HOME/.ssh/known_hosts + - chmod 600 ~/.ssh/id_ed25519 + - chmod 644 ~/.ssh/known_hosts ~/.ssh/id_ed25519.pub + - ssh $DOCKER_HOST "cd $PATH_STAGING" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull registry.gitlab.e.foundation:5000/e/cloud/my-spot/env -deploy:spot.test.cloud.global: +deploy:spot.eeo.one.backend1: extends: .deploy:template when: manual only: - branches environment: - name: test - url: https://spot.test.ecloud.global + name: eeo1 + url: https://spot.eeo.one variables: - DOCKER_HOST: ssh://root@spot.test.ecloud.global - FILTRON_RULES: /etc/filtron/rules.json - SPOT_HOSTNAME: spot.test.ecloud.global - SPOT_MORTY_HOSTNAME: proxy.spot.test.ecloud.global - SEARX_MORTY_URL: https://proxy.spot.test.ecloud.global - COMPOSE_PROJECT_NAME: my-spot - PRIVATE_KEY: ${PRIVATE_KEY_TEST} + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} + SPOT_HOSTNAME: spot.eeo.one + SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one + SEARX_MORTY_URL: https://proxy.spot.eeo.one + COMPOSE_PROJECT_NAME: staging-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND1_WG_IP} + script: + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose restart filtron + +deploy:spot.eeo.one.backend2: + extends: .deploy:template + when: manual + only: + - branches + environment: + name: eeo2 + url: https://spot.eeo.one + variables: + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} + SPOT_HOSTNAME: spot.eeo.one + SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one + SEARX_MORTY_URL: https://proxy.spot.eeo.one + COMPOSE_PROJECT_NAME: staging-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND2_WG_IP} script: - - docker-compose up -d --build --scale tor=5 + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - docker-compose up -d --build --scale tor-socks-proxy=5 - docker-compose restart filtron + deploy:spot.cloud.global: extends: .deploy:template only: diff --git a/docker-compose.yml b/docker-compose.yml index 5630cabfc..738e73843 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,17 +10,23 @@ x-logging: services: redis: image: redis:5.0.7-alpine + # container_name: spot-redis logging: *default-logging restart: unless-stopped + networks: + - spot-default command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru spot: image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG} + # container_name: spot build: context: . dockerfile: Dockerfile logging: *default-logging restart: unless-stopped + networks: + - spot-default environment: SEARX_SECRET: "${SEARX_SECRET}" SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -32,35 +38,46 @@ services: GUNICORN_LOGGER: 1 GUNICORN_LEVEL: INFO - nginx: + spot-nginx: image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG} + # container_name: spot-nginx build: context: . dockerfile: nginx.Dockerfile logging: *default-logging restart: unless-stopped + networks: + - spot-default environment: SEARX_MORTY_URL: "${SEARX_MORTY_URL}" filtron: image: dalf/filtron:latest + # container_name: spot-filtron logging: *default-logging restart: unless-stopped - command: -listen :3000 -rules /etc/filtron/rules.json -target nginx + command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx + networks: + - spot-default + - spot-wireguarded ports: - - "8088:3000" + - ${WIREGUARD_IP}:8088:3000 volumes: - - ${FILTRON_RULES:-./etc/filtron/rules.json}:/etc/filtron/rules.json + - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json labels: - "traefik.enable=true" - "traefik.http.routers.filtron.rule=Host(`${SPOT_HOSTNAME}`)" morty: image: dalf/morty:latest + # container_name: spot-morty logging: *default-logging restart: unless-stopped + networks: + - spot-default + - spot-wireguarded ports: - - "8089:3000" + - ${WIREGUARD_IP}:8089:3000 environment: DEBUG: "false" MORTY_ADDRESS: ":3000" @@ -70,6 +87,14 @@ services: - "traefik.http.routers.morty.rule=Host(`${SPOT_MORTY_HOSTNAME}`)" tor: - image: osminogin/tor-simple + image: peterdavehello/tor-socks-proxy:latest logging: *default-logging restart: unless-stopped + networks: + - spot-default + +networks: + spot-default: + external: true + spot-wireguarded: + external: true -- GitLab From 1d62e772a19322d3a6cd16321c19b86fd4522573 Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 8 Jun 2021 14:37:50 +0200 Subject: [PATCH 02/20] update tor container service name --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 738e73843..35ec2d3aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,7 +86,7 @@ services: - "traefik.enable=true" - "traefik.http.routers.morty.rule=Host(`${SPOT_MORTY_HOSTNAME}`)" - tor: + tor-socks-proxy: image: peterdavehello/tor-socks-proxy:latest logging: *default-logging restart: unless-stopped -- GitLab From 9c6d596189d9a88e6f9cfdfb3d223d409849cf15 Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 8 Jun 2021 15:46:43 +0200 Subject: [PATCH 03/20] filtron and morty ports added as vars to be able to overwrite them with gitlab ci. repo path too. --- .env | 4 +++- .gitlab-ci.yml | 49 ++++++++++++++++++++++++++++++++++++++-------- docker-compose.yml | 4 ++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/.env b/.env index 0bfc8dfa2..5ae275ee9 100644 --- a/.env +++ b/.env @@ -10,4 +10,6 @@ SEARX_UI_DEFAULT_THEME=eelo SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 WIREGUARD_IP=127.0.0.1 -REPO_BASE=/mnt/repo-base/staging-spot \ No newline at end of file +REPO_BASE=/mnt/repo-base/staging-spot +FILTRON_PORT=8088 +MORTY_PORT=8089 \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6526ac95..e1e50a635 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,6 +93,9 @@ deploy:spot.eeo.one.backend1: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} + REPO_BASE: /mnt/repo-base/staging-spot + FILTRON_PORT: 8088 + MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - docker-compose up -d --build --scale tor-socks-proxy=5 @@ -115,30 +118,60 @@ deploy:spot.eeo.one.backend2: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} + REPO_BASE: /mnt/repo-base/staging-spot + FILTRON_PORT: 8088 + MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - docker-compose up -d --build --scale tor-socks-proxy=5 - docker-compose restart filtron -deploy:spot.cloud.global: +deploy:spot.ecloud.global.backend1: extends: .deploy:template only: - tags environment: - name: prod + name: ecloud1 url: https://spot.ecloud.global variables: - DOCKER_HOST: ssh://spot@spot.ecloud.global - FILTRON_RULES: /home/spot/filtron/rules.json + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} SPOT_HOSTNAME: spot.ecloud.global SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global SEARX_MORTY_URL: https://proxy.spot.ecloud.global - COMPOSE_PROJECT_NAME: my-spot + COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - PRIVATE_KEY: ${PRIVATE_KEY_PROD} + WIREGUARD_IP: ${BACKEND1_WG_IP} + REPO_BASE: /mnt/repo-base/production-spot + FILTRON_PORT: 8098 + MORTY_PORT: 8099 script: - - docker-compose pull - - docker-compose up -d --scale tor=5 + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - docker-compose up -d --build --scale tor-socks-proxy=5 - docker-compose restart filtron + +deploy:spot.ecloud.global.backend2: + extends: .deploy:template + only: + - tags + environment: + name: ecloud2 + url: https://spot.ecloud.global + variables: + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} + SPOT_HOSTNAME: spot.ecloud.global + SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global + SEARX_MORTY_URL: https://proxy.spot.ecloud.global + COMPOSE_PROJECT_NAME: production-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND2_WG_IP} + REPO_BASE: /mnt/repo-base/production-spot + FILTRON_PORT: 8098 + MORTY_PORT: 8099 + script: + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose restart filtron + diff --git a/docker-compose.yml b/docker-compose.yml index 35ec2d3aa..3b795b13c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,7 @@ services: - spot-default - spot-wireguarded ports: - - ${WIREGUARD_IP}:8088:3000 + - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 volumes: - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json labels: @@ -77,7 +77,7 @@ services: - spot-default - spot-wireguarded ports: - - ${WIREGUARD_IP}:8089:3000 + - ${WIREGUARD_IP}:${MORTY_PORT}:3000 environment: DEBUG: "false" MORTY_ADDRESS: ":3000" -- GitLab From e528f35217eed130912fe982b89e4b397abe07f3 Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 8 Jun 2021 16:09:10 +0200 Subject: [PATCH 04/20] no need to build when deploying from tags as builts are available in the gitlab registry --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1e50a635..9ee886df1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,8 @@ deploy:spot.ecloud.global.backend1: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose pull + - docker-compose up -d --scale tor-socks-proxy=5 - docker-compose restart filtron deploy:spot.ecloud.global.backend2: @@ -172,6 +173,7 @@ deploy:spot.ecloud.global.backend2: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose pull + - docker-compose up -d --scale tor-socks-proxy=5 - docker-compose restart filtron -- GitLab From 240df06ffcc3039436eb9f276811bb5a1d1ddd49 Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 10 Jun 2021 14:02:11 +0200 Subject: [PATCH 05/20] remove spot-default network on staging stack, to let docker create a internal network didecated to this stack only, thus avoiding collision with prod spot --- docker-compose.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b795b13c..4908de7e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: # container_name: spot-redis logging: *default-logging restart: unless-stopped - networks: - - spot-default command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru spot: @@ -25,8 +23,6 @@ services: dockerfile: Dockerfile logging: *default-logging restart: unless-stopped - networks: - - spot-default environment: SEARX_SECRET: "${SEARX_SECRET}" SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -46,8 +42,6 @@ services: dockerfile: nginx.Dockerfile logging: *default-logging restart: unless-stopped - networks: - - spot-default environment: SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -58,7 +52,6 @@ services: restart: unless-stopped command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx networks: - - spot-default - spot-wireguarded ports: - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 @@ -74,7 +67,6 @@ services: logging: *default-logging restart: unless-stopped networks: - - spot-default - spot-wireguarded ports: - ${WIREGUARD_IP}:${MORTY_PORT}:3000 @@ -90,8 +82,7 @@ services: image: peterdavehello/tor-socks-proxy:latest logging: *default-logging restart: unless-stopped - networks: - - spot-default + networks: spot-default: -- GitLab From 62107df53faf2bc58fdb91ab0eafa68c8b633323 Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 10 Jun 2021 17:05:55 +0200 Subject: [PATCH 06/20] spicify a staging-spot-default network --- docker-compose.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4908de7e9..12f27d78d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,8 @@ services: # container_name: spot-redis logging: *default-logging restart: unless-stopped + networks: + - staging-spot-default command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru spot: @@ -23,6 +25,8 @@ services: dockerfile: Dockerfile logging: *default-logging restart: unless-stopped + networks: + - staging-spot-default environment: SEARX_SECRET: "${SEARX_SECRET}" SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -42,6 +46,8 @@ services: dockerfile: nginx.Dockerfile logging: *default-logging restart: unless-stopped + networks: + - staging-spot-default environment: SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -52,6 +58,7 @@ services: restart: unless-stopped command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx networks: + - staging-spot-default - spot-wireguarded ports: - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 @@ -67,6 +74,7 @@ services: logging: *default-logging restart: unless-stopped networks: + - staging-spot-default - spot-wireguarded ports: - ${WIREGUARD_IP}:${MORTY_PORT}:3000 @@ -82,10 +90,11 @@ services: image: peterdavehello/tor-socks-proxy:latest logging: *default-logging restart: unless-stopped - + networks: + - spot-default networks: - spot-default: + staging-spot-default: external: true spot-wireguarded: external: true -- GitLab From 8662d304de25888bf10a24b2ba7c3b14821ad606 Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 15 Jun 2021 15:11:58 +0200 Subject: [PATCH 07/20] error in tor-proxy image network, and first test of ecloud ipv4 proxy --- .env | 4 ++-- .gitlab-ci.yml | 4 ++-- docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 5ae275ee9..51b46ff36 100644 --- a/.env +++ b/.env @@ -7,8 +7,8 @@ SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo -SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 -SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 +SEARX_PROXY_HTTP=proxy01.ecloud.global:1099 +SEARX_PROXY_HTTPS=proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 REPO_BASE=/mnt/repo-base/staging-spot FILTRON_PORT=8088 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9ee886df1..018dae177 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -98,7 +98,7 @@ deploy:spot.eeo.one.backend1: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose up -d --build - docker-compose restart filtron deploy:spot.eeo.one.backend2: @@ -123,7 +123,7 @@ deploy:spot.eeo.one.backend2: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - docker-compose up -d --build --scale tor-socks-proxy=5 + - docker-compose up -d --build - docker-compose restart filtron diff --git a/docker-compose.yml b/docker-compose.yml index 12f27d78d..627fc27e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,7 +91,7 @@ services: logging: *default-logging restart: unless-stopped networks: - - spot-default + - staging-spot-default networks: staging-spot-default: -- GitLab From e097efa4005f49c23e34f6b39d696a754e940917 Mon Sep 17 00:00:00 2001 From: diroots Date: Tue, 15 Jun 2021 15:37:52 +0200 Subject: [PATCH 08/20] add http:// scheme to proxy env vars --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 51b46ff36..bf184ce38 100644 --- a/.env +++ b/.env @@ -7,8 +7,8 @@ SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo -SEARX_PROXY_HTTP=proxy01.ecloud.global:1099 -SEARX_PROXY_HTTPS=proxy01.ecloud.global:1099 +SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 +SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 REPO_BASE=/mnt/repo-base/staging-spot FILTRON_PORT=8088 -- GitLab From 6caa41765aade812f01bf6c0e8745163a90bc100 Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 17 Jun 2021 14:45:34 +0200 Subject: [PATCH 09/20] give distinct proxy to each spot backend, so they are autonomous --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 018dae177..342085879 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,6 +89,8 @@ deploy:spot.eeo.one.backend1: SPOT_HOSTNAME: spot.eeo.one SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one SEARX_MORTY_URL: https://proxy.spot.eeo.one + SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} @@ -114,6 +116,8 @@ deploy:spot.eeo.one.backend2: SPOT_HOSTNAME: spot.eeo.one SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one SEARX_MORTY_URL: https://proxy.spot.eeo.one + SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} @@ -139,6 +143,8 @@ deploy:spot.ecloud.global.backend1: SPOT_HOSTNAME: spot.ecloud.global SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global SEARX_MORTY_URL: https://proxy.spot.ecloud.global + SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} @@ -164,6 +170,8 @@ deploy:spot.ecloud.global.backend2: SPOT_HOSTNAME: spot.ecloud.global SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global SEARX_MORTY_URL: https://proxy.spot.ecloud.global + SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} -- GitLab From c65fdd343a65c03051ec43ddb1ad4a72efd3b86f Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 24 Jun 2021 12:02:44 +0200 Subject: [PATCH 10/20] force local network separation staging / prod (and some cleanup) --- .env | 3 ++- .gitlab-ci.yml | 8 ++++++-- docker-compose.yml | 27 +++++++-------------------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.env b/.env index bf184ce38..68e58b8d4 100644 --- a/.env +++ b/.env @@ -12,4 +12,5 @@ SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 REPO_BASE=/mnt/repo-base/staging-spot FILTRON_PORT=8088 -MORTY_PORT=8089 \ No newline at end of file +MORTY_PORT=8089 +INTERNAL_NETWORK_NAME=staging-spot-default \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 342085879..769978e2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,6 +96,7 @@ deploy:spot.eeo.one.backend1: SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} REPO_BASE: /mnt/repo-base/staging-spot + INTERNAL_NETWORK_NAME: staging-spot-default FILTRON_PORT: 8088 MORTY_PORT: 8089 script: @@ -123,6 +124,7 @@ deploy:spot.eeo.one.backend2: SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} REPO_BASE: /mnt/repo-base/staging-spot + INTERNAL_NETWORK_NAME: staging-spot-default FILTRON_PORT: 8088 MORTY_PORT: 8089 script: @@ -150,12 +152,13 @@ deploy:spot.ecloud.global.backend1: SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} REPO_BASE: /mnt/repo-base/production-spot + INTERNAL_NETWORK_NAME: spot-default FILTRON_PORT: 8098 MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - docker-compose pull - - docker-compose up -d --scale tor-socks-proxy=5 + - docker-compose up -d - docker-compose restart filtron deploy:spot.ecloud.global.backend2: @@ -177,11 +180,12 @@ deploy:spot.ecloud.global.backend2: SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} REPO_BASE: /mnt/repo-base/production-spot + INTERNAL_NETWORK_NAME: spot-default FILTRON_PORT: 8098 MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - docker-compose pull - - docker-compose up -d --scale tor-socks-proxy=5 + - docker-compose up -d - docker-compose restart filtron diff --git a/docker-compose.yml b/docker-compose.yml index 627fc27e8..e508245f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,19 +14,18 @@ services: logging: *default-logging restart: unless-stopped networks: - - staging-spot-default + - ${INTERNAL_NETWORK_NAME} command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru spot: image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG} - # container_name: spot build: context: . dockerfile: Dockerfile logging: *default-logging restart: unless-stopped networks: - - staging-spot-default + - ${INTERNAL_NETWORK_NAME} environment: SEARX_SECRET: "${SEARX_SECRET}" SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -47,34 +46,29 @@ services: logging: *default-logging restart: unless-stopped networks: - - staging-spot-default + - ${INTERNAL_NETWORK_NAME} environment: SEARX_MORTY_URL: "${SEARX_MORTY_URL}" filtron: image: dalf/filtron:latest - # container_name: spot-filtron logging: *default-logging restart: unless-stopped command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx networks: - - staging-spot-default + - ${INTERNAL_NETWORK_NAME} - spot-wireguarded ports: - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 volumes: - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json - labels: - - "traefik.enable=true" - - "traefik.http.routers.filtron.rule=Host(`${SPOT_HOSTNAME}`)" morty: image: dalf/morty:latest - # container_name: spot-morty logging: *default-logging restart: unless-stopped networks: - - staging-spot-default + - ${INTERNAL_NETWORK_NAME} - spot-wireguarded ports: - ${WIREGUARD_IP}:${MORTY_PORT}:3000 @@ -82,19 +76,12 @@ services: DEBUG: "false" MORTY_ADDRESS: ":3000" MORTY_KEY: "${SEARX_MORTY_KEY}" - labels: - - "traefik.enable=true" - - "traefik.http.routers.morty.rule=Host(`${SPOT_MORTY_HOSTNAME}`)" - tor-socks-proxy: - image: peterdavehello/tor-socks-proxy:latest - logging: *default-logging - restart: unless-stopped - networks: - - staging-spot-default networks: staging-spot-default: external: true + spot-default: + external: true spot-wireguarded: external: true -- GitLab From 219677c895f206342c05d8b42f003ca7a22c4503 Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 24 Jun 2021 13:33:15 +0200 Subject: [PATCH 11/20] update local .env file in each server to be able to manipulate stack out of cicd --- .gitlab-ci.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 769978e2c..c482a903d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,6 +103,18 @@ deploy:spot.eeo.one.backend1: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - docker-compose up -d --build - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.eeo.one.backend2: extends: .deploy:template @@ -131,6 +143,18 @@ deploy:spot.eeo.one.backend2: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - docker-compose up -d --build - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i '/proxy01.ecloud.global/proxy02.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.ecloud.global.backend1: @@ -160,6 +184,18 @@ deploy:spot.ecloud.global.backend1: - docker-compose pull - docker-compose up -d - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.ecloud.global.backend2: extends: .deploy:template @@ -188,4 +224,16 @@ deploy:spot.ecloud.global.backend2: - docker-compose pull - docker-compose up -d - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" -- GitLab From 56e15fd6da9e5f4b6e26c3c8bc9a0b6dcfec00fa Mon Sep 17 00:00:00 2001 From: diroots Date: Thu, 24 Jun 2021 13:35:37 +0200 Subject: [PATCH 12/20] oups proxy diff typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c482a903d..eb41d72c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -230,7 +230,7 @@ deploy:spot.ecloud.global.backend2: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env - && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i '/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env -- GitLab From 5249ec4f84f3f956c0a7bcadbf3d4a437aa8986a Mon Sep 17 00:00:00 2001 From: diroots Date: Fri, 25 Jun 2021 09:39:45 +0200 Subject: [PATCH 13/20] error in one sed command --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb41d72c5..9b522e903 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -109,7 +109,7 @@ deploy:spot.eeo.one.backend1: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env - && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env @@ -149,7 +149,7 @@ deploy:spot.eeo.one.backend2: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env - && sed -i '/proxy01.ecloud.global/proxy02.ecloud.global/g' .env + && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env @@ -190,7 +190,7 @@ deploy:spot.ecloud.global.backend1: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env - && sed -i '/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env @@ -230,7 +230,7 @@ deploy:spot.ecloud.global.backend2: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env - && sed -i '/proxy01.ecloud.global/proxy02.ecloud.global/g' .env + && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env -- GitLab From 960bc35fc07bd90077b4efbb917c040f0704b45e Mon Sep 17 00:00:00 2001 From: diroots Date: Fri, 25 Jun 2021 10:20:52 +0200 Subject: [PATCH 14/20] retry sed without the replacement containing a / --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b522e903..fd716d927 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,6 @@ deploy:spot.eeo.one.backend1: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -151,7 +150,6 @@ deploy:spot.eeo.one.backend2: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" -- GitLab From 50591328b0d0b203098fcc1401360ef587396236 Mon Sep 17 00:00:00 2001 From: diroots Date: Fri, 25 Jun 2021 10:45:06 +0200 Subject: [PATCH 15/20] generic vars in env for ci sed substitutions --- .env | 4 ++-- .gitlab-ci.yml | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.env b/.env index 68e58b8d4..e8016675b 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ SEARX_UI_DEFAULT_THEME=eelo SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 -REPO_BASE=/mnt/repo-base/staging-spot +REPO_BASE=/mnt/repo-base/PATH FILTRON_PORT=8088 MORTY_PORT=8089 -INTERNAL_NETWORK_NAME=staging-spot-default \ No newline at end of file +INTERNAL_NETWORK_NAME=default \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd716d927..e552c3b8c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,9 +111,10 @@ deploy:spot.eeo.one.backend1: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/PATH/staging-spot/g' && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env - && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.eeo.one.backend2: extends: .deploy:template @@ -142,7 +143,7 @@ deploy:spot.eeo.one.backend2: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - docker-compose up -d --build - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env @@ -150,9 +151,10 @@ deploy:spot.eeo.one.backend2: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/PATH/staging-spot/g' && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env - && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.ecloud.global.backend1: @@ -190,10 +192,10 @@ deploy:spot.ecloud.global.backend1: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/PATH/production-spot/g' && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env - && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" deploy:spot.ecloud.global.backend2: extends: .deploy:template @@ -222,7 +224,7 @@ deploy:spot.ecloud.global.backend2: - docker-compose pull - docker-compose up -d - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env @@ -230,8 +232,8 @@ deploy:spot.ecloud.global.backend2: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/\/mnt\/repo-base\/staging-spot/$REPO_BASE/g' .env + && sed -i 's/PATH/production-spot/g' && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env - && sed -i 's/INTERNAL_NETWORK_NAME=staging-spot-default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" -- GitLab From 4f13a97fa37cd5d76649c0a4ac288a2b6e38f111 Mon Sep 17 00:00:00 2001 From: diroots Date: Fri, 25 Jun 2021 10:58:14 +0200 Subject: [PATCH 16/20] grrr, missed a filename in the last sed added, dummy! --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e552c3b8c..8d7d80b22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -111,7 +111,7 @@ deploy:spot.eeo.one.backend1: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/staging-spot/g' + && sed -i 's/PATH/staging-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -151,7 +151,7 @@ deploy:spot.eeo.one.backend2: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/staging-spot/g' + && sed -i 's/PATH/staging-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -192,7 +192,7 @@ deploy:spot.ecloud.global.backend1: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/production-spot/g' + && sed -i 's/PATH/production-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -232,7 +232,7 @@ deploy:spot.ecloud.global.backend2: && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/production-spot/g' + && sed -i 's/PATH/production-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" -- GitLab From 72eaab630bdcc5d02e5a7cbb0285a1470f0ec72e Mon Sep 17 00:00:00 2001 From: diroots Date: Fri, 25 Jun 2021 12:16:01 +0200 Subject: [PATCH 17/20] updates for repo to handle /e/ specific production setup AND public use --- .env | 10 ++--- .env.e | 16 ++++++++ .gitlab-ci.yml | 4 ++ README.md | 7 ++++ docker-compose.e.yml | 87 ++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 28 ++++---------- 6 files changed, 125 insertions(+), 27 deletions(-) create mode 100644 .env.e create mode 100644 docker-compose.e.yml diff --git a/.env b/.env index e8016675b..9bc260d89 100644 --- a/.env +++ b/.env @@ -7,10 +7,8 @@ SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo -SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 -SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 -WIREGUARD_IP=127.0.0.1 -REPO_BASE=/mnt/repo-base/PATH +SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 +SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 +REPO_BASE=/PATH/TO/SPOT/FOLDER FILTRON_PORT=8088 -MORTY_PORT=8089 -INTERNAL_NETWORK_NAME=default \ No newline at end of file +MORTY_PORT=8089 \ No newline at end of file diff --git a/.env.e b/.env.e new file mode 100644 index 000000000..e8016675b --- /dev/null +++ b/.env.e @@ -0,0 +1,16 @@ +SPOT_HOSTNAME=spot.ecloud.global +SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global +SPOT_DOCKER_TAG=latest +SPOT_NGINX_DOCKER_TAG=latest +SEARX_MORTY_URL=https://localhost:8089 +SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" +SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" +SEARX_REDIS_HOST=redis +SEARX_UI_DEFAULT_THEME=eelo +SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 +SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 +WIREGUARD_IP=127.0.0.1 +REPO_BASE=/mnt/repo-base/PATH +FILTRON_PORT=8088 +MORTY_PORT=8089 +INTERNAL_NETWORK_NAME=default \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d7d80b22..4878d82d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -101,6 +101,7 @@ deploy:spot.eeo.one.backend1: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE @@ -141,6 +142,7 @@ deploy:spot.eeo.one.backend2: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE @@ -181,6 +183,7 @@ deploy:spot.ecloud.global.backend1: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron @@ -221,6 +224,7 @@ deploy:spot.ecloud.global.backend2: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron diff --git a/README.md b/README.md index e408798d6..86a43987b 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,13 @@ below to run spot for production or local environment. ### Like production + +update .env file with the path where spot is cloned : + +``` +REPO_BASE=/PATH/TO/SPOT/FOLDER +``` + Run the docker-compose to start the project ``` diff --git a/docker-compose.e.yml b/docker-compose.e.yml new file mode 100644 index 000000000..e508245f1 --- /dev/null +++ b/docker-compose.e.yml @@ -0,0 +1,87 @@ +version: '3.6' + +x-logging: + &default-logging + options: + max-size: '100m' + max-file: '3' + driver: json-file + +services: + redis: + image: redis:5.0.7-alpine + # container_name: spot-redis + logging: *default-logging + restart: unless-stopped + networks: + - ${INTERNAL_NETWORK_NAME} + command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru + + spot: + image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG} + build: + context: . + dockerfile: Dockerfile + logging: *default-logging + restart: unless-stopped + networks: + - ${INTERNAL_NETWORK_NAME} + environment: + SEARX_SECRET: "${SEARX_SECRET}" + SEARX_MORTY_URL: "${SEARX_MORTY_URL}" + SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}" + SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}" + SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}" + SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}" + SEARX_UI_DEFAULT_THEME: "${SEARX_UI_DEFAULT_THEME}" + GUNICORN_LOGGER: 1 + GUNICORN_LEVEL: INFO + + spot-nginx: + image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG} + # container_name: spot-nginx + build: + context: . + dockerfile: nginx.Dockerfile + logging: *default-logging + restart: unless-stopped + networks: + - ${INTERNAL_NETWORK_NAME} + environment: + SEARX_MORTY_URL: "${SEARX_MORTY_URL}" + + filtron: + image: dalf/filtron:latest + logging: *default-logging + restart: unless-stopped + command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx + networks: + - ${INTERNAL_NETWORK_NAME} + - spot-wireguarded + ports: + - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 + volumes: + - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json + + morty: + image: dalf/morty:latest + logging: *default-logging + restart: unless-stopped + networks: + - ${INTERNAL_NETWORK_NAME} + - spot-wireguarded + ports: + - ${WIREGUARD_IP}:${MORTY_PORT}:3000 + environment: + DEBUG: "false" + MORTY_ADDRESS: ":3000" + MORTY_KEY: "${SEARX_MORTY_KEY}" + + +networks: + staging-spot-default: + external: true + spot-default: + external: true + spot-wireguarded: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index e508245f1..bbb840c7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,6 @@ services: # container_name: spot-redis logging: *default-logging restart: unless-stopped - networks: - - ${INTERNAL_NETWORK_NAME} command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru spot: @@ -24,8 +22,6 @@ services: dockerfile: Dockerfile logging: *default-logging restart: unless-stopped - networks: - - ${INTERNAL_NETWORK_NAME} environment: SEARX_SECRET: "${SEARX_SECRET}" SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -45,8 +41,6 @@ services: dockerfile: nginx.Dockerfile logging: *default-logging restart: unless-stopped - networks: - - ${INTERNAL_NETWORK_NAME} environment: SEARX_MORTY_URL: "${SEARX_MORTY_URL}" @@ -55,11 +49,8 @@ services: logging: *default-logging restart: unless-stopped command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx - networks: - - ${INTERNAL_NETWORK_NAME} - - spot-wireguarded ports: - - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 + - ${FILTRON_PORT}:3000 volumes: - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json @@ -67,21 +58,16 @@ services: image: dalf/morty:latest logging: *default-logging restart: unless-stopped - networks: - - ${INTERNAL_NETWORK_NAME} - - spot-wireguarded ports: - - ${WIREGUARD_IP}:${MORTY_PORT}:3000 + - ${MORTY_PORT}:3000 environment: DEBUG: "false" MORTY_ADDRESS: ":3000" MORTY_KEY: "${SEARX_MORTY_KEY}" + tor-socks-proxy: + image: peterdavehello/tor-socks-proxy:latest + logging: *default-logging + restart: unless-stopped + -networks: - staging-spot-default: - external: true - spot-default: - external: true - spot-wireguarded: - external: true -- GitLab From 72e85bef5e0b7fc646828cebfff69b889a7a690f Mon Sep 17 00:00:00 2001 From: diroots Date: Wed, 30 Jun 2021 13:57:55 +0200 Subject: [PATCH 18/20] modifications from latest reviews --- .env | 1 - .env.e => .env.prod | 5 ++--- .gitlab-ci.yml | 16 ++++++++-------- README.md | 6 ------ docker-compose.e.yml => docker-compose.prod.yml | 12 ++++++------ docker-compose.yml | 4 +--- 6 files changed, 17 insertions(+), 27 deletions(-) rename .env.e => .env.prod (70%) rename docker-compose.e.yml => docker-compose.prod.yml (90%) diff --git a/.env b/.env index 9bc260d89..e26f47e0f 100644 --- a/.env +++ b/.env @@ -9,6 +9,5 @@ SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 -REPO_BASE=/PATH/TO/SPOT/FOLDER FILTRON_PORT=8088 MORTY_PORT=8089 \ No newline at end of file diff --git a/.env.e b/.env.prod similarity index 70% rename from .env.e rename to .env.prod index e8016675b..3c72b24c9 100644 --- a/.env.e +++ b/.env.prod @@ -3,14 +3,13 @@ SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_TAG=latest SEARX_MORTY_URL=https://localhost:8089 -SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" -SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" +SEARX_SECRET="SECRET2BEREPLACED" +SEARX_MORTY_KEY="KEY2BEREPLACED" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 -REPO_BASE=/mnt/repo-base/PATH FILTRON_PORT=8088 MORTY_PORT=8089 INTERNAL_NETWORK_NAME=default \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4878d82d0..7f9114787 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,7 +95,6 @@ deploy:spot.eeo.one.backend1: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} - REPO_BASE: /mnt/repo-base/staging-spot INTERNAL_NETWORK_NAME: staging-spot-default FILTRON_PORT: 8088 MORTY_PORT: 8089 @@ -110,9 +109,10 @@ deploy:spot.eeo.one.backend1: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/staging-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -136,7 +136,6 @@ deploy:spot.eeo.one.backend2: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} - REPO_BASE: /mnt/repo-base/staging-spot INTERNAL_NETWORK_NAME: staging-spot-default FILTRON_PORT: 8088 MORTY_PORT: 8089 @@ -151,9 +150,10 @@ deploy:spot.eeo.one.backend2: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/staging-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -177,7 +177,6 @@ deploy:spot.ecloud.global.backend1: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} - REPO_BASE: /mnt/repo-base/production-spot INTERNAL_NETWORK_NAME: spot-default FILTRON_PORT: 8098 MORTY_PORT: 8099 @@ -193,9 +192,10 @@ deploy:spot.ecloud.global.backend1: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/production-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" @@ -218,7 +218,6 @@ deploy:spot.ecloud.global.backend2: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} - REPO_BASE: /mnt/repo-base/production-spot INTERNAL_NETWORK_NAME: spot-default FILTRON_PORT: 8098 MORTY_PORT: 8099 @@ -234,9 +233,10 @@ deploy:spot.ecloud.global.backend2: && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env - && sed -i 's/PATH/production-spot/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" diff --git a/README.md b/README.md index 86a43987b..a3e0aeacb 100644 --- a/README.md +++ b/README.md @@ -47,12 +47,6 @@ below to run spot for production or local environment. ### Like production -update .env file with the path where spot is cloned : - -``` -REPO_BASE=/PATH/TO/SPOT/FOLDER -``` - Run the docker-compose to start the project ``` diff --git a/docker-compose.e.yml b/docker-compose.prod.yml similarity index 90% rename from docker-compose.e.yml rename to docker-compose.prod.yml index e508245f1..45ffd4eb8 100644 --- a/docker-compose.e.yml +++ b/docker-compose.prod.yml @@ -12,7 +12,7 @@ services: image: redis:5.0.7-alpine # container_name: spot-redis logging: *default-logging - restart: unless-stopped + restart: always networks: - ${INTERNAL_NETWORK_NAME} command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru @@ -23,7 +23,7 @@ services: context: . dockerfile: Dockerfile logging: *default-logging - restart: unless-stopped + restart: always networks: - ${INTERNAL_NETWORK_NAME} environment: @@ -44,7 +44,7 @@ services: context: . dockerfile: nginx.Dockerfile logging: *default-logging - restart: unless-stopped + restart: always networks: - ${INTERNAL_NETWORK_NAME} environment: @@ -53,7 +53,7 @@ services: filtron: image: dalf/filtron:latest logging: *default-logging - restart: unless-stopped + restart: always command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx networks: - ${INTERNAL_NETWORK_NAME} @@ -61,12 +61,12 @@ services: ports: - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 volumes: - - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json + - ./etc/filtron/rules.json:/etc/filtron/rules.json morty: image: dalf/morty:latest logging: *default-logging - restart: unless-stopped + restart: always networks: - ${INTERNAL_NETWORK_NAME} - spot-wireguarded diff --git a/docker-compose.yml b/docker-compose.yml index bbb840c7b..c583de8a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ x-logging: services: redis: image: redis:5.0.7-alpine - # container_name: spot-redis logging: *default-logging restart: unless-stopped command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru @@ -35,7 +34,6 @@ services: spot-nginx: image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG} - # container_name: spot-nginx build: context: . dockerfile: nginx.Dockerfile @@ -52,7 +50,7 @@ services: ports: - ${FILTRON_PORT}:3000 volumes: - - ${REPO_BASE}/etc/filtron/rules.json:/etc/filtron/rules.json + - ./etc/filtron/rules.json:/etc/filtron/rules.json morty: image: dalf/morty:latest -- GitLab From 93cd0bcc5bebef9f5fda0ffdf06a743657a9a719 Mon Sep 17 00:00:00 2001 From: diroots Date: Wed, 30 Jun 2021 14:13:15 +0200 Subject: [PATCH 19/20] $REPO_BASE exists no more in env vars. --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f9114787..f3e6c4ca9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,10 +100,10 @@ deploy:spot.eeo.one.backend1: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env @@ -141,10 +141,10 @@ deploy:spot.eeo.one.backend2: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env @@ -182,11 +182,11 @@ deploy:spot.ecloud.global.backend1: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND1_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env @@ -223,11 +223,11 @@ deploy:spot.ecloud.global.backend2: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron - - ssh $SSH_USER@$BACKEND2_HOST "cd $REPO_BASE + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env -- GitLab From e5d38ef707c8674e81fa37dc15d9fd0fcaa3cf6e Mon Sep 17 00:00:00 2001 From: diroots Date: Wed, 30 Jun 2021 14:24:05 +0200 Subject: [PATCH 20/20] .e. files renamed to .prod. files --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3e6c4ca9..37a070a24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,7 +100,7 @@ deploy:spot.eeo.one.backend1: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot @@ -141,7 +141,7 @@ deploy:spot.eeo.one.backend2: MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - docker-compose up -d --build - docker-compose restart filtron - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot @@ -182,7 +182,7 @@ deploy:spot.ecloud.global.backend1: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron @@ -223,7 +223,7 @@ deploy:spot.ecloud.global.backend2: MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.e .env && rm docker-compose.yml && mv docker-compose.e.yml docker-compose.yml" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - docker-compose pull - docker-compose up -d - docker-compose restart filtron -- GitLab