Commit dfc5b6c3 authored by Florent VINCENT's avatar Florent VINCENT
Browse files

Merge branch 'spot-migration' into 'master'

Spot migration

See merge request e/cloud/my-spot!63
parents 65f7430f e5d38ef7
......@@ -2,10 +2,12 @@ SPOT_HOSTNAME=spot.ecloud.global
SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global
SPOT_DOCKER_TAG=latest
SPOT_NGINX_DOCKER_TAG=latest
SEARX_MORTY_URL=http://localhost:8089
SEARX_MORTY_URL=https://localhost:8089
SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj"
SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ="
SEARX_REDIS_HOST=redis
SEARX_UI_DEFAULT_THEME=eelo
SEARX_PROXY_HTTP=socks5://tor:9050
SEARX_PROXY_HTTPS=socks5://tor:9050
SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150
SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150
FILTRON_PORT=8088
MORTY_PORT=8089
\ No newline at end of file
SPOT_HOSTNAME=spot.ecloud.global
SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global
SPOT_DOCKER_TAG=latest
SPOT_NGINX_DOCKER_TAG=latest
SEARX_MORTY_URL=https://localhost:8089
SEARX_SECRET="SECRET2BEREPLACED"
SEARX_MORTY_KEY="KEY2BEREPLACED"
SEARX_REDIS_HOST=redis
SEARX_UI_DEFAULT_THEME=eelo
SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099
SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099
WIREGUARD_IP=127.0.0.1
FILTRON_PORT=8088
MORTY_PORT=8089
INTERNAL_NETWORK_NAME=default
\ No newline at end of file
......@@ -6,6 +6,9 @@ stages:
- test
- deploy
services:
- docker:20.10-dind
python:
stage: check
before_script:
......@@ -28,8 +31,6 @@ build:web:
image: docker:git
tags:
- generic_privileged
services:
- docker:18-dind
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
script:
......@@ -64,54 +65,179 @@ test:unit:
.deploy:template:
stage: deploy
before_script:
- eval $(ssh-agent -s)
- echo "$PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- ssh $DOCKER_HOST "mkdir -p $(dirname $FILTRON_RULES) && echo '$(cat ./etc/filtron/rules.json)' > $FILTRON_RULES"
- echo "$SSH_PRIVATE_KEY_ED" > $HOME/.ssh/id_ed25519
- echo "$SSH_PUBKEY_ED" > $HOME/.ssh/id_ed25519.pub
- echo "$SSH_KNOWN_HOSTS" > $HOME/.ssh/known_hosts
- chmod 600 ~/.ssh/id_ed25519
- chmod 644 ~/.ssh/known_hosts ~/.ssh/id_ed25519.pub
- ssh $DOCKER_HOST "cd $PATH_STAGING"
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull registry.gitlab.e.foundation:5000/e/cloud/my-spot/env
deploy:spot.test.cloud.global:
deploy:spot.eeo.one.backend1:
extends: .deploy:template
when: manual
only:
- branches
environment:
name: eeo1
url: https://spot.eeo.one
variables:
DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST}
SPOT_HOSTNAME: spot.eeo.one
SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one
SEARX_MORTY_URL: https://proxy.spot.eeo.one
SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099
SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099
COMPOSE_PROJECT_NAME: staging-spot
SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
WIREGUARD_IP: ${BACKEND1_WG_IP}
INTERNAL_NETWORK_NAME: staging-spot-default
FILTRON_PORT: 8088
MORTY_PORT: 8089
script:
- echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)"
- ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"
- docker-compose up -d --build
- docker-compose restart filtron
- ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot
&& sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env
&& sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env
&& sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env
&& sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env
&& sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env
&& sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env
&& sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env
&& sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env
&& sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env
&& sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"
deploy:spot.eeo.one.backend2:
extends: .deploy:template
when: manual
only:
- branches
environment:
name: test
url: https://spot.test.ecloud.global
name: eeo2
url: https://spot.eeo.one
variables:
DOCKER_HOST: ssh://root@spot.test.ecloud.global
FILTRON_RULES: /etc/filtron/rules.json
SPOT_HOSTNAME: spot.test.ecloud.global
SPOT_MORTY_HOSTNAME: proxy.spot.test.ecloud.global
SEARX_MORTY_URL: https://proxy.spot.test.ecloud.global
COMPOSE_PROJECT_NAME: my-spot
PRIVATE_KEY: ${PRIVATE_KEY_TEST}
DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST}
SPOT_HOSTNAME: spot.eeo.one
SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one
SEARX_MORTY_URL: https://proxy.spot.eeo.one
SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099
SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099
COMPOSE_PROJECT_NAME: staging-spot
SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
WIREGUARD_IP: ${BACKEND2_WG_IP}
INTERNAL_NETWORK_NAME: staging-spot-default
FILTRON_PORT: 8088
MORTY_PORT: 8089
script:
- docker-compose up -d --build --scale tor=5
- echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)"
- ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"
- docker-compose up -d --build
- docker-compose restart filtron
- ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot
&& sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env
&& sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env
&& sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env
&& sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env
&& sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env
&& sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env
&& sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env
&& sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env
&& sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env
&& sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"
deploy:spot.cloud.global:
deploy:spot.ecloud.global.backend1:
extends: .deploy:template
only:
- tags
environment:
name: prod
name: ecloud1
url: https://spot.ecloud.global
variables:
DOCKER_HOST: ssh://spot@spot.ecloud.global
FILTRON_RULES: /home/spot/filtron/rules.json
DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST}
SPOT_HOSTNAME: spot.ecloud.global
SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global
SEARX_MORTY_URL: https://proxy.spot.ecloud.global
COMPOSE_PROJECT_NAME: my-spot
SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099
SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099
COMPOSE_PROJECT_NAME: production-spot
SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
PRIVATE_KEY: ${PRIVATE_KEY_PROD}
WIREGUARD_IP: ${BACKEND1_WG_IP}
INTERNAL_NETWORK_NAME: spot-default
FILTRON_PORT: 8098
MORTY_PORT: 8099
script:
- echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)"
- ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"
- docker-compose pull
- docker-compose up -d --scale tor=5
- docker-compose up -d
- docker-compose restart filtron
- ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot
&& sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env
&& sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env
&& sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env
&& sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env
&& sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env
&& sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env
&& sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env
&& sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env
&& sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env
&& sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"
deploy:spot.ecloud.global.backend2:
extends: .deploy:template
only:
- tags
environment:
name: ecloud2
url: https://spot.ecloud.global
variables:
DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST}
SPOT_HOSTNAME: spot.ecloud.global
SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global
SEARX_MORTY_URL: https://proxy.spot.ecloud.global
SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099
SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099
COMPOSE_PROJECT_NAME: production-spot
SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
WIREGUARD_IP: ${BACKEND2_WG_IP}
INTERNAL_NETWORK_NAME: spot-default
FILTRON_PORT: 8098
MORTY_PORT: 8099
script:
- echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)"
- ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"
- docker-compose pull
- docker-compose up -d
- docker-compose restart filtron
- ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot
&& sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env
&& sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env
&& sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env
&& sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env
&& sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env
&& sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env
&& sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env
&& sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env
&& sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env
&& sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env
&& sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"
......@@ -46,6 +46,7 @@ below to run spot for production or local environment.
### Like production
Run the docker-compose to start the project
```
......
version: '3.6'
x-logging:
&default-logging
options:
max-size: '100m'
max-file: '3'
driver: json-file
services:
redis:
image: redis:5.0.7-alpine
# container_name: spot-redis
logging: *default-logging
restart: always
networks:
- ${INTERNAL_NETWORK_NAME}
command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru
spot:
image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG}
build:
context: .
dockerfile: Dockerfile
logging: *default-logging
restart: always
networks:
- ${INTERNAL_NETWORK_NAME}
environment:
SEARX_SECRET: "${SEARX_SECRET}"
SEARX_MORTY_URL: "${SEARX_MORTY_URL}"
SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}"
SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}"
SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}"
SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}"
SEARX_UI_DEFAULT_THEME: "${SEARX_UI_DEFAULT_THEME}"
GUNICORN_LOGGER: 1
GUNICORN_LEVEL: INFO
spot-nginx:
image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG}
# container_name: spot-nginx
build:
context: .
dockerfile: nginx.Dockerfile
logging: *default-logging
restart: always
networks:
- ${INTERNAL_NETWORK_NAME}
environment:
SEARX_MORTY_URL: "${SEARX_MORTY_URL}"
filtron:
image: dalf/filtron:latest
logging: *default-logging
restart: always
command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx
networks:
- ${INTERNAL_NETWORK_NAME}
- spot-wireguarded
ports:
- ${WIREGUARD_IP}:${FILTRON_PORT}:3000
volumes:
- ./etc/filtron/rules.json:/etc/filtron/rules.json
morty:
image: dalf/morty:latest
logging: *default-logging
restart: always
networks:
- ${INTERNAL_NETWORK_NAME}
- spot-wireguarded
ports:
- ${WIREGUARD_IP}:${MORTY_PORT}:3000
environment:
DEBUG: "false"
MORTY_ADDRESS: ":3000"
MORTY_KEY: "${SEARX_MORTY_KEY}"
networks:
staging-spot-default:
external: true
spot-default:
external: true
spot-wireguarded:
external: true
......@@ -32,7 +32,7 @@ services:
GUNICORN_LOGGER: 1
GUNICORN_LEVEL: INFO
nginx:
spot-nginx:
image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG}
build:
context: .
......@@ -46,30 +46,26 @@ services:
image: dalf/filtron:latest
logging: *default-logging
restart: unless-stopped
command: -listen :3000 -rules /etc/filtron/rules.json -target nginx
command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx
ports:
- "8088:3000"
- ${FILTRON_PORT}:3000
volumes:
- ${FILTRON_RULES:-./etc/filtron/rules.json}:/etc/filtron/rules.json
labels:
- "traefik.enable=true"
- "traefik.http.routers.filtron.rule=Host(`${SPOT_HOSTNAME}`)"
- ./etc/filtron/rules.json:/etc/filtron/rules.json
morty:
image: dalf/morty:latest
logging: *default-logging
restart: unless-stopped
ports:
- "8089:3000"
- ${MORTY_PORT}:3000
environment:
DEBUG: "false"
MORTY_ADDRESS: ":3000"
MORTY_KEY: "${SEARX_MORTY_KEY}"
labels:
- "traefik.enable=true"
- "traefik.http.routers.morty.rule=Host(`${SPOT_MORTY_HOSTNAME}`)"
tor:
image: osminogin/tor-simple
tor-socks-proxy:
image: peterdavehello/tor-socks-proxy:latest
logging: *default-logging
restart: unless-stopped
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment