Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit dfc5b6c3 authored by Florent VINCENT's avatar Florent VINCENT 👾
Browse files

Merge branch 'spot-migration' into 'master' 

Spot migration

See merge request e/cloud/my-spot!63
parents 65f7430f e5d38ef7

.env

+5 −3
Original line number Diff line number Diff line
@@ -2,10 +2,12 @@ SPOT_HOSTNAME=spot.ecloud.global 
SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global

SPOT_DOCKER_TAG=latest

SPOT_NGINX_DOCKER_TAG=latest

SEARX_MORTY_URL=http://localhost:8089

SEARX_MORTY_URL=https://localhost:8089

SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj"

SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ="

SEARX_REDIS_HOST=redis

SEARX_UI_DEFAULT_THEME=eelo

SEARX_PROXY_HTTP=socks5://tor:9050

SEARX_PROXY_HTTPS=socks5://tor:9050
 
SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150

SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150

FILTRON_PORT=8088

MORTY_PORT=8089
 
 No newline at end of file

.env.prod

0 → 100644
+15 −0
Original line number Diff line number Diff line 
SPOT_HOSTNAME=spot.ecloud.global

SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global

SPOT_DOCKER_TAG=latest

SPOT_NGINX_DOCKER_TAG=latest

SEARX_MORTY_URL=https://localhost:8089

SEARX_SECRET="SECRET2BEREPLACED"

SEARX_MORTY_KEY="KEY2BEREPLACED"

SEARX_REDIS_HOST=redis

SEARX_UI_DEFAULT_THEME=eelo

SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099

SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099

WIREGUARD_IP=127.0.0.1 

FILTRON_PORT=8088

MORTY_PORT=8089

INTERNAL_NETWORK_NAME=default
 
 No newline at end of file

.gitlab-ci.yml

+151 −25
Original line number Diff line number Diff line
@@ -6,6 +6,9 @@ stages: 
 - test

 - deploy



services:  

  - docker:20.10-dind 



python:

  stage: check

  before_script:
@@ -28,8 +31,6 @@ build:web: 
  image: docker:git

  tags:

    - generic_privileged

  services:

    - docker:18-dind

  before_script:

    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY

  script:
@@ -64,54 +65,179 @@ test:unit: 
.deploy:template:

  stage: deploy

  before_script:

    - eval $(ssh-agent -s)

    - echo "$PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

    - mkdir -p ~/.ssh

    - chmod 700 ~/.ssh

    - echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts

    - chmod 644 ~/.ssh/known_hosts

    - ssh $DOCKER_HOST "mkdir -p $(dirname $FILTRON_RULES) && echo '$(cat ./etc/filtron/rules.json)' > $FILTRON_RULES"

    - echo "$SSH_PRIVATE_KEY_ED" > $HOME/.ssh/id_ed25519

    - echo "$SSH_PUBKEY_ED" > $HOME/.ssh/id_ed25519.pub

    - echo "$SSH_KNOWN_HOSTS" > $HOME/.ssh/known_hosts

    - chmod 600 ~/.ssh/id_ed25519

    - chmod 644 ~/.ssh/known_hosts ~/.ssh/id_ed25519.pub

    - ssh $DOCKER_HOST "cd $PATH_STAGING"

    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY

    - docker pull registry.gitlab.e.foundation:5000/e/cloud/my-spot/env



deploy:spot.test.cloud.global:

deploy:spot.eeo.one.backend1:

  extends: .deploy:template

  when: manual

  only:

    - branches

  environment:

    name: test

    url: https://spot.test.ecloud.global

    name: eeo1

    url: https://spot.eeo.one

  variables:

    DOCKER_HOST: ssh://root@spot.test.ecloud.global

    FILTRON_RULES: /etc/filtron/rules.json

    SPOT_HOSTNAME: spot.test.ecloud.global

    SPOT_MORTY_HOSTNAME: proxy.spot.test.ecloud.global

    SEARX_MORTY_URL: https://proxy.spot.test.ecloud.global

    COMPOSE_PROJECT_NAME: my-spot

    PRIVATE_KEY: ${PRIVATE_KEY_TEST}

    DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST}

    SPOT_HOSTNAME: spot.eeo.one

    SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one

    SEARX_MORTY_URL: https://proxy.spot.eeo.one

    SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099

    SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099

    COMPOSE_PROJECT_NAME: staging-spot

    SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    WIREGUARD_IP: ${BACKEND1_WG_IP}

    INTERNAL_NETWORK_NAME: staging-spot-default

    FILTRON_PORT: 8088

    MORTY_PORT: 8089

  script:

    - docker-compose up -d --build --scale tor=5

    - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" 

    - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"

    - docker-compose up -d --build

    - docker-compose restart filtron

    - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot 

      && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env 

      && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env 

      && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env 

      && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env

      && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env

      && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env

      && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env 

      && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env

      && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env

      && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"



deploy:spot.eeo.one.backend2:

  extends: .deploy:template

  when: manual

  only:

    - branches

  environment:

    name: eeo2

    url: https://spot.eeo.one

  variables:

    DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST}

    SPOT_HOSTNAME: spot.eeo.one

    SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one

    SEARX_MORTY_URL: https://proxy.spot.eeo.one

    SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099

    SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099

    COMPOSE_PROJECT_NAME: staging-spot

    SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    WIREGUARD_IP: ${BACKEND2_WG_IP}

    INTERNAL_NETWORK_NAME: staging-spot-default

    FILTRON_PORT: 8088

    MORTY_PORT: 8089

  script:

    - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" 

    - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"

    - docker-compose up -d --build

    - docker-compose restart filtron

    - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot 

      && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env 

      && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env 

      && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env 

      && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env

      && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env

      && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env

      && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env 

      && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env

      && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env

      && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"





deploy:spot.cloud.global:

deploy:spot.ecloud.global.backend1:

  extends: .deploy:template

  only:

    - tags

  environment:

    name: prod

    name: ecloud1

    url: https://spot.ecloud.global

  variables:

    DOCKER_HOST: ssh://spot@spot.ecloud.global

    FILTRON_RULES: /home/spot/filtron/rules.json

    DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST}

    SPOT_HOSTNAME: spot.ecloud.global

    SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global

    SEARX_MORTY_URL: https://proxy.spot.ecloud.global

    COMPOSE_PROJECT_NAME: my-spot

    SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099

    SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099

    COMPOSE_PROJECT_NAME: production-spot

    SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    PRIVATE_KEY: ${PRIVATE_KEY_PROD}

    WIREGUARD_IP: ${BACKEND1_WG_IP}

    INTERNAL_NETWORK_NAME: spot-default

    FILTRON_PORT: 8098

    MORTY_PORT: 8099

  script:

    - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" 

    - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"

    - docker-compose pull

    - docker-compose up -d --scale tor=5

    - docker-compose up -d

    - docker-compose restart filtron

    - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot 

      && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env 

      && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env 

      && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env 

      && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env

      && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env

      && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env

      && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env 

      && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env

      && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env

      && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"



deploy:spot.ecloud.global.backend2:

  extends: .deploy:template

  only:

    - tags

  environment:

    name: ecloud2

    url: https://spot.ecloud.global

  variables:

    DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST}

    SPOT_HOSTNAME: spot.ecloud.global

    SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global

    SEARX_MORTY_URL: https://proxy.spot.ecloud.global

    SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099

    SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099

    COMPOSE_PROJECT_NAME: production-spot

    SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG}

    WIREGUARD_IP: ${BACKEND2_WG_IP}

    INTERNAL_NETWORK_NAME: spot-default

    FILTRON_PORT: 8098

    MORTY_PORT: 8099

  script:

    - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" 

    - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml"

    - docker-compose pull

    - docker-compose up -d

    - docker-compose restart filtron

    - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot 

      && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env 

      && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env 

      && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env 

      && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env 

      && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env

      && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env

      && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env

      && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env 

      && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env

      && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env

      && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env"

README.md

+1 −0
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ below to run spot for production or local environment. 


### Like production





Run the docker-compose to start the project



```

docker-compose.prod.yml

0 → 100644
+87 −0
Original line number Diff line number Diff line 
version: '3.6'



x-logging:

  &default-logging

  options:

    max-size: '100m'

    max-file: '3'

  driver: json-file



services:

  redis:

    image: redis:5.0.7-alpine

    # container_name: spot-redis

    logging: *default-logging

    restart: always

    networks:

      - ${INTERNAL_NETWORK_NAME}

    command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru



  spot:

    image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG}

    build:

      context: .

      dockerfile: Dockerfile

    logging: *default-logging

    restart: always

    networks:

      - ${INTERNAL_NETWORK_NAME}

    environment:

      SEARX_SECRET: "${SEARX_SECRET}"

      SEARX_MORTY_URL: "${SEARX_MORTY_URL}"

      SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}"

      SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}"

      SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}"

      SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}"

      SEARX_UI_DEFAULT_THEME: "${SEARX_UI_DEFAULT_THEME}"

      GUNICORN_LOGGER: 1

      GUNICORN_LEVEL: INFO



  spot-nginx:

    image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG}

    # container_name: spot-nginx

    build:

      context: .

      dockerfile: nginx.Dockerfile

    logging: *default-logging

    restart: always

    networks:

      - ${INTERNAL_NETWORK_NAME}

    environment:

      SEARX_MORTY_URL: "${SEARX_MORTY_URL}"



  filtron:

    image: dalf/filtron:latest

    logging: *default-logging

    restart: always

    command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx

    networks:

      - ${INTERNAL_NETWORK_NAME}

      - spot-wireguarded

    ports:

      - ${WIREGUARD_IP}:${FILTRON_PORT}:3000

    volumes:

      - ./etc/filtron/rules.json:/etc/filtron/rules.json



  morty:

    image: dalf/morty:latest

    logging: *default-logging

    restart: always

    networks:

      - ${INTERNAL_NETWORK_NAME}

      - spot-wireguarded

    ports:

      - ${WIREGUARD_IP}:${MORTY_PORT}:3000

    environment:

      DEBUG: "false"

      MORTY_ADDRESS: ":3000"

      MORTY_KEY: "${SEARX_MORTY_KEY}"





networks:

  staging-spot-default:

    external: true

  spot-default:

    external: true

  spot-wireguarded:

    external: true