diff --git a/.env b/.env index 61ecbf8c16176597deffa1ae9126627ae16f731d..e26f47e0f933681a99c40940c0b743c26998ac1c 100644 --- a/.env +++ b/.env @@ -2,10 +2,12 @@ SPOT_HOSTNAME=spot.ecloud.global SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_TAG=latest -SEARX_MORTY_URL=http://localhost:8089 +SEARX_MORTY_URL=https://localhost:8089 SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=eelo -SEARX_PROXY_HTTP=socks5://tor:9050 -SEARX_PROXY_HTTPS=socks5://tor:9050 +SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 +SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 +FILTRON_PORT=8088 +MORTY_PORT=8089 \ No newline at end of file diff --git a/.env.prod b/.env.prod new file mode 100644 index 0000000000000000000000000000000000000000..3c72b24c92873cf14ac7d11dc823c55f1310702d --- /dev/null +++ b/.env.prod @@ -0,0 +1,15 @@ +SPOT_HOSTNAME=spot.ecloud.global +SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global +SPOT_DOCKER_TAG=latest +SPOT_NGINX_DOCKER_TAG=latest +SEARX_MORTY_URL=https://localhost:8089 +SEARX_SECRET="SECRET2BEREPLACED" +SEARX_MORTY_KEY="KEY2BEREPLACED" +SEARX_REDIS_HOST=redis +SEARX_UI_DEFAULT_THEME=eelo +SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 +SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 +WIREGUARD_IP=127.0.0.1 +FILTRON_PORT=8088 +MORTY_PORT=8089 +INTERNAL_NETWORK_NAME=default \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a1d861dc2af9ec912297abb5ba759c3ef3c49a2..37a070a242cc06b34d5462ba26dd4de5b9b2733b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,9 @@ stages: - test - deploy +services: + - docker:20.10-dind + python: stage: check before_script: @@ -28,8 +31,6 @@ build:web: image: docker:git tags: - generic_privileged - services: - - docker:18-dind before_script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY script: @@ -64,54 +65,179 @@ test:unit: .deploy:template: stage: deploy before_script: - - eval $(ssh-agent -s) - - echo "$PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh - chmod 700 ~/.ssh - - echo "$KNOWN_HOSTS" > ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - ssh $DOCKER_HOST "mkdir -p $(dirname $FILTRON_RULES) && echo '$(cat ./etc/filtron/rules.json)' > $FILTRON_RULES" + - echo "$SSH_PRIVATE_KEY_ED" > $HOME/.ssh/id_ed25519 + - echo "$SSH_PUBKEY_ED" > $HOME/.ssh/id_ed25519.pub + - echo "$SSH_KNOWN_HOSTS" > $HOME/.ssh/known_hosts + - chmod 600 ~/.ssh/id_ed25519 + - chmod 644 ~/.ssh/known_hosts ~/.ssh/id_ed25519.pub + - ssh $DOCKER_HOST "cd $PATH_STAGING" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull registry.gitlab.e.foundation:5000/e/cloud/my-spot/env -deploy:spot.test.cloud.global: +deploy:spot.eeo.one.backend1: + extends: .deploy:template + when: manual + only: + - branches + environment: + name: eeo1 + url: https://spot.eeo.one + variables: + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} + SPOT_HOSTNAME: spot.eeo.one + SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one + SEARX_MORTY_URL: https://proxy.spot.eeo.one + SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 + COMPOSE_PROJECT_NAME: staging-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND1_WG_IP} + INTERNAL_NETWORK_NAME: staging-spot-default + FILTRON_PORT: 8088 + MORTY_PORT: 8089 + script: + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" + - docker-compose up -d --build + - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env + && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + +deploy:spot.eeo.one.backend2: extends: .deploy:template when: manual only: - branches environment: - name: test - url: https://spot.test.ecloud.global + name: eeo2 + url: https://spot.eeo.one variables: - DOCKER_HOST: ssh://root@spot.test.ecloud.global - FILTRON_RULES: /etc/filtron/rules.json - SPOT_HOSTNAME: spot.test.ecloud.global - SPOT_MORTY_HOSTNAME: proxy.spot.test.ecloud.global - SEARX_MORTY_URL: https://proxy.spot.test.ecloud.global - COMPOSE_PROJECT_NAME: my-spot - PRIVATE_KEY: ${PRIVATE_KEY_TEST} + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} + SPOT_HOSTNAME: spot.eeo.one + SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one + SEARX_MORTY_URL: https://proxy.spot.eeo.one + SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 + COMPOSE_PROJECT_NAME: staging-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND2_WG_IP} + INTERNAL_NETWORK_NAME: staging-spot-default + FILTRON_PORT: 8088 + MORTY_PORT: 8089 script: - - docker-compose up -d --build --scale tor=5 + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" + - docker-compose up -d --build - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env + && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" -deploy:spot.cloud.global: + +deploy:spot.ecloud.global.backend1: extends: .deploy:template only: - tags environment: - name: prod + name: ecloud1 url: https://spot.ecloud.global variables: - DOCKER_HOST: ssh://spot@spot.ecloud.global - FILTRON_RULES: /home/spot/filtron/rules.json + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} SPOT_HOSTNAME: spot.ecloud.global SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global SEARX_MORTY_URL: https://proxy.spot.ecloud.global - COMPOSE_PROJECT_NAME: my-spot + SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 + COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - PRIVATE_KEY: ${PRIVATE_KEY_PROD} + WIREGUARD_IP: ${BACKEND1_WG_IP} + INTERNAL_NETWORK_NAME: spot-default + FILTRON_PORT: 8098 + MORTY_PORT: 8099 script: + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - docker-compose pull - - docker-compose up -d --scale tor=5 + - docker-compose up -d - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env + && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + +deploy:spot.ecloud.global.backend2: + extends: .deploy:template + only: + - tags + environment: + name: ecloud2 + url: https://spot.ecloud.global + variables: + DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} + SPOT_HOSTNAME: spot.ecloud.global + SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global + SEARX_MORTY_URL: https://proxy.spot.ecloud.global + SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 + SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 + COMPOSE_PROJECT_NAME: production-spot + SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} + WIREGUARD_IP: ${BACKEND2_WG_IP} + INTERNAL_NETWORK_NAME: spot-default + FILTRON_PORT: 8098 + MORTY_PORT: 8099 + script: + - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git stash && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" + - docker-compose pull + - docker-compose up -d + - docker-compose restart filtron + - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot + && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env + && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env + && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env + && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env + && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env + && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env + && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env + && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env + && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env + && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env" + diff --git a/README.md b/README.md index e408798d6f61972f68a457d097e510073eea3d3b..a3e0aeacb5ef1150e6fb79d88f60c3ee37626de0 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ below to run spot for production or local environment. ### Like production + Run the docker-compose to start the project ``` diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000000000000000000000000000000000000..45ffd4eb82c8f125579e993dc1c06281acd1d9be --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,87 @@ +version: '3.6' + +x-logging: + &default-logging + options: + max-size: '100m' + max-file: '3' + driver: json-file + +services: + redis: + image: redis:5.0.7-alpine + # container_name: spot-redis + logging: *default-logging + restart: always + networks: + - ${INTERNAL_NETWORK_NAME} + command: redis-server --maxmemory 8G --maxmemory-policy allkeys-lru + + spot: + image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG} + build: + context: . + dockerfile: Dockerfile + logging: *default-logging + restart: always + networks: + - ${INTERNAL_NETWORK_NAME} + environment: + SEARX_SECRET: "${SEARX_SECRET}" + SEARX_MORTY_URL: "${SEARX_MORTY_URL}" + SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}" + SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}" + SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}" + SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}" + SEARX_UI_DEFAULT_THEME: "${SEARX_UI_DEFAULT_THEME}" + GUNICORN_LOGGER: 1 + GUNICORN_LEVEL: INFO + + spot-nginx: + image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG} + # container_name: spot-nginx + build: + context: . + dockerfile: nginx.Dockerfile + logging: *default-logging + restart: always + networks: + - ${INTERNAL_NETWORK_NAME} + environment: + SEARX_MORTY_URL: "${SEARX_MORTY_URL}" + + filtron: + image: dalf/filtron:latest + logging: *default-logging + restart: always + command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx + networks: + - ${INTERNAL_NETWORK_NAME} + - spot-wireguarded + ports: + - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 + volumes: + - ./etc/filtron/rules.json:/etc/filtron/rules.json + + morty: + image: dalf/morty:latest + logging: *default-logging + restart: always + networks: + - ${INTERNAL_NETWORK_NAME} + - spot-wireguarded + ports: + - ${WIREGUARD_IP}:${MORTY_PORT}:3000 + environment: + DEBUG: "false" + MORTY_ADDRESS: ":3000" + MORTY_KEY: "${SEARX_MORTY_KEY}" + + +networks: + staging-spot-default: + external: true + spot-default: + external: true + spot-wireguarded: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 5630cabfc96ca825671fd3935edf6f4024801a46..c583de8a2fd4f811cbfc9b8d7f21f0511c0c55ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: GUNICORN_LOGGER: 1 GUNICORN_LEVEL: INFO - nginx: + spot-nginx: image: registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx:${SPOT_NGINX_DOCKER_TAG} build: context: . @@ -46,30 +46,26 @@ services: image: dalf/filtron:latest logging: *default-logging restart: unless-stopped - command: -listen :3000 -rules /etc/filtron/rules.json -target nginx + command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx ports: - - "8088:3000" + - ${FILTRON_PORT}:3000 volumes: - - ${FILTRON_RULES:-./etc/filtron/rules.json}:/etc/filtron/rules.json - labels: - - "traefik.enable=true" - - "traefik.http.routers.filtron.rule=Host(`${SPOT_HOSTNAME}`)" + - ./etc/filtron/rules.json:/etc/filtron/rules.json morty: image: dalf/morty:latest logging: *default-logging restart: unless-stopped ports: - - "8089:3000" + - ${MORTY_PORT}:3000 environment: DEBUG: "false" MORTY_ADDRESS: ":3000" MORTY_KEY: "${SEARX_MORTY_KEY}" - labels: - - "traefik.enable=true" - - "traefik.http.routers.morty.rule=Host(`${SPOT_MORTY_HOSTNAME}`)" - tor: - image: osminogin/tor-simple + tor-socks-proxy: + image: peterdavehello/tor-socks-proxy:latest logging: *default-logging restart: unless-stopped + +