From 1e27b766cb6a48200819f967c278ac9d81de5cd5 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 15 Apr 2022 16:14:23 +0530 Subject: [PATCH 01/10] remove morty from docker --- .env.prod | 4 ---- .gitlab-ci.yml | 28 ---------------------------- docker-compose.prod.yml | 31 +------------------------------ docker-compose.yml | 28 +--------------------------- 4 files changed, 2 insertions(+), 89 deletions(-) diff --git a/.env.prod b/.env.prod index 969758dc8..66591b5c7 100644 --- a/.env.prod +++ b/.env.prod @@ -1,17 +1,13 @@ SPOT_HOSTNAME=spot.ecloud.global -SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_TAG=latest -SEARX_MORTY_URL=https://localhost:8089 SEARX_SECRET="SECRET2BEREPLACED" -SEARX_MORTY_KEY="KEY2BEREPLACED" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=etheme SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 WIREGUARD_IP=127.0.0.1 FILTRON_PORT=8088 -MORTY_PORT=8089 INTERNAL_NETWORK_NAME=default NODE_NAME=spot11 CI_REGISTRY_IMAGE=image diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2852d4abd..772cbf34f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,8 +85,6 @@ deploy:spot.eeo.one.backend1: variables: DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} SPOT_HOSTNAME: spot.eeo.one - SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one - SEARX_MORTY_URL: https://proxy.spot.eeo.one SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot @@ -96,22 +94,17 @@ deploy:spot.eeo.one.backend1: NODE_NAME: ${NODE_NAME} INTERNAL_NETWORK_NAME: staging-spot-default FILTRON_PORT: 8088 - MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && git reset --hard && git clean -xfdf && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env - && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env - && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env - && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env - && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env && sed -i 's/NODE_NAME=spot11/NODE_NAME=$NODE_NAME/g' .env && sed -i 's|CI_REGISTRY_IMAGE=image|CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE|g' .env @@ -128,8 +121,6 @@ deploy:spot.eeo.one.backend2: variables: DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} SPOT_HOSTNAME: spot.eeo.one - SPOT_MORTY_HOSTNAME: proxy.spot.eeo.one - SEARX_MORTY_URL: https://proxy.spot.eeo.one SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot @@ -139,22 +130,17 @@ deploy:spot.eeo.one.backend2: INTERNAL_NETWORK_NAME: staging-spot-default NODE_NAME: ${NODE_NAME} FILTRON_PORT: 8088 - MORTY_PORT: 8089 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && git reset --hard && git clean -xfdf && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env - && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env - && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env - && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env - && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env && sed -i 's/NODE_NAME=spot11/NODE_NAME=$NODE_NAME/g' .env && sed -i 's|CI_REGISTRY_IMAGE=image|CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE|g' .env @@ -170,8 +156,6 @@ deploy:spot.ecloud.global.backend1: variables: DOCKER_HOST: ssh://${SSH_USER}@${BACKEND1_HOST} SPOT_HOSTNAME: spot.ecloud.global - SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global - SEARX_MORTY_URL: https://proxy.spot.ecloud.global SEARX_PROXY_HTTP: http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot @@ -181,22 +165,17 @@ deploy:spot.ecloud.global.backend1: INTERNAL_NETWORK_NAME: spot-default NODE_NAME: ${NODE_NAME} FILTRON_PORT: 8098 - MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND1_HOST - $BACKEND1_WG_IP - $WIREGUARD_IP)" - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && git reset --hard && git clean -xfdf && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env - && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env - && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env - && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env - && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env && sed -i 's/NODE_NAME=spot11/NODE_NAME=$NODE_NAME/g' .env && sed -i 's|CI_REGISTRY_IMAGE=image|CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE|g' .env @@ -212,8 +191,6 @@ deploy:spot.ecloud.global.backend2: variables: DOCKER_HOST: ssh://${SSH_USER}@${BACKEND2_HOST} SPOT_HOSTNAME: spot.ecloud.global - SPOT_MORTY_HOSTNAME: proxy.spot.ecloud.global - SEARX_MORTY_URL: https://proxy.spot.ecloud.global SEARX_PROXY_HTTP: http://proxy02.ecloud.global:1099 SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot @@ -223,22 +200,17 @@ deploy:spot.ecloud.global.backend2: INTERNAL_NETWORK_NAME: spot-default NODE_NAME: ${NODE_NAME} FILTRON_PORT: 8098 - MORTY_PORT: 8099 script: - echo "Deploying to $CI_ENVIRONMENT_NAME ($SSH_USER@$BACKEND2_HOST - $BACKEND2_WG_IP - $WIREGUARD_IP)" - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && git reset --hard && git clean -xfdf && git pull && rm .env && mv .env.prod .env && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env - && sed -i 's/SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global/SPOT_MORTY_HOSTNAME=$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env - && sed -i 's/localhost:8089/$SPOT_MORTY_HOSTNAME/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env - && sed -i 's/KEY2BEREPLACED/$SEARX_MORTY_KEY/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env && sed -i 's/FILTRON_PORT=8088/FILTRON_PORT=$FILTRON_PORT/g' .env - && sed -i 's/MORTY_PORT=8089/MORTY_PORT=$MORTY_PORT/g' .env && sed -i 's/INTERNAL_NETWORK_NAME=default/INTERNAL_NETWORK_NAME=$INTERNAL_NETWORK_NAME/g' .env && sed -i 's/NODE_NAME=spot11/NODE_NAME=$NODE_NAME/g' .env && sed -i 's|CI_REGISTRY_IMAGE=image|CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE|g' .env diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e4d82b3a4..b0b927dfd 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -28,8 +28,6 @@ services: - ${INTERNAL_NETWORK_NAME} environment: SEARX_SECRET: "${SEARX_SECRET}" - SEARX_MORTY_URL: "${SEARX_MORTY_URL}" - SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}" SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}" SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}" SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}" @@ -38,24 +36,11 @@ services: GUNICORN_LEVEL: INFO NODE_NAME: "${NODE_NAME}" - spot-nginx: - image: ${CI_REGISTRY_IMAGE}/nginx:${SPOT_NGINX_DOCKER_TAG} - # container_name: spot-nginx - build: - context: . - dockerfile: nginx.Dockerfile - logging: *default-logging - restart: always - networks: - - ${INTERNAL_NETWORK_NAME} - environment: - SEARX_MORTY_URL: "${SEARX_MORTY_URL}" - filtron: image: dalf/filtron:latest logging: *default-logging restart: always - command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx + command: -listen :3000 -rules /etc/filtron/rules.json -target spot networks: - ${INTERNAL_NETWORK_NAME} - spot-wireguarded @@ -64,20 +49,6 @@ services: volumes: - ./etc/filtron/rules.json:/etc/filtron/rules.json - morty: - image: dalf/morty:latest - logging: *default-logging - restart: always - networks: - - ${INTERNAL_NETWORK_NAME} - - spot-wireguarded - ports: - - ${WIREGUARD_IP}:${MORTY_PORT}:3000 - environment: - DEBUG: "false" - MORTY_ADDRESS: ":3000" - MORTY_KEY: "${SEARX_MORTY_KEY}" - networks: staging-spot-default: diff --git a/docker-compose.yml b/docker-compose.yml index e11fe1793..e65492a6e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,8 +23,6 @@ services: restart: unless-stopped environment: SEARX_SECRET: "${SEARX_SECRET}" - SEARX_MORTY_URL: "${SEARX_MORTY_URL}" - SEARX_MORTY_KEY: "${SEARX_MORTY_KEY}" SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}" SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}" SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}" @@ -33,40 +31,16 @@ services: GUNICORN_LEVEL: INFO NODE_NAME: ${NODE_NAME} - spot-nginx: - image: ${CI_REGISTRY_IMAGE}/nginx:${SPOT_NGINX_DOCKER_TAG} - build: - context: . - dockerfile: nginx.Dockerfile - logging: *default-logging - restart: unless-stopped - environment: - SEARX_MORTY_URL: "${SEARX_MORTY_URL}" - filtron: image: dalf/filtron:latest logging: *default-logging restart: unless-stopped - command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx + command: -listen :3000 -rules /etc/filtron/rules.json -target spot ports: - ${FILTRON_PORT}:3000 volumes: - ./etc/filtron/rules.json:/etc/filtron/rules.json - morty: - image: dalf/morty:latest - logging: *default-logging - restart: unless-stopped - ports: - - ${MORTY_PORT}:3000 - environment: - DEBUG: "false" - MORTY_ADDRESS: ":3000" - MORTY_KEY: "${SEARX_MORTY_KEY}" - tor-socks-proxy: - image: peterdavehello/tor-socks-proxy:latest - logging: *default-logging - restart: unless-stopped -- GitLab From 1d9a81fa59ffefdba1219b2703e4c9f9b77404dd Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 15 Apr 2022 16:55:10 +0530 Subject: [PATCH 02/10] do not remove nginx --- docker-compose.prod.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b0b927dfd..f22c4cf37 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -36,11 +36,22 @@ services: GUNICORN_LEVEL: INFO NODE_NAME: "${NODE_NAME}" + spot-nginx: + image: ${CI_REGISTRY_IMAGE}/nginx:${SPOT_NGINX_DOCKER_TAG} + # container_name: spot-nginx + build: + context: . + dockerfile: nginx.Dockerfile + logging: *default-logging + restart: always + networks: + - ${INTERNAL_NETWORK_NAME} + filtron: image: dalf/filtron:latest logging: *default-logging restart: always - command: -listen :3000 -rules /etc/filtron/rules.json -target spot + command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx networks: - ${INTERNAL_NETWORK_NAME} - spot-wireguarded -- GitLab From 688c79a25334f59667d2b3c72d961d05ba04a591 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 15 Apr 2022 17:07:50 +0530 Subject: [PATCH 03/10] use offical nginx image --- docker-compose.prod.yml | 11 ++++++----- etc/nginx/conf.d/spot.conf | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f22c4cf37..4bbaed51e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -37,15 +37,16 @@ services: NODE_NAME: "${NODE_NAME}" spot-nginx: - image: ${CI_REGISTRY_IMAGE}/nginx:${SPOT_NGINX_DOCKER_TAG} - # container_name: spot-nginx - build: - context: . - dockerfile: nginx.Dockerfile + image: nginx:1.21 + # container_name: spot-nginx logging: *default-logging restart: always networks: - ${INTERNAL_NETWORK_NAME} + volumes: + - ./etc/nginx/conf.d/spot.conf:/etc/nginx/conf.d/default.conf + - ./etc/nginx/proxy_spot_params:/etc/nginx/proxy_spot_params + - ./searx/static:/var/www/spot/static filtron: image: dalf/filtron:latest diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf index a95b5aa6b..76eb7641b 100644 --- a/etc/nginx/conf.d/spot.conf +++ b/etc/nginx/conf.d/spot.conf @@ -15,7 +15,7 @@ server { server_name _; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; - add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' __SEARX_MORTY_URL__ data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; + add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self'; data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; add_header X-Frame-Options "SAMEORIGIN"; root /var/www/spot; -- GitLab From f7aec31c8f6715f0f24702069741f8097a53f81f Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 15 Apr 2022 18:29:33 +0530 Subject: [PATCH 04/10] add nginx.conf as volume --- .env | 4 ---- .env.prod | 1 - .gitlab-ci.yml | 12 ------------ docker-compose.prod.yml | 1 + etc/nginx/nginx.conf | 32 ++++++++++++++++++++++++++++++++ nginx-docker-entrypoint.sh | 8 -------- nginx.Dockerfile | 10 ---------- 7 files changed, 33 insertions(+), 35 deletions(-) create mode 100644 etc/nginx/nginx.conf delete mode 100755 nginx-docker-entrypoint.sh delete mode 100644 nginx.Dockerfile diff --git a/.env b/.env index d61ff1153..125455dd4 100644 --- a/.env +++ b/.env @@ -1,15 +1,11 @@ SPOT_HOSTNAME=spot.ecloud.global -SPOT_MORTY_HOSTNAME=proxy.spot.ecloud.global SPOT_DOCKER_TAG=latest -SPOT_NGINX_DOCKER_TAG=latest SEARX_MORTY_URL=https://localhost:8089 SEARX_SECRET=":@)%NN0+OqNdy:{prWQlZ{p9|oO9p-UyJq@%V!~G:arrSx6fXz.{jd%=XF44ncj" -SEARX_MORTY_KEY="taKB1WGTa63LEI6RdjWWKshS4oYSHQWGu9Eyjr1OlpQ=" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=etheme SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 FILTRON_PORT=8088 -MORTY_PORT=8089 NODE_NAME=local CI_REGISTRY_IMAGE=registry.gitlab.e.foundation/e/infra/spot \ No newline at end of file diff --git a/.env.prod b/.env.prod index 66591b5c7..e78fc9739 100644 --- a/.env.prod +++ b/.env.prod @@ -1,6 +1,5 @@ SPOT_HOSTNAME=spot.ecloud.global SPOT_DOCKER_TAG=latest -SPOT_NGINX_DOCKER_TAG=latest SEARX_SECRET="SECRET2BEREPLACED" SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=etheme diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 772cbf34f..aa6c0b2e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,8 +36,6 @@ build:web: - docker tag $CI_REGISTRY_IMAGE/env base-env-image - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - docker build -t $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG -f nginx.Dockerfile . - - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG build:docker: extends: .build:docker @@ -49,8 +47,6 @@ build:docker:tags: - docker tag $CI_REGISTRY_IMAGE/env base-env-image - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - docker build -t $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG -f nginx.Dockerfile . - - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG only: - tags @@ -89,7 +85,6 @@ deploy:spot.eeo.one.backend1: SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} NODE_NAME: ${NODE_NAME} INTERNAL_NETWORK_NAME: staging-spot-default @@ -100,7 +95,6 @@ deploy:spot.eeo.one.backend1: - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env - && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env @@ -125,7 +119,6 @@ deploy:spot.eeo.one.backend2: SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: staging-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} INTERNAL_NETWORK_NAME: staging-spot-default NODE_NAME: ${NODE_NAME} @@ -136,7 +129,6 @@ deploy:spot.eeo.one.backend2: - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/staging-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env - && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env @@ -160,7 +152,6 @@ deploy:spot.ecloud.global.backend1: SEARX_PROXY_HTTPS: http://proxy01.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND1_WG_IP} INTERNAL_NETWORK_NAME: spot-default NODE_NAME: ${NODE_NAME} @@ -171,7 +162,6 @@ deploy:spot.ecloud.global.backend1: - ssh $SSH_USER@$BACKEND1_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env - && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env && sed -i 's/proxy01.ecloud.global/proxy01.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env @@ -195,7 +185,6 @@ deploy:spot.ecloud.global.backend2: SEARX_PROXY_HTTPS: http://proxy02.ecloud.global:1099 COMPOSE_PROJECT_NAME: production-spot SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} - SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} WIREGUARD_IP: ${BACKEND2_WG_IP} INTERNAL_NETWORK_NAME: spot-default NODE_NAME: ${NODE_NAME} @@ -206,7 +195,6 @@ deploy:spot.ecloud.global.backend2: - ssh $SSH_USER@$BACKEND2_HOST "cd /mnt/repo-base/production-spot && sed -i 's/SPOT_HOSTNAME=spot.ecloud.global/SPOT_HOSTNAME=$SPOT_HOSTNAME/g' .env && sed -i 's/SPOT_DOCKER_TAG=latest/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env - && sed -i 's/SPOT_NGINX_DOCKER_TAG=latest/SPOT_NGINX_DOCKER_TAG=$SPOT_NGINX_DOCKER_TAG/g' .env && sed -i 's/SECRET2BEREPLACED/$SEARX_SECRET/g' .env && sed -i 's/proxy01.ecloud.global/proxy02.ecloud.global/g' .env && sed -i 's/WIREGUARD_IP=127.0.0.1/WIREGUARD_IP=$WIREGUARD_IP/g' .env diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4bbaed51e..768f3721c 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -45,6 +45,7 @@ services: - ${INTERNAL_NETWORK_NAME} volumes: - ./etc/nginx/conf.d/spot.conf:/etc/nginx/conf.d/default.conf + - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf - ./etc/nginx/proxy_spot_params:/etc/nginx/proxy_spot_params - ./searx/static:/var/www/spot/static diff --git a/etc/nginx/nginx.conf b/etc/nginx/nginx.conf new file mode 100644 index 000000000..235b9309d --- /dev/null +++ b/etc/nginx/nginx.conf @@ -0,0 +1,32 @@ + +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$remote_addr"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/nginx-docker-entrypoint.sh b/nginx-docker-entrypoint.sh deleted file mode 100755 index 202524103..000000000 --- a/nginx-docker-entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -e - -if [ -n "$SEARX_MORTY_URL" ]; then - sed -i 's!__SEARX_MORTY_URL__!'$SEARX_MORTY_URL'!g' /etc/nginx/conf.d/default.conf -fi - -exec "$@" diff --git a/nginx.Dockerfile b/nginx.Dockerfile deleted file mode 100644 index 9492e112c..000000000 --- a/nginx.Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM nginx:1.17.6-alpine - -COPY etc/nginx/conf.d/spot.conf /etc/nginx/conf.d/default.conf -COPY etc/nginx/proxy_spot_params /etc/nginx/proxy_spot_params -RUN sed -i 's!http_x_forwarded_for!remote_addr!g' /etc/nginx/nginx.conf -COPY --chown=nginx:nginx searx/static /var/www/spot/static -COPY nginx-docker-entrypoint.sh /docker-entrypoint.sh - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["nginx", "-g", "daemon off;"] -- GitLab From c89d6f113e9832113297fd1643a0e53cdd4a4e48 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 15 Apr 2022 19:50:45 +0530 Subject: [PATCH 05/10] proxify images in thumbnails --- searx/webapp.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/searx/webapp.py b/searx/webapp.py index 4ad907247..efcbcc7a2 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -662,6 +662,12 @@ def search(): if 'url' in result: result['pretty_url'] = prettify_url(result['url']) + if 'thumbnail_src' in result: + result['thumbnail_src'] = image_proxify(result['thumbnail_src']) + + if 'img_src' in result: + result['img_src'] = image_proxify(result['img_src']) + # TODO, check if timezone is calculated right if result.get('publishedDate'): # do not try to get a date from an empty string or a None type try: # test if publishedDate >= 1900 (datetime module bug) -- GitLab From dffe5a09d3487dafb18c19b7922f192d5d46cc3d Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Sat, 16 Apr 2022 03:04:57 +0530 Subject: [PATCH 06/10] proxify images in backend --- etc/nginx/conf.d/spot.conf | 2 +- searx/templates/etheme/components/infobox.html | 4 ++-- searx/templates/etheme/result_templates/images.html | 2 +- searx/templates/etheme/result_templates/videos.html | 2 +- searx/webapp.py | 3 +++ 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf index 76eb7641b..9455d6c55 100644 --- a/etc/nginx/conf.d/spot.conf +++ b/etc/nginx/conf.d/spot.conf @@ -15,7 +15,7 @@ server { server_name _; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; - add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self'; data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; + add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data:; data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; add_header X-Frame-Options "SAMEORIGIN"; root /var/www/spot; diff --git a/searx/templates/etheme/components/infobox.html b/searx/templates/etheme/components/infobox.html index d4f343336..f06666972 100644 --- a/searx/templates/etheme/components/infobox.html +++ b/searx/templates/etheme/components/infobox.html @@ -11,7 +11,7 @@
{% if infobox.img_src %}
- +
{% endif %} @@ -26,7 +26,7 @@ {{ attribute.label }} {% if attribute.image %} - {{ attribute.image.alt }} + {{ attribute.image.alt }} {% else %} {{ attribute.value }} {% endif %} diff --git a/searx/templates/etheme/result_templates/images.html b/searx/templates/etheme/result_templates/images.html index 56c0b2772..07a919f8b 100644 --- a/searx/templates/etheme/result_templates/images.html +++ b/searx/templates/etheme/result_templates/images.html @@ -1,4 +1,4 @@ - {{ result.title|striptags }} + {{ result.title|striptags }} diff --git a/searx/templates/etheme/result_templates/videos.html b/searx/templates/etheme/result_templates/videos.html index a7b831d61..d89032a53 100644 --- a/searx/templates/etheme/result_templates/videos.html +++ b/searx/templates/etheme/result_templates/videos.html @@ -2,7 +2,7 @@
- {{ result.title|striptags }} {{ result.engine }} + {{ result.title|striptags }} {{ result.engine }}
{{ result_header(result, favicons, False) }} diff --git a/searx/webapp.py b/searx/webapp.py index efcbcc7a2..6716756de 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -662,6 +662,9 @@ def search(): if 'url' in result: result['pretty_url'] = prettify_url(result['url']) + if 'thumbnail' in result: + result['thumbnail'] = image_proxify(result['thumbnail']) + if 'thumbnail_src' in result: result['thumbnail_src'] = image_proxify(result['thumbnail_src']) -- GitLab From 5ea045dc5995ed92384d10c94a1b62ec9f60075e Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Mon, 18 Apr 2022 14:23:09 +0530 Subject: [PATCH 07/10] filtron rules to except image_proxy path --- etc/filtron/rules.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/filtron/rules.json b/etc/filtron/rules.json index a0313b2c8..aa294336e 100644 --- a/etc/filtron/rules.json +++ b/etc/filtron/rules.json @@ -6,7 +6,8 @@ "actions": [{ "name": "log"}] }, { - "name": "IP limit, all paths", + "name": "IP limit, all paths except image proxy", + "filters": ["Param:url", "Path=^(!/|/image_proxy)$"], "interval": 3, "limit": 25, "aggregations": ["Header:X-Forwarded-For"], @@ -16,7 +17,8 @@ ] }, { - "name": "useragent limit, all paths", + "name": "useragent limit, all paths except image_proxy", + "filters": ["Param:url", "Path=^(!/|/image_proxy)$"], "interval": 30, "limit": 200, "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], -- GitLab From eef6c964c41f6f72468e5a6ba861d48234d1bd2c Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Mon, 18 Apr 2022 14:36:40 +0530 Subject: [PATCH 08/10] proxify image in infobox --- searx/templates/etheme/components/infobox.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/searx/templates/etheme/components/infobox.html b/searx/templates/etheme/components/infobox.html index f06666972..d4f343336 100644 --- a/searx/templates/etheme/components/infobox.html +++ b/searx/templates/etheme/components/infobox.html @@ -11,7 +11,7 @@
{% if infobox.img_src %}
- +
{% endif %} @@ -26,7 +26,7 @@ {{ attribute.label }} {% if attribute.image %} - {{ attribute.image.alt }} + {{ attribute.image.alt }} {% else %} {{ attribute.value }} {% endif %} -- GitLab From dd2120035e4a0634b8720638acfdb7012f343dd1 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Mon, 18 Apr 2022 14:55:18 +0530 Subject: [PATCH 09/10] fix rule for filtering image_proxy --- etc/filtron/rules.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/filtron/rules.json b/etc/filtron/rules.json index aa294336e..73f53b3b4 100644 --- a/etc/filtron/rules.json +++ b/etc/filtron/rules.json @@ -7,7 +7,7 @@ }, { "name": "IP limit, all paths except image proxy", - "filters": ["Param:url", "Path=^(!/|/image_proxy)$"], + "filters": ["Param:url", "Path=^(!image_proxy)$"], "interval": 3, "limit": 25, "aggregations": ["Header:X-Forwarded-For"], @@ -18,7 +18,7 @@ }, { "name": "useragent limit, all paths except image_proxy", - "filters": ["Param:url", "Path=^(!/|/image_proxy)$"], + "filters": ["Param:url", "Path=^(!image_proxy)$"], "interval": 30, "limit": 200, "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], -- GitLab From 644598a78b4d19b2488adfaf0db17187f71cfec4 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Mon, 18 Apr 2022 15:35:42 +0530 Subject: [PATCH 10/10] update nginx csp headers --- etc/nginx/conf.d/spot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf index 9455d6c55..424966b89 100644 --- a/etc/nginx/conf.d/spot.conf +++ b/etc/nginx/conf.d/spot.conf @@ -15,7 +15,7 @@ server { server_name _; add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; - add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data:; data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; + add_header Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data:; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; add_header X-Frame-Options "SAMEORIGIN"; root /var/www/spot; -- GitLab