From 15c376430f58ca8fc60f4450f0e58d84b3c834da Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Tue, 21 Jan 2020 23:12:03 +0100 Subject: [PATCH 1/6] Introduce filtron to ban botnet Ref: #44 --- .env | 2 + .gitlab-ci.yml | 6 +- README.md | 10 ++-- docker-compose-build.yml | 15 +++++ docker-compose-prod.yml | 9 --- docker-compose.yml | 11 ++-- etc/filtron/rules.json | 120 +++++++++++++++++++++++++++++++++++++++ filtron.Dockerfile | 12 ++++ 8 files changed, 166 insertions(+), 19 deletions(-) create mode 100644 docker-compose-build.yml delete mode 100644 docker-compose-prod.yml create mode 100644 etc/filtron/rules.json create mode 100644 filtron.Dockerfile diff --git a/.env b/.env index e01a5e92b..7463602d4 100644 --- a/.env +++ b/.env @@ -3,3 +3,5 @@ SPOT_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx SPOT_NGINX_DOCKER_TAG=latest +SPOT_FILTRON_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot/filtron +SPOT_FILTRON_DOCKER_TAG=latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index afcaa4c92..948d3e9f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,8 @@ build:web: - docker push $CI_REGISTRY_IMAGE - docker build -t $CI_REGISTRY_IMAGE/nginx -f nginx.Dockerfile . - docker push $CI_REGISTRY_IMAGE/nginx + - docker build -t $CI_REGISTRY_IMAGE/filtron -f filtron.Dockerfile . + - docker push $CI_REGISTRY_IMAGE/filtron build:docker:master: extends: .build:docker @@ -48,6 +50,8 @@ build:docker:tags: - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - docker build -t $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG -f nginx.Dockerfile . - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG + - docker build -t $CI_REGISTRY_IMAGE/filtron:$CI_COMMIT_REF_SLUG -f filtron.Dockerfile . + - docker push $CI_REGISTRY_IMAGE/filtron:$CI_COMMIT_REF_SLUG only: - tags @@ -83,6 +87,7 @@ deploy:spot.test.cloud.global: DOCKER_HOST: ssh://root@spot.test.ecloud.global SPOT_HOSTNAME: spot.test.ecloud.global COMPOSE_PROJECT_NAME: my-spot + COMPOSE_FILE: docker-compose.yml:docker-compose-build.yml SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_TEST} script: - docker-compose up -d --build @@ -98,7 +103,6 @@ deploy:spot.cloud.global: DOCKER_HOST: ssh://root@spot.ecloud.global SPOT_HOSTNAME: spot.ecloud.global COMPOSE_PROJECT_NAME: my-spot - COMPOSE_FILE: docker-compose.yml:docker-compose-prod.yml SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SPOT_NGINX_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_PROD} diff --git a/README.md b/README.md index 9c38fd5c7..ac1b40f15 100644 --- a/README.md +++ b/README.md @@ -20,17 +20,17 @@ below to run spot for production or local environment. ### Like production -3 containes are used for production, traefik as edge router, -nginx to server static files and spot as backend. +3 containers are used for production, traefik as edge router, +filtron to drop malicious requests, nginx to server static files and spot as backend. * Run the docker-compose up command to start the project ``` -docker-compose up --build spot nginx +COMPOSE_FILE=docker-compose.yml:docker-compose-build.yml docker-compose up --build spot nginx filtron ``` -* Getting the ip of the nginx service and go to `http://`, below the docker way to get the IP of the nginx container +* Getting the ip of the nginx service and go to `http://`, below the docker way to get the IP of the filtron container ``` -docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_nginx_1 +docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_filtron_1 ``` ### For developer diff --git a/docker-compose-build.yml b/docker-compose-build.yml new file mode 100644 index 000000000..afef173af --- /dev/null +++ b/docker-compose-build.yml @@ -0,0 +1,15 @@ +version: '3.6' + +services: + spot: + build: . + + nginx: + build: + context: . + dockerfile: nginx.Dockerfile + + filtron: + build: + context: . + dockerfile: filtron.Dockerfile diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml deleted file mode 100644 index e6937e840..000000000 --- a/docker-compose-prod.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3.6' - -services: - spot: - image: ${SPOT_DOCKER_IMG}:${SPOT_DOCKER_TAG} - - nginx: - image: ${SPOT_NGINX_DOCKER_IMG}:${SPOT_NGINX_DOCKER_TAG} - diff --git a/docker-compose.yml b/docker-compose.yml index 9090d9d79..7804e932a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ x-logging: services: spot: - build: . + image: ${SPOT_DOCKER_IMG}:${SPOT_DOCKER_TAG} logging: *default-logging restart: unless-stopped environment: @@ -18,9 +18,12 @@ services: GUNICORN_LEVEL: INFO nginx: - build: - context: . - dockerfile: nginx.Dockerfile + image: ${SPOT_NGINX_DOCKER_IMG}:${SPOT_NGINX_DOCKER_TAG} + logging: *default-logging + restart: unless-stopped + + filtron: + image: ${SPOT_FILTRON_DOCKER_IMG}:${SPOT_FILTRON_DOCKER_TAG} logging: *default-logging restart: unless-stopped labels: diff --git a/etc/filtron/rules.json b/etc/filtron/rules.json new file mode 100644 index 000000000..8d7085134 --- /dev/null +++ b/etc/filtron/rules.json @@ -0,0 +1,120 @@ +[ + { + "name": "stats.searx.xyz", + "interval": 300, + "limit": 10, + "filters": ["Header:X-Forwarded-For=(2a01:4f8:161:542e::2|5.9.58.49)"], + "stop": true, + "actions": [{ "name": "log"}] + }, + { + "name": "IP limit, all paths", + "interval": 3, + "limit": 25, + "aggregations": ["Header:X-Forwarded-For"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded, try again later."}} + ] + }, + { + "name": "useragent limit, all paths", + "interval": 30, + "limit": 200, + "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded, try again later."}} + ] + }, + { + "name": "search request", + "filters": ["Param:q", "Path=^(/|/search)$"], + "subrules": [ + { + "name": "robot agent forbidden", + "limit": 0, + "stop": true, + "filters": ["Header:User-Agent=([Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser)"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "bot forbidden", + "limit": 0, + "stop": true, + "filters": ["Header:User-Agent=(Googlebot|GoogleImageProxy|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot|ZmEu|BLEXBot|bitlybot)"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "block missing accept-language", + "filters": ["!Header:Accept-Language"], + "limit": 0, + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "block Connection:close", + "filters": ["Header:Connection=close"], + "limit": 0, + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "block accept everything", + "filters": ["!Header:Accept=text/html"], + "limit": 0, + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "rss/json limit", + "interval": 3600, + "limit": 4, + "stop": true, + "filters": ["Param:format=(csv|json|rss)"], + "aggregations": ["Header:X-Forwarded-For"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded, try again later."}} + ] + }, + { + "name": "IP limit", + "interval": 3, + "limit": 3, + "aggregations": ["Header:X-Forwarded-For"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded, try again later."}} + ] + }, + { + "name": "IP and useragent limit", + "interval": 600, + "limit": 60, + "stop": true, + "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded, try again later."}} + ] + } + ] + } +] diff --git a/filtron.Dockerfile b/filtron.Dockerfile new file mode 100644 index 000000000..cb28d921e --- /dev/null +++ b/filtron.Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.13-alpine as builder + +RUN apk add git && go get github.com/asciimoo/filtron + + +FROM alpine:3.11 + +COPY --from=builder /go/bin/filtron /usr/bin/filtron +COPY etc/filtron/rules.json /etc/filtron/rules.json + +EXPOSE 80 +CMD ["filtron", "-listen", ":80", "-rules", "/etc/filtron/rules.json", "-target", "nginx"] -- GitLab From 6b57b0518020110b665d792f29be5777aec35309 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Tue, 21 Jan 2020 23:12:03 +0100 Subject: [PATCH 2/6] Add CSP policy Close: #44 --- etc/nginx/conf.d/spot.conf | 5 +- searx/settings.yml | 2 +- searx/static/themes/eelo/js/searx.js | 53 +++++++++--------- searx/static/themes/eelo/js/searx.min.js | Bin 5631 -> 5658 bytes .../static/themes/eelo/js/searx_src/custom.js | 52 ++++++++--------- searx/templates/eelo/base.html | 2 +- .../eelo/result_templates/images.html | 4 +- 7 files changed, 62 insertions(+), 56 deletions(-) diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf index 90af2a927..57acce75a 100644 --- a/etc/nginx/conf.d/spot.conf +++ b/etc/nginx/conf.d/spot.conf @@ -4,12 +4,13 @@ server { listen 80; server_name _; - add_header Content-Security-Policy "frame-ancestors 'self'"; add_header Referrer-Policy "no-referrer, strict-origin-when-cross-origin"; - add_header Strict-Transport-Security "max-age=31536000"; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; + add_header Content-Security-Policy "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"; add_header X-Frame-Options "SAMEORIGIN"; add_header X-XSS-Protection "1; mode=block"; add_header X-Content-Type-Options "nosniff"; + add_header X-Robots-Tag "noindex, noarchive, nofollow"; root /var/www/spot; diff --git a/searx/settings.yml b/searx/settings.yml index fa58d1a99..572cb5fa5 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -13,7 +13,7 @@ server: bind_address : "0.0.0.0" # address to listen on secret_key : "ultrasecretkey" # change this! base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" - image_proxy : False # Proxying image results through searx + image_proxy : True # Proxying image results through searx http_protocol_version : "1.0" # 1.0 and 1.1 are supported ui: diff --git a/searx/static/themes/eelo/js/searx.js b/searx/static/themes/eelo/js/searx.js index 1f5b9e7eb..ec0c9cab3 100644 --- a/searx/static/themes/eelo/js/searx.js +++ b/searx/static/themes/eelo/js/searx.js @@ -88,34 +88,37 @@ $(document).ready(function(){ }); } }); -; -function configure_image_view(target) { - document.getElementById("image_view_image").src = target.href; - document.getElementById("image_view_file_link").href = target.href; - document.getElementById("image_view_url_link").href = target.dataset.url; -} - -function show_image_view_modal(event) { - event.preventDefault(); - var target = event.target; - if (target.tagName == "IMG") { - target = target.parentElement; +;$(document).ready(function(){ + function configure_image_view(target, view_url) { + document.getElementById("image_view_image").src = view_url; + document.getElementById("image_view_file_link").href = target.href; + document.getElementById("image_view_url_link").href = target.dataset.url; } - var modal = document.getElementById("image_view_modal"); - modal.classList.remove("hidden"); - modal.style.top = window.scrollY + "px"; - configure_image_view(target); - document.body.classList.add("lock"); -} + $(".result.result-images").click(function (event) { + event.preventDefault(); + var target = event.target; + var view_url = target.src; + if (target.tagName == "IMG") { + target = target.parentElement; + } -function close_image_view_modal() { - document.getElementById("image_view_modal").classList.add("hidden"); - document.getElementById("image_view_image").src = ""; - document.getElementById("image_view_file_link").href = "#"; - document.getElementById("image_view_url_link").href = "#"; - document.body.classList.remove("lock"); -};/** + var modal = document.getElementById("image_view_modal"); + modal.classList.remove("hidden"); + modal.style.top = window.scrollY + "px"; + configure_image_view(target, view_url); + document.body.classList.add("lock"); + }); + + $("#close_image_view_modal").click(function () { + document.getElementById("image_view_modal").classList.add("hidden"); + document.getElementById("image_view_image").src = ""; + document.getElementById("image_view_file_link").href = "#"; + document.getElementById("image_view_url_link").href = "#"; + document.body.classList.remove("lock"); + }); +}); +;/** * searx is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or diff --git a/searx/static/themes/eelo/js/searx.min.js b/searx/static/themes/eelo/js/searx.min.js index d9dd67f9a71c04650813a1178c1b69fd8371136a..28d6dcd927c5f6cf56be8fb10f9dc984c41ab70c 100644 GIT binary patch delta 236 zcmeybJxgbT1e1~RL@9&KY0Q6_1ky_Ll1no4^Ar*_5_OU^t0zxjRGHY%!!%uP&BEmqRhOU}tm&enkG)=1KA z%}LEo%_{+EW-d}lo!rl;%a}TOFQW`&>f~FD@~nv($vP>M`I*$%G<8%ol$0miF^NrX ZU}BwY#>qc<0;BfiCZ;Ewo!IvB0|21QP1FDY delta 327 zcmbQG^Iv;{1e2lJL@5LLw9>rflFa-(h2;Faw9NF~LwUVc=yOOPKl3qz-x?f^$s+yWcl5LV+L1IyAUWsc?YHn&?2}Dz} zZAyM}DTuF^o(hth9Lp*bVx6p)oRe5w?2}nsqF0ofn_rfyp_GxCl9HOIq^XmvS6ot= zld4yeUtn9FnU|7Zu2-B~l%JCmsjXB{p`?S=qe+v0u&5ygMoxY)azH?Qx7nLjh;gIW LU#87S}jl diff --git a/searx/static/themes/eelo/js/searx_src/custom.js b/searx/static/themes/eelo/js/searx_src/custom.js index 848300231..9a6bc1f0f 100644 --- a/searx/static/themes/eelo/js/searx_src/custom.js +++ b/searx/static/themes/eelo/js/searx_src/custom.js @@ -1,28 +1,30 @@ - -function configure_image_view(target) { - document.getElementById("image_view_image").src = target.href; - document.getElementById("image_view_file_link").href = target.href; - document.getElementById("image_view_url_link").href = target.dataset.url; -} - -function show_image_view_modal(event) { - event.preventDefault(); - var target = event.target; - if (target.tagName == "IMG") { - target = target.parentElement; +$(document).ready(function(){ + function configure_image_view(target, view_url) { + document.getElementById("image_view_image").src = view_url; + document.getElementById("image_view_file_link").href = target.href; + document.getElementById("image_view_url_link").href = target.dataset.url; } - var modal = document.getElementById("image_view_modal"); - modal.classList.remove("hidden"); - modal.style.top = window.scrollY + "px"; - configure_image_view(target); - document.body.classList.add("lock"); -} + $(".result.result-images").click(function (event) { + event.preventDefault(); + var target = event.target; + var view_url = target.src; + if (target.tagName == "IMG") { + target = target.parentElement; + } + + var modal = document.getElementById("image_view_modal"); + modal.classList.remove("hidden"); + modal.style.top = window.scrollY + "px"; + configure_image_view(target, view_url); + document.body.classList.add("lock"); + }); -function close_image_view_modal() { - document.getElementById("image_view_modal").classList.add("hidden"); - document.getElementById("image_view_image").src = ""; - document.getElementById("image_view_file_link").href = "#"; - document.getElementById("image_view_url_link").href = "#"; - document.body.classList.remove("lock"); -} \ No newline at end of file + $("#close_image_view_modal").click(function () { + document.getElementById("image_view_modal").classList.add("hidden"); + document.getElementById("image_view_image").src = ""; + document.getElementById("image_view_file_link").href = "#"; + document.getElementById("image_view_url_link").href = "#"; + document.body.classList.remove("lock"); + }); +}); diff --git a/searx/templates/eelo/base.html b/searx/templates/eelo/base.html index cb2ab9c1b..67148459b 100644 --- a/searx/templates/eelo/base.html +++ b/searx/templates/eelo/base.html @@ -66,7 +66,7 @@