diff --git a/.env b/.env index f7eba7e87e06772171c8353f887ea28172470399..df0d91866dc7ec7799e1104ec34a729ce7b3d956 100644 --- a/.env +++ b/.env @@ -6,6 +6,6 @@ SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=etheme SEARX_PROXY_HTTP=socks5h://tor-socks-proxy:9150 SEARX_PROXY_HTTPS=socks5h://tor-socks-proxy:9150 -FILTRON_PORT=8088 +NGINX_PORT=8088 NODE_NAME=local CI_REGISTRY_IMAGE=registry.gitlab.e.foundation/e/infra/spot diff --git a/.env.prod b/.env.prod index 9ed4a2762759d7736e1ce0de6bc4a418cc50a5a7..4c3a34925f012eb6773f37d032ac84c24ca7dd1a 100644 --- a/.env.prod +++ b/.env.prod @@ -5,8 +5,8 @@ SEARX_REDIS_HOST=redis SEARX_UI_DEFAULT_THEME=etheme SEARX_PROXY_HTTP=http://proxy01.ecloud.global:1099 SEARX_PROXY_HTTPS=http://proxy01.ecloud.global:1099 -WIREGUARD_IP=127.0.0.1 -FILTRON_PORT=8088 +WIREGUARD_IP=127.0.0.1 +NGINX_PORT=8088 INTERNAL_NETWORK_NAME=default NODE_NAME=spot11 CI_REGISTRY_IMAGE=image diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 68e5f0e73d76fadb8ff300aa16e33701f746d563..478cabdb16eb24214e7d451ec29f20514d2c96ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,8 +68,8 @@ test:unit: - ssh $SSH_USER@$BACKEND_HOST "cd $DEPLOYMENT_PATH && git stash && git pull && git checkout $BRANCH && rm .env .env.prod && rm docker-compose.yml && mv docker-compose.prod.yml docker-compose.yml" - scp $ENV_FILE $SSH_USER@$BACKEND_HOST:$DEPLOYMENT_PATH/.env - ssh $SSH_USER@$BACKEND_HOST "cd $DEPLOYMENT_PATH - && sed -i 's/SPOT_DOCKER_TAG=master/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env - && docker-compose pull && docker-compose stop filtron && sleep 2 && docker-compose up -d && docker-compose restart spot-nginx" + && sed -i 's/SPOT_DOCKER_TAG=master/SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG/g' .env + && docker-compose pull && sleep 2 && docker-compose up -d && docker-compose restart spot-nginx" deploy:spot.murenatest.io.backend1: extends: .deploy:template diff --git a/README.md b/README.md index 24cd0e0c3c4fbfc3de93385383500cfb21218e44..6f4be31ef462bc9d3b7e91c444e8ece104657f4b 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,8 @@ Spot was forked from searx: read [documentation](https://asciimoo.github.io/sear ## Architecture -6 services are used for production: +3 services are used for production: -* [filtron](https://github.com/asciimoo/filtron) as reverse HTTP proxy to filter requests by different rules. * [nginx](https://www.nginx.com/) as http server to serve static files. * Spot the meta search engine. * [redis](https://redis.io/) as memory storage to cache http requests @@ -24,8 +23,7 @@ Spot was forked from searx: read [documentation](https://asciimoo.github.io/sear ```mermaid graph TD - A(reverse proxy) --> |http://localhost:8088| B(filtron) - B --> C(nginx) + A(reverse proxy) --> |http://localhost:8088| C(nginx) C --> |static file| C C --> |API| D(spot) D --> E(proxy service) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 325f0229b0ed54e4108728833cf0012148ab5ab6..99e2ffedacbd1a3d160bcfec8a1e7ae164041435 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,6 +26,7 @@ services: SEARX_PROXY_HTTP: "${SEARX_PROXY_HTTP}" SEARX_PROXY_HTTPS: "${SEARX_PROXY_HTTPS}" SEARX_REDIS_HOST: "${SEARX_REDIS_HOST}" + SEARXNG_REDIS_URL: "redis://${SEARX_REDIS_HOST}:6379" SEARX_UI_DEFAULT_THEME: "${SEARX_UI_DEFAULT_THEME}" GUNICORN_LOGGER: 1 GUNICORN_LEVEL: INFO @@ -43,6 +44,8 @@ services: tag: docker-${ENVIRONMENT_NAME}-spot-nginx networks: - ${INTERNAL_NETWORK_NAME} +- ports: +- - ${WIREGUARD_IP}:${NGINX_PORT}:80 volumes: - ./etc/nginx/conf.d/spot.conf:/etc/nginx/conf.d/default.conf - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf @@ -51,18 +54,6 @@ services: labels: - "com.centurylinklabs.watchtower.scope=staging-spot" - filtron: - image: dalf/filtron:latest - restart: always - command: -listen :3000 -rules /etc/filtron/rules.json -target spot-nginx - networks: - - ${INTERNAL_NETWORK_NAME} - - spot-wireguarded - ports: - - ${WIREGUARD_IP}:${FILTRON_PORT}:3000 - volumes: - - ./etc/filtron/rules.json:/etc/filtron/rules.json - watchtower: image: containrrr/watchtower volumes: diff --git a/etc/filtron/rules.json b/etc/filtron/rules.json deleted file mode 100644 index 9752d7419ea24bde811b19d20a8cb06dd242a25f..0000000000000000000000000000000000000000 --- a/etc/filtron/rules.json +++ /dev/null @@ -1,118 +0,0 @@ -[ - { - "name": "searx.space", - "filters": ["Header:X-Forwarded-For=nslookup(check.searx.space)"], - "stop": true, - "actions": [{ "name": "log"}] - }, - { - "name": "IP limit, all paths except image proxy", - "filters": ["Param:url", "Path=^(!image_proxy)$"], - "interval": 3, - "limit": 25, - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 9001, try again later."}} - ] - }, - { - "name": "useragent limit, all paths except image_proxy", - "filters": ["Param:url", "Path=^(!image_proxy)$"], - "interval": 30, - "limit": 200, - "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"], - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 9002, try again later."}} - ] - }, - { - "name": "search request", - "filters": ["Param:q", "Path=^(/|/search)$"], - "subrules": [ - { - "name": "allow Firefox Android (issue #48 and #60)", - "filters": [ - "Param:q=^1$", - "Header:User-Agent=(^MozacFetch/[0-9]{2,3}.[0-9].[0-9]+$|^Mozilla/5.0 \\(Android [0-9]{1,2}(.[0-9]{1,2}.[0-9]{1,2})?; Mobile; rv:[0-9]{2,3}.[0-9]\\) Gecko/[0-9]{2,3}.[0-9] Firefox/[0-9]{2,3}.[0-9]$)" - ], - "stop": true, - "actions": [{"name": "log"}] - }, - { - "name": "robot agent forbidden", - "limit": 0, - "stop": true, - "filters": ["Header:User-Agent=([Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10001"}} - ] - }, - { - "name": "bot forbidden", - "limit": 0, - "stop": true, - "filters": ["Header:User-Agent=(Googlebot|GoogleImageProxy|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot|ZmEu|BLEXBot|bitlybot)"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10002"}} - ] - }, - { - "name": "block missing accept-language", - "filters": ["!Header:Accept-Language"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10003"}} - ] - }, - { - "name": "block Connection:close", - "filters": ["Header:Connection=close"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10004"}} - ] - }, - { - "name": "block no gzip support", - "filters": ["!Header:Accept-Encoding=(^gzip$|^gzip[;,]|[; ]gzip$|[; ]gzip[;,])"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10005"}} - ] - }, - { - "name": "block no deflate support", - "filters": ["!Header:Accept-Encoding=(^deflate$|^deflate[;,]|[; ]deflate$|[; ]deflate[;,])"], - "limit": 0, - "stop": true, - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10006"}} - ] - }, - { - "name": "rss/json limit", - "interval": 3, - "limit": 10, - "stop": true, - "filters": ["Param:format=(csv|json|rss)"], - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "block", - "params": {"message": "Rate limit exceeded 10007, try again later."}} - ] - } - ] - } -] diff --git a/searx/settings.yml b/searx/settings.yml index 05375f6273894cff5f687d4c4c1a5c0ed204a2b3..33f821aae7220d5f863e4ee1962c70f367970d03 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -35,6 +35,7 @@ server: X-Download-Options : noopen X-Robots-Tag : noindex, nofollow Referrer-Policy : no-referrer + limiter: true redis: # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.