Commit ebd5dee0 authored by Nicolas Gelot's avatar Nicolas Gelot

Add cache feature on valid http requests

Redis LRU cache is added on http requests, only
response with a valid status code are cached during 1 day.

Close: #50
parent e556575c
......@@ -9,12 +9,13 @@ Spot was forked from searx: read [documentation](https://asciimoo.github.io/sear
## Changes between Spot and Searx
* eelo theme
* redis cache on http requests (TTL 1 day)
* docker packaging thinking to be production ready
* better locale support
## Architecture
6 services are used for production:
7 services are used for production:
* [traefik](https://docs.traefik.io/) as edge router to publish services.
* [filtron](https://github.com/asciimoo/filtron) as reverse HTTP proxy to filter requests by different rules.
......@@ -22,6 +23,7 @@ Spot was forked from searx: read [documentation](https://asciimoo.github.io/sear
* [nginx](https://www.nginx.com/) as http server to serve static files.
* Spot the meta search engine.
* [tor](https://www.torproject.org) as open network that helps you defend against traffic analysis.
* [redis](https://redis.io/) as memory storage to cache http requests
```mermaid
......@@ -35,6 +37,7 @@ graph TD
E --> H(tor1)
E --> I(tor2)
E --> J(torN)
E --> |cache| K(redis)
```
## Getting Started
......@@ -63,8 +66,9 @@ You can directly run spot, with a python command inside a docker container which
contains all dependencies.
```
docker run -it --rm -v $(pwd):/ws -w /ws registry.gitlab.e.foundation:5000/e/cloud/my-spot/env sh
SEARX_DEBUG=1 python -X dev searx/webapp.py
docker-compose up -d redis
docker run -it --rm -v $(pwd):/ws -w /ws --network=my-spot_default registry.gitlab.e.foundation:5000/e/cloud/my-spot/env sh
PYTHONPATH=$(pwd) SEARX_REDIS_HOST=redis SEARX_DEBUG=1 python -X dev searx/webapp.py
```
Then, open your browser and navigate to the container IP.
......@@ -8,6 +8,15 @@ x-logging:
driver: json-file
services:
redis:
image: redis:5.0.7-alpine
logging: *default-logging
restart: unless-stopped
command:
- "redis-server"
- "--maxmemory 20G"
- "--maxmemory-policy allkeys-lru"
spot:
image: ${SPOT_DOCKER_IMG}:${SPOT_DOCKER_TAG}
logging: *default-logging
......@@ -18,6 +27,7 @@ services:
SEARX_MORTY_KEY: "${SEARX_MORTY_KEY:-KHN0ZGluKT0gNWNmNzQ0Y2JlNjI4MDRjODAwZGUyMGY5ZjZlZTFmZWI1NTg2YTg5OAo=}"
SEARX_PROXY_HTTP: "socks5://tor:9050"
SEARX_PROXY_HTTPS: "socks5://tor:9050"
SEARX_REDIS_HOST: "redis"
GUNICORN_LOGGER: 1
GUNICORN_LEVEL: INFO
......
......@@ -10,3 +10,5 @@ pyopenssl==19.0.0
python-dateutil==2.8.0
pyyaml==5.1
requests[socks]==2.22.0
redis==3.4.1
ring==0.7.3
......@@ -108,3 +108,5 @@ if 'SEARX_PROXY_HTTP' in environ:
settings['outgoing'].setdefault('proxies', {})['http'] = environ['SEARX_PROXY_HTTP']
if 'SEARX_PROXY_HTTPS' in environ:
settings['outgoing'].setdefault('proxies', {})['https'] = environ['SEARX_PROXY_HTTPS']
if 'SEARX_REDIS_HOST' in environ:
settings['server']['redis_host'] = environ['SEARX_REDIS_HOST']
import functools
import redis
import ring
from ring.func import base as fbase
from ring.func.sync import CacheUserInterface
from searx import settings
redis_cache = None
class RequestCacheUserInterface(CacheUserInterface):
@fbase.interface_attrs(
transform_args=fbase.transform_kwargs_only, return_annotation=str)
def key(self, wire, **kwargs):
kwargs["kwargs"] = {}
return wire._rope.compose_key(*wire._bound_objects, **kwargs)
@fbase.interface_attrs(transform_args=fbase.transform_kwargs_only)
def get_or_update(self, wire, **kwargs):
key = self.key(wire, **kwargs)
try:
result = wire.storage.get(key)
except fbase.NotFound:
result = self.execute(wire, **kwargs)
if result.status_code >= 300:
return result
wire.storage.set(key, result)
return result
if "redis_host" in settings["server"]:
client = redis.StrictRedis(host=settings["server"]["redis_host"])
redis_cache = functools.partial(
ring.redis,
client,
coder="pickle",
user_interface=RequestCacheUserInterface,
expire=86400
)
......@@ -3,6 +3,7 @@ import requests
from itertools import cycle
from threading import RLock, local
from searx import settings
from searx.cache import redis_cache
from time import time
......@@ -128,9 +129,10 @@ def request(method, url, **kwargs):
return response
def get(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
if "redis_host" not in settings["server"]:
def get(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
def options(url, **kwargs):
......@@ -143,8 +145,9 @@ def head(url, **kwargs):
return request('head', url, **kwargs)
def post(url, data=None, **kwargs):
return request('post', url, data=data, **kwargs)
if "redis_host" not in settings["server"]:
def post(url, data=None, **kwargs):
return request('post', url, data=data, **kwargs)
def put(url, data=None, **kwargs):
......@@ -157,3 +160,16 @@ def patch(url, data=None, **kwargs):
def delete(url, **kwargs):
return request('delete', url, **kwargs)
if "redis_host" in settings["server"]:
@redis_cache()
def get(url, **kwargs):
kwargs = kwargs.get("kwargs", kwargs)
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
@redis_cache()
def post(url, data=None, **kwargs):
kwargs = kwargs.get("kwargs", kwargs)
return request('post', url, data=data, **kwargs)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment