Commit 7721632c authored by Nicolas Gelot's avatar Nicolas Gelot

Add cache interface

In order to use another cache tool or to use spot without cache
system.
parent 38118ff8
...@@ -89,6 +89,7 @@ if 'BASE_URL' in environ: ...@@ -89,6 +89,7 @@ if 'BASE_URL' in environ:
if 'IMAGE_PROXY' in environ: if 'IMAGE_PROXY' in environ:
settings['server']['image_proxy'] = environ['IMAGE_PROXY'] settings['server']['image_proxy'] = environ['IMAGE_PROXY']
if 'SEARX_REDIS_HOST' in environ: if 'SEARX_REDIS_HOST' in environ:
settings['redis']['enable'] = True
settings['redis']['host'] = environ['SEARX_REDIS_HOST'] settings['redis']['host'] = environ['SEARX_REDIS_HOST']
if 'HTTP_PROXY_URL' in environ: if 'HTTP_PROXY_URL' in environ:
settings['proxies']['http'] = environ['HTTP_PROXY_URL'] settings['proxies']['http'] = environ['HTTP_PROXY_URL']
......
...@@ -185,27 +185,25 @@ def default_request_params(): ...@@ -185,27 +185,25 @@ def default_request_params():
} }
def search(request): class Search:
""" Entry point to perform search request on engines """Search information manager"""
"""
search = Search() def __init__(self, cachecls=search_database.CacheInterface):
search_query = search.get_search_query_from_webapp(request.preferences, request.form) self.cache = cachecls()
searchData = search_database.read(search_query)
if searchData is None: def __call__(self, request):
result_container = search.search(search_query) """ Entry point to perform search request on engines
searchData = search.create_search_data(search_query, result_container) """
threading.Thread( search_query = self.get_search_query_from_webapp(request.preferences, request.form)
target=search_database.save, searchData = self.cache.read(search_query)
args=(searchData,),
name='save_search_' + str(searchData) if searchData is None:
).start() result_container = self.search(search_query)
searchData = self.create_search_data(search_query, result_container)
search.search_with_plugins(request, searchData) self.cache.save(searchData)
return searchData
self.search_with_plugins(request, searchData)
return searchData
class Search(object):
"""Search information container"""
def search(self, search_query): def search(self, search_query):
""" do search-request """ do search-request
......
import threading
import redis import redis
import pickle import pickle
...@@ -5,81 +6,104 @@ from searx import settings ...@@ -5,81 +6,104 @@ from searx import settings
from searx.query import SearchQuery from searx.query import SearchQuery
def make_key(q): class CacheInterface:
if q.time_range is None: """ Cache interface to store SearchData object
q.time_range = "" """
def read(self, q):
pass
return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format( def save(self, d):
q.query, pass
q.engines,
q.categories[0],
q.language,
q.safesearch,
q.pageno,
q.time_range,
)
def update(self, d):
pass
def _get_connection(): def get_twenty_queries(self, x):
return redis.Redis(host=settings['redis']['host']) return []
def read(q): class RedisCache(CacheInterface):
conn = _get_connection() def __init__(self):
key = make_key(q) self.pool = redis.ConnectionPool(host=settings['redis']['host'])
response = conn.get(key) self.running = threading.Event()
if not response:
return None
return pickle.loads(response)
def make_key(self, q):
if q.time_range is None:
q.time_range = ""
def save(d): return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
conn = _get_connection() q.query,
key = make_key(d) q.engines,
history = conn.incr("SEARCH_HISTORY_INDEX") q.categories[0],
conn.zadd("SEARCH_HISTORY_KEYS", {key: history}) q.language,
conn.set(key, pickle.dumps(d, protocol=4)) q.safesearch,
q.pageno,
q.time_range,
)
def get_twenty_queries(x): def _get_connection(self):
result = [] return redis.Redis(connection_pool=self.pool)
def read(self, q):
conn = self._get_connection()
key = self.make_key(q)
response = conn.get(key)
if not response:
return None
return pickle.loads(response)
def _save(self, d):
conn = self._get_connection()
key = self.make_key(d)
history = conn.incr("SEARCH_HISTORY_INDEX")
conn.zadd("SEARCH_HISTORY_KEYS", {key: history})
conn.set(key, pickle.dumps(d, protocol=4))
def save(self, d):
threading.Thread(
target=self._save,
args=(d,),
name='save_search_' + str(d)
).start()
def get_twenty_queries(self, x):
result = []
conn = self._get_connection()
keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
if not keys:
return result
pipe = conn.pipeline()
for key in keys:
pipe.get(key)
output = pipe.execute()
for row in output:
row = pickle.loads(row)
result.append(
SearchQuery(
row.query,
row.engines,
row.categories,
row.language,
row.safesearch,
row.pageno,
row.time_range,
)
)
conn = _get_connection()
keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
if not keys:
return result return result
pipe = conn.pipeline() def update(self, d):
for key in keys: conn = self._get_connection()
pipe.get(key) key = self.make_key(d)
output = pipe.execute() current = self.read(d)
for row in output: current.results = d.results
row = pickle.loads(row) current.paging = d.paging
result.append( current.results_number = d.results_number
SearchQuery( current.answers = d.answers
row.query, current.corrections = d.corrections
row.engines, current.infoboxes = d.infoboxes
row.categories, current.suggestions = d.suggestions
row.language, current.unresponsive_engines = d.unresponsive_engines
row.safesearch, conn.set(key, pickle.dumps(current, protocol=4))
row.pageno,
row.time_range,
)
)
return result
def update(d):
conn = _get_connection()
key = make_key(d)
current = read(d)
current.results = d.results
current.paging = d.paging
current.results_number = d.results_number
current.answers = d.answers
current.corrections = d.corrections
current.infoboxes = d.infoboxes
current.suggestions = d.suggestions
current.unresponsive_engines = d.unresponsive_engines
conn.set(key, pickle.dumps(current, protocol=4))
...@@ -17,6 +17,7 @@ server: ...@@ -17,6 +17,7 @@ server:
http_protocol_version : "1.0" # 1.0 and 1.1 are supported http_protocol_version : "1.0" # 1.0 and 1.1 are supported
redis: redis:
enable: False
host : "127.0.0.1" host : "127.0.0.1"
user : "searx" user : "searx"
password : "password" # change this! password : "password" # change this!
......
...@@ -33,7 +33,7 @@ import copy ...@@ -33,7 +33,7 @@ import copy
import requests import requests
from searx import logger, search_database from searx import logger
logger = logger.getChild('webapp') logger = logger.getChild('webapp')
...@@ -68,7 +68,8 @@ from searx.utils import ( ...@@ -68,7 +68,8 @@ from searx.utils import (
) )
from searx.version import VERSION_STRING from searx.version import VERSION_STRING
from searx.languages import language_codes as languages from searx.languages import language_codes as languages
from searx.search import Search, search from searx.search import Search
from searx.search_database import RedisCache
from searx.query import RawTextQuery from searx.query import RawTextQuery
from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.autocomplete import searx_bang, backends as autocomplete_backends
from searx.plugins import plugins from searx.plugins import plugins
...@@ -77,7 +78,6 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES ...@@ -77,7 +78,6 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
from searx.answerers import answerers from searx.answerers import answerers
from searx.url_utils import urlencode, urlparse, urljoin from searx.url_utils import urlencode, urlparse, urljoin
from searx.utils import new_hmac from searx.utils import new_hmac
from searx.search_database import get_twenty_queries
import threading import threading
# check if the pyopenssl package is installed. # check if the pyopenssl package is installed.
...@@ -131,6 +131,8 @@ if not searx_debug \ ...@@ -131,6 +131,8 @@ if not searx_debug \
babel = Babel(app) babel = Babel(app)
search = Search(RedisCache) if settings["redis"]["enable"] else Search()
rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he', rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi'] 'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
...@@ -842,18 +844,17 @@ def wait_updating(start_time): ...@@ -842,18 +844,17 @@ def wait_updating(start_time):
def update_results(): def update_results():
search = Search()
start_time = time.time() start_time = time.time()
x = 0 x = 0
while not running.is_set(): while not running.is_set():
queries = get_twenty_queries(x) queries = search.cache.get_twenty_queries(x)
for query in queries: for query in queries:
result_container = search.search(query) result_container = search.search(query)
searchData = search.create_search_data(query, result_container) searchData = search.create_search_data(query, result_container)
search_database.update(searchData) search.cache.update(searchData)
if running.is_set(): if running.is_set():
return return
x += 20 x += len(queries)
if len(queries) < 20: if len(queries) < 20:
x = 0 x = 0
wait_updating(start_time) wait_updating(start_time)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment