Commit 38118ff8 authored by Nicolas Gelot's avatar Nicolas Gelot

Update redis package and use pickle for serialize data

The new redis version is more strict about the kind of object
to store, we cannot use the same way that with the version 2.X.
In redis we want to store the SearchData class. Instead of
store field one by one we store the full serialized object with pickle.

Pickle is not very secure but we dont care for now because the redis
storage will be deploy in the same host that spot service. In a second time
we can use another way to serialize SearchData with msgpack and mashumaro
for example.
parent 43c2b056
redis==2.10.6
redis==3.2.1
certifi==2017.11.5
flask==1.0.2
flask-babel==0.11.2
......
import json
import threading
import urllib.parse
import redis
import pickle
from searx import settings
from searx.plugins import plugins
from searx.query import SearchQuery
from searx.url_utils import urlparse
from searx.results import SearchData
from searx import settings
def make_key(q):
......@@ -17,8 +10,8 @@ def make_key(q):
q.time_range = ""
return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
e(q.query),
je(q.engines),
q.query,
q.engines,
q.categories[0],
q.language,
q.safesearch,
......@@ -27,36 +20,25 @@ def make_key(q):
)
def _get_connection(decode_responses=True):
return redis.StrictRedis(host=settings['redis']['host'], decode_responses=decode_responses)
def _get_connection():
return redis.Redis(host=settings['redis']['host'])
def read(q):
conn = _get_connection()
key = make_key(q)
response = conn.hgetall(key)
response = conn.get(key)
if not response:
return None
results = jd(response['results'])
for result in results:
result['parsed_url'] = urlparse(result['url'])
return SearchData(q, results, response['paging'], int(response['results_number']),
jds(response['answers']), jds(response['corrections']), jd(response['infoboxes']),
jds(response['suggestions']), jds(response['unresponsive_engines']))
return pickle.loads(response)
def save(d):
conn = _get_connection()
key = make_key(d)
mapping = {
'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safesearch': d.safesearch,
'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),
'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),
'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),
'unresponsive_engines': jes(d.unresponsive_engines)
}
conn.zadd('SEARCH_HISTORY_KEYS', conn.incr('SEARCH_HISTORY_INDEX'), key)
conn.hmset(key, mapping)
history = conn.incr("SEARCH_HISTORY_INDEX")
conn.zadd("SEARCH_HISTORY_KEYS", {key: history})
conn.set(key, pickle.dumps(d, protocol=4))
def get_twenty_queries(x):
......@@ -69,46 +51,35 @@ def get_twenty_queries(x):
pipe = conn.pipeline()
for key in keys:
pipe.hgetall(key)
pipe.get(key)
output = pipe.execute()
for row in output:
result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],
int(row['safesearch']), int(row['pageno']), row['time_range']))
row = pickle.loads(row)
result.append(
SearchQuery(
row.query,
row.engines,
row.categories,
row.language,
row.safesearch,
row.pageno,
row.time_range,
)
)
return result
def e(obj):
return urllib.parse.quote_plus(obj)
def d(coded):
return urllib.parse.unquote_plus(coded)
def je(obj):
return e(json.dumps(obj))
def jd(coded):
return json.loads(d(coded))
def jes(set):
return je(list(set))
def jds(coded):
return jd(coded)
def update(d):
conn = _get_connection(decode_responses=False)
conn = _get_connection()
key = make_key(d)
current = conn.hgetall(key)
current.update({
'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,
'answers': jes(d.answers), 'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes),
'suggestions': jes(d.suggestions), 'unresponsive_engines': jes(d.unresponsive_engines)
})
conn.hmset(key, current)
current = read(d)
current.results = d.results
current.paging = d.paging
current.results_number = d.results_number
current.answers = d.answers
current.corrections = d.corrections
current.infoboxes = d.infoboxes
current.suggestions = d.suggestions
current.unresponsive_engines = d.unresponsive_engines
conn.set(key, pickle.dumps(current, protocol=4))
......@@ -515,27 +515,27 @@ def index():
else:
return index_error(e, output), 500
results_copy = copy.deepcopy(search_data.results)
if is_general_first_page:
result_copy = copy.copy(search_data.results)
for res in result_copy:
for res in results_copy:
if res.get('category') == 'images':
if len(images) < 5:
images.append(res)
search_data.results.remove(res)
results_copy.remove(res)
elif res.get('category') == 'videos':
if len(videos) < 2:
videos.append(res)
search_data.results.remove(res)
results_copy.remove(res)
elif res.get('category') is None:
search_data.results.remove(res)
results_copy.remove(res)
# output
config_results(search_data.results, search_data.query)
config_results(results_copy, search_data.query)
config_results(images, search_data.query)
config_results(videos, search_data.query)
response = dict(
results=search_data.results,
results=results_copy,
q=search_data.query,
selected_category=selected_category,
pageno=search_data.pageno,
......
......@@ -63,7 +63,7 @@ class ViewsTestCase(TestCase):
self.assertEqual(result.status_code, 200)
self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)
@patch('redis.StrictRedis', mock_strict_redis_client)
@patch('redis.Redis', mock_strict_redis_client)
def test_index_html(self):
result = self.app.post('/', data={'q': 'test'})
self.assertIn(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment