Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 38118ff8 authored by Nicolas Gelot's avatar Nicolas Gelot
Browse files

Update redis package and use pickle for serialize data 

The new redis version is more strict about the kind of object
to store, we cannot use the same way that with the version 2.X.
In redis we want to store the SearchData class. Instead of
store field one by one we store the full serialized object with pickle.

Pickle is not very secure but we dont care for now because the redis
storage will be deploy in the same host that spot service. In a second time
we can use another way to serialize SearchData with msgpack and mashumaro
for example.
parent 43c2b056

requirements.txt

+1 −1
Original line number Diff line number Diff line 
redis==2.10.6

redis==3.2.1

certifi==2017.11.5

flask==1.0.2

flask-babel==0.11.2

searx/search_database.py

+34 −63
Original line number Diff line number Diff line 
import json

import threading

import urllib.parse



import redis

import pickle



from searx import settings

from searx.plugins import plugins

from searx.query import SearchQuery

from searx.url_utils import urlparse

from searx.results import SearchData

from searx import settings





def make_key(q):
@@ -17,8 +10,8 @@ def make_key(q): 
        q.time_range = ""



    return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(

        e(q.query),

        je(q.engines),

        q.query,

        q.engines,

        q.categories[0],

        q.language,

        q.safesearch,
@@ -27,36 +20,25 @@ def make_key(q): 
    )





def _get_connection(decode_responses=True):

    return redis.StrictRedis(host=settings['redis']['host'], decode_responses=decode_responses)

def _get_connection():

    return redis.Redis(host=settings['redis']['host'])





def read(q):

    conn = _get_connection()

    key = make_key(q)

    response = conn.hgetall(key)

    response = conn.get(key)

    if not response:

        return None

    results = jd(response['results'])

    for result in results:

        result['parsed_url'] = urlparse(result['url'])

    return SearchData(q, results, response['paging'], int(response['results_number']),

                      jds(response['answers']), jds(response['corrections']), jd(response['infoboxes']),

                      jds(response['suggestions']), jds(response['unresponsive_engines']))

    return pickle.loads(response)





def save(d):

    conn = _get_connection()

    key = make_key(d)

    mapping = {

        'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safesearch': d.safesearch,

        'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),

        'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),

        'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),

        'unresponsive_engines': jes(d.unresponsive_engines)

    }

    conn.zadd('SEARCH_HISTORY_KEYS', conn.incr('SEARCH_HISTORY_INDEX'), key)

    conn.hmset(key, mapping)

    history = conn.incr("SEARCH_HISTORY_INDEX")

    conn.zadd("SEARCH_HISTORY_KEYS", {key: history})

    conn.set(key, pickle.dumps(d, protocol=4))





def get_twenty_queries(x):
@@ -69,46 +51,35 @@ def get_twenty_queries(x): 


    pipe = conn.pipeline()

    for key in keys:

        pipe.hgetall(key)

        pipe.get(key)

    output = pipe.execute()

    for row in output:

        result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],

                                  int(row['safesearch']), int(row['pageno']), row['time_range']))

        row = pickle.loads(row)

        result.append(

            SearchQuery(

                row.query,

                row.engines,

                row.categories,

                row.language,

                row.safesearch,

                row.pageno,

                row.time_range,

            )

        )



    return result





def e(obj):

    return urllib.parse.quote_plus(obj)





def d(coded):

    return urllib.parse.unquote_plus(coded)





def je(obj):

    return e(json.dumps(obj))





def jd(coded):

    return json.loads(d(coded))





def jes(set):

    return je(list(set))





def jds(coded):

    return jd(coded)





def update(d):

    conn = _get_connection(decode_responses=False)

    conn = _get_connection()

    key = make_key(d)

    current = conn.hgetall(key)

    current.update({

        'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,

        'answers': jes(d.answers), 'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes),

        'suggestions': jes(d.suggestions), 'unresponsive_engines': jes(d.unresponsive_engines)

    })

    conn.hmset(key, current)
 
    current = read(d)

    current.results = d.results

    current.paging = d.paging

    current.results_number = d.results_number

    current.answers = d.answers

    current.corrections = d.corrections

    current.infoboxes = d.infoboxes

    current.suggestions = d.suggestions

    current.unresponsive_engines = d.unresponsive_engines

    conn.set(key, pickle.dumps(current, protocol=4))

searx/webapp.py

+7 −7
Original line number Diff line number Diff line
@@ -515,27 +515,27 @@ def index(): 
        else:

            return index_error(e, output), 500



    results_copy = copy.deepcopy(search_data.results)

    if is_general_first_page:

        result_copy = copy.copy(search_data.results)

        for res in result_copy:

        for res in results_copy:

            if res.get('category') == 'images':

                if len(images) < 5:

                    images.append(res)

                search_data.results.remove(res)

                results_copy.remove(res)

            elif res.get('category') == 'videos':

                if len(videos) < 2:

                    videos.append(res)

                search_data.results.remove(res)

                results_copy.remove(res)

            elif res.get('category') is None:

                search_data.results.remove(res)

                results_copy.remove(res)



    # output

    config_results(search_data.results, search_data.query)

    config_results(results_copy, search_data.query)

    config_results(images, search_data.query)

    config_results(videos, search_data.query)



    response = dict(

        results=search_data.results,

        results=results_copy,

        q=search_data.query,

        selected_category=selected_category,

        pageno=search_data.pageno,

tests/unit/test_webapp.py

+1 −1
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ class ViewsTestCase(TestCase): 
        self.assertEqual(result.status_code, 200)

        self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)



    @patch('redis.StrictRedis', mock_strict_redis_client)

    @patch('redis.Redis', mock_strict_redis_client)

    def test_index_html(self):

        result = self.app.post('/', data={'q': 'test'})

        self.assertIn(