From 05a73019b16f21d9c1d45fe7d5add838364cce3c Mon Sep 17 00:00:00 2001
From: Venca24 <Vaclav.Zouzalik@seznam.cz>
Date: Mon, 4 Feb 2019 16:31:17 +0100
Subject: [PATCH 01/32] [fix] google scholar suggestions

---
 searx/settings.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/searx/settings.yml b/searx/settings.yml
index ff7782b65..c1a907ae1 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -296,7 +296,7 @@ engines:
     url_xpath : .//h3/a/@href
     title_xpath : .//h3/a
     content_xpath : .//div[@class="gs_rs"]
-    suggestion_xpath : //div[@id="gs_qsuggest"]/ul/li
+    suggestion_xpath : //div[@id="gs_res_ccl_top"]//a/b
     page_size : 10
     first_page_num : 0
     categories : science
-- 
GitLab


From f7bdd827c4cfd92fe182d5806f1e2c35352feed9 Mon Sep 17 00:00:00 2001
From: Jonas Zohren <jonas.zohren@tu-dortmund.de>
Date: Wed, 13 Feb 2019 00:37:29 +0100
Subject: [PATCH 02/32] [enh] adds apkmirror search engine

---
 searx/engines/apkmirror.py | 61 ++++++++++++++++++++++++++++++++++++++
 searx/settings.yml         |  6 ++++
 2 files changed, 67 insertions(+)
 create mode 100644 searx/engines/apkmirror.py

diff --git a/searx/engines/apkmirror.py b/searx/engines/apkmirror.py
new file mode 100644
index 000000000..f2ee12b29
--- /dev/null
+++ b/searx/engines/apkmirror.py
@@ -0,0 +1,61 @@
+"""
+ APK Mirror
+
+ @website     https://www.apkmirror.com
+
+ @using-api   no
+ @results     HTML
+ @stable      no (HTML can change)
+ @parse       url, title, thumbnail_src
+"""
+
+from lxml import html
+from searx.engines.xpath import extract_text
+from searx.url_utils import urlencode
+
+# engine dependent config
+categories = ['it']
+paging = True
+
+# I am not 100% certain about this, as apkmirror appears to be a wordpress site,
+# which might support time_range searching. If you want to implement it, go ahead.
+time_range_support = False
+
+# search-url
+base_url = 'https://www.apkmirror.com'
+search_url = base_url + '/?post_type=app_release&searchtype=apk&page={pageno}&{query}'
+
+
+# do search-request
+def request(query, params):
+
+    params['url'] = search_url.format(pageno=params['pageno'],
+                                      query=urlencode({'s': query}))
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+
+    dom = html.fromstring(resp.text)
+
+    # parse results
+    for result in dom.xpath('.//div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):
+
+        link = result.xpath('.//h5/a')[0]
+        url = base_url + link.attrib.get('href') + '#downloads'
+        title = extract_text(link)
+        thumbnail_src = base_url + result.xpath('.//img')[0].attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')
+
+        res = {
+            'url': url,
+            'title': title,
+            'thumbnail_src': thumbnail_src
+        }
+
+        # append result
+        results.append(res)
+
+    # return results
+    return results
diff --git a/searx/settings.yml b/searx/settings.yml
index ff7782b65..3851f49d5 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -49,6 +49,12 @@ outgoing: # communication with search engines
 #        - 1.1.1.2
 
 engines:
+  - name: apk mirror
+    engine: apkmirror
+    timeout: 4.0
+    shortcut: apkm
+    disabled: True
+
   - name : arch linux wiki
     engine : archlinux
     shortcut : al
-- 
GitLab


From 2478c5395d5504529e5d4b8ee09092fedbc71fbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Mon, 7 Jan 2019 21:06:53 +0100
Subject: [PATCH 03/32] update pyyaml

---
 requirements.txt  | 2 +-
 searx/__init__.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index fd61b8109..701e08de9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,5 +6,5 @@ idna==2.7
 pygments==2.1.3
 pyopenssl==18.0.0
 python-dateutil==2.7.3
-pyyaml==3.13
+pyyaml==4.2b1
 requests[socks]==2.19.1
diff --git a/searx/__init__.py b/searx/__init__.py
index b1010f25f..4d7b2a8d3 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -22,7 +22,7 @@ from os.path import realpath, dirname, join, abspath, isfile
 from io import open
 from ssl import OPENSSL_VERSION_INFO, OPENSSL_VERSION
 try:
-    from yaml import load
+    from yaml import safe_load
 except:
     from sys import exit, stderr
     stderr.write('[E] install pyyaml\n')
@@ -52,7 +52,7 @@ if not settings_path:
 
 # load settings
 with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
-    settings = load(settings_yaml)
+    settings = safe_load(settings_yaml)
 
 '''
 enable debug if
-- 
GitLab


From 30bdc8704c5a90a8911b764d925da9330433391c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Mon, 7 Jan 2019 21:07:32 +0100
Subject: [PATCH 04/32] require jinja to have unique function

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 701e08de9..e8a6c33b6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ flask==1.0.2
 flask-babel==0.11.2
 lxml==4.2.3
 idna==2.7
+jinja2==2.10
 pygments==2.1.3
 pyopenssl==18.0.0
 python-dateutil==2.7.3
-- 
GitLab


From 252ba92fddad357c820cdc6219377f0fb633c997 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= <sitbackandwait@gmail.com>
Date: Sat, 23 Feb 2019 18:53:32 +0100
Subject: [PATCH 05/32] add more updates

---
 requirements.txt | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index e8a6c33b6..b2ae2d453 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,11 @@
-certifi==2017.11.5
+certifi==2018.11.29
+flask-babel==0.12.2
 flask==1.0.2
-flask-babel==0.11.2
-lxml==4.2.3
-idna==2.7
+idna==2.8
 jinja2==2.10
+lxml==4.3.0
 pygments==2.1.3
-pyopenssl==18.0.0
-python-dateutil==2.7.3
+pyopenssl==19.0.0
+python-dateutil==2.7.5
 pyyaml==4.2b1
-requests[socks]==2.19.1
+requests[socks]==2.21.0
-- 
GitLab


From 8039a577a8d0501870c12facc9eba372ed4d1570 Mon Sep 17 00:00:00 2001
From: Elias Ojala <git@eliasojala.me>
Date: Tue, 12 Mar 2019 12:30:47 +0000
Subject: [PATCH 06/32] Use HTTPS for crossref

---
 searx/settings.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/searx/settings.yml b/searx/settings.yml
index 6e7c37fb0..f346647d1 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -127,7 +127,7 @@ engines:
   - name : crossref
     engine : json_engine
     paging : True
-    search_url : http://search.crossref.org/dois?q={query}&page={pageno}
+    search_url : https://search.crossref.org/dois?q={query}&page={pageno}
     url_query : doi
     title_query : title
     content_query : fullCitation
-- 
GitLab


From f2d49a697124b8f4c6a4df68626b3d29ec959e70 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Tue, 26 Mar 2019 20:33:36 -0600
Subject: [PATCH 07/32] [fix] get youtube results from js object

Results are not appearing in the html document anymore,
instead they are found inside an object embedded in a script.
---
 searx/engines/youtube_noapi.py           |  70 +++++-----
 tests/unit/engines/test_youtube_noapi.py | 162 ++++++++---------------
 2 files changed, 90 insertions(+), 142 deletions(-)

diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py
index 9f01841f6..3bf25932b 100644
--- a/searx/engines/youtube_noapi.py
+++ b/searx/engines/youtube_noapi.py
@@ -8,7 +8,8 @@
 # @stable      no
 # @parse       url, title, content, publishedDate, thumbnail, embedded
 
-from lxml import html
+from functools import reduce
+from json import loads
 from searx.engines.xpath import extract_text
 from searx.utils import list_get
 from searx.url_utils import quote_plus
@@ -34,20 +35,6 @@ embedded_url = '<iframe width="540" height="304" ' +\
 
 base_youtube_url = 'https://www.youtube.com/watch?v='
 
-# specific xpath variables
-results_xpath = "//ol/li/div[contains(@class, 'yt-lockup yt-lockup-tile yt-lockup-video vve-check')]"
-url_xpath = './/h3/a/@href'
-title_xpath = './/div[@class="yt-lockup-content"]/h3/a'
-content_xpath = './/div[@class="yt-lockup-content"]/div[@class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2"]'
-
-
-# returns extract_text on the first result selected by the xpath or None
-def extract_text_from_dom(result, xpath):
-    r = result.xpath(xpath)
-    if len(r) > 0:
-        return extract_text(r[0])
-    return None
-
 
 # do search-request
 def request(query, params):
@@ -63,27 +50,38 @@ def request(query, params):
 def response(resp):
     results = []
 
-    dom = html.fromstring(resp.text)
-
-    # parse results
-    for result in dom.xpath(results_xpath):
-        videoid = list_get(result.xpath('@data-context-item-id'), 0)
-        if videoid is not None:
-            url = base_youtube_url + videoid
-            thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg'
-
-            title = extract_text_from_dom(result, title_xpath) or videoid
-            content = extract_text_from_dom(result, content_xpath)
-
-            embedded = embedded_url.format(videoid=videoid)
-
-            # append result
-            results.append({'url': url,
-                            'title': title,
-                            'content': content,
-                            'template': 'videos.html',
-                            'embedded': embedded,
-                            'thumbnail': thumbnail})
+    results_data = resp.text[resp.text.find('ytInitialData'):]
+    results_data = results_data[results_data.find('{'):results_data.find(';\n')]
+
+    results_json = loads(results_data) if results_data else {}
+    sections = results_json.get('contents', {})\
+                           .get('twoColumnSearchResultsRenderer', {})\
+                           .get('primaryContents', {})\
+                           .get('sectionListRenderer', {})\
+                           .get('contents', [])
+
+    for section in sections:
+        for video_container in section.get('itemSectionRenderer', {}).get('contents', []):
+            video = video_container.get('videoRenderer', {})
+            videoid = video.get('videoId')
+            if videoid is not None:
+                url = base_youtube_url + videoid
+                thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg'
+                title = video.get('title', {}).get('simpleText', videoid)
+                description_snippet = video.get('descriptionSnippet', {})
+                if 'runs' in description_snippet:
+                    content = reduce(lambda a, b: a + b.get('text', ''), description_snippet.get('runs'), '')
+                else:
+                    content = description_snippet.get('simpleText', '')
+                embedded = embedded_url.format(videoid=videoid)
+
+                # append result
+                results.append({'url': url,
+                                'title': title,
+                                'content': content,
+                                'template': 'videos.html',
+                                'embedded': embedded,
+                                'thumbnail': thumbnail})
 
     # return results
     return results
diff --git a/tests/unit/engines/test_youtube_noapi.py b/tests/unit/engines/test_youtube_noapi.py
index 41dcbb749..cbf7b9bcd 100644
--- a/tests/unit/engines/test_youtube_noapi.py
+++ b/tests/unit/engines/test_youtube_noapi.py
@@ -46,121 +46,71 @@ class TestYoutubeNoAPIEngine(SearxTestCase):
         self.assertEqual(youtube_noapi.response(response), [])
 
         html = """
-        <ol id="item-section-063864" class="item-section">
-            <li>
-                <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile"
-                data-context-item-id="DIVZCPfAOeM"
-                data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB">
-                <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto">
-                <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link"
-                data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA">
-                <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg"
-                width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a>
-                <span class="thumb-menu dark-overflow-action-menu video-actions">
-                </span>
-                </div>
-                <div class="yt-lockup-content">
-                <h3 class="yt-lockup-title">
-                <a href="/watch?v=DIVZCPfAOeM"
-                class="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link"
-                data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"
-                title="Top Speed Test Kawasaki Ninja H2 (Thailand) By. MEHAY SUPERBIKE"
-                aria-describedby="description-id-259079" rel="spf-prefetch" dir="ltr">
-                Title
-                </a>
-                <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span>
-                </h3>
-                <div class="yt-lockup-byline">de
-                <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard"
-                data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA"
-                data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta">
-                <ul class="yt-lockup-meta-info">
-                    <li>il y a 20 heures</li>
-                    <li>8 424 vues</li>
-                </ul>
-                </div>
-                <div class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2" dir="ltr">
-                Description
-                </div>
-                <div class="yt-lockup-badges">
-                <ul class="yt-badge-list ">
-                    <li class="yt-badge-item" >
-                        <span class="yt-badge">Nouveauté</span>
-                    </li>
-                    <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li>
-                </ul>
-                </div>
-                <div class="yt-lockup-action-menu yt-uix-menu-container">
-                <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded"
-                data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu">
-                </div>
-                </div>
-                </div>
-                </div>
-                </div>
-            </li>
-        </ol>
+        <div></div>
+        <script>
+            window["ytInitialData"] = {
+                "contents": {
+                    "twoColumnSearchResultsRenderer": {
+                        "primaryContents": {
+                            "sectionListRenderer": {
+                                "contents": [
+                                    {
+                                        "itemSectionRenderer": {
+                                            "contents": [
+                                                {
+                                                    "videoRenderer": {
+                                                        "videoId": "DIVZCPfAOeM",
+                                                        "title": {
+                                                            "simpleText": "Title"
+                                                        },
+                                                        "descriptionSnippet": {
+                                                            "runs": [
+                                                                {
+                                                                    "text": "Des"
+                                                                },
+                                                                {
+                                                                    "text": "cription"
+                                                                }
+                                                            ]
+                                                        }
+                                                    }
+                                                },
+                                                {
+                                                    "videoRenderer": {
+                                                        "videoId": "9C_HReR_McQ",
+                                                        "title": {
+                                                            "simpleText": "Title"
+                                                        },
+                                                        "descriptionSnippet": {
+                                                            "simpleText": "Description"
+                                                        }
+                                                    }
+                                                }
+                                            ]
+                                        }
+                                    }
+                                ]
+                            }
+                        }
+                    }
+                }
+            };
+        </script>
         """
         response = mock.Mock(text=html)
         results = youtube_noapi.response(response)
         self.assertEqual(type(results), list)
-        self.assertEqual(len(results), 1)
+        self.assertEqual(len(results), 2)
         self.assertEqual(results[0]['title'], 'Title')
         self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM')
         self.assertEqual(results[0]['content'], 'Description')
         self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg')
         self.assertTrue('DIVZCPfAOeM' in results[0]['embedded'])
-
-        html = """
-        <ol id="item-section-063864" class="item-section">
-            <li>
-                <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile"
-                data-context-item-id="DIVZCPfAOeM"
-                data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB">
-                <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto">
-                <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link"
-                data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA">
-                <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg"
-                width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a>
-                <span class="thumb-menu dark-overflow-action-menu video-actions">
-                </span>
-                </div>
-                <div class="yt-lockup-content">
-                <h3 class="yt-lockup-title">
-                <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span>
-                </h3>
-                <div class="yt-lockup-byline">de
-                <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard"
-                data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA"
-                data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta">
-                <ul class="yt-lockup-meta-info">
-                    <li>il y a 20 heures</li>
-                    <li>8 424 vues</li>
-                </ul>
-                </div>
-                <div class="yt-lockup-badges">
-                <ul class="yt-badge-list ">
-                    <li class="yt-badge-item" >
-                        <span class="yt-badge">Nouveauté</span>
-                    </li>
-                    <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li>
-                </ul>
-                </div>
-                <div class="yt-lockup-action-menu yt-uix-menu-container">
-                <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded"
-                data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu">
-                </div>
-                </div>
-                </div>
-                </div>
-                </div>
-            </li>
-        </ol>
-        """
-        response = mock.Mock(text=html)
-        results = youtube_noapi.response(response)
-        self.assertEqual(type(results), list)
-        self.assertEqual(len(results), 1)
+        self.assertEqual(results[1]['title'], 'Title')
+        self.assertEqual(results[1]['url'], 'https://www.youtube.com/watch?v=9C_HReR_McQ')
+        self.assertEqual(results[1]['content'], 'Description')
+        self.assertEqual(results[1]['thumbnail'], 'https://i.ytimg.com/vi/9C_HReR_McQ/hqdefault.jpg')
+        self.assertTrue('9C_HReR_McQ' in results[1]['embedded'])
 
         html = """
         <ol id="item-section-063864" class="item-section">
-- 
GitLab


From aaa0524cec65d7155ae473e3ea67e16c4b757573 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Thu, 28 Mar 2019 19:25:20 -0600
Subject: [PATCH 08/32] [fix] stop engine from sending duplicated requests

---
 searx/query.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/searx/query.py b/searx/query.py
index 0b5143d39..f76bd91d3 100644
--- a/searx/query.py
+++ b/searx/query.py
@@ -113,19 +113,16 @@ class RawTextQuery(object):
                     parse_next = True
                     engine_name = engine_shortcuts[prefix]
                     if engine_name in engines:
-                        for engine_category in engines[engine_name].categories:
-                            self.engines.append({'category': engine_category,
-                                                 'name': engine_name,
-                                                 'from_bang': True})
+                        self.engines.append({'category': 'none',
+                                             'name': engine_name,
+                                             'from_bang': True})
 
                 # check if prefix is equal with engine name
                 elif prefix in engines:
                     parse_next = True
-                    if prefix in engines:
-                        for engine_category in engines[prefix].categories:
-                            self.engines.append({'category': engine_category,
-                                                 'name': prefix,
-                                                 'from_bang': True})
+                    self.engines.append({'category': 'none',
+                                         'name': prefix,
+                                         'from_bang': True})
 
                 # check if prefix is equal with categorie name
                 elif prefix in categories:
-- 
GitLab


From b842cb40a0e30bafc6e196d8bf68a305157a0b8d Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 09/32] Move search entry point in search module

---
 searx/search.py          | 24 ++++++++++++++++++++++++
 searx/search_database.py | 17 -----------------
 searx/webapp.py          |  4 ++--
 3 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index b5777c7ed..195d836c7 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -24,6 +24,7 @@ import requests.exceptions
 from flask_babel import gettext
 
 import searx.poolrequests as requests_lib
+from searx import search_database
 from searx import logger
 from searx.answerers import ask
 from searx.engines import (
@@ -34,6 +35,7 @@ from searx.plugins import plugins
 from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
 from searx.results import ResultContainer
 from searx.utils import gen_useragent
+from searx.plugins import plugins
 
 from _thread import start_new_thread
 
@@ -266,6 +268,28 @@ def get_search_query_from_webapp(preferences, form):
                        query_time_range)
 
 
+def search(request, host):
+    """ Entry point to perform search request on engines
+    """
+    search_query = get_search_query_from_webapp(request.preferences, request.form)
+    searchData = search_database.read(search_query, host)
+    if searchData is None:
+        result_container = Search(search_query).search()
+        searchData = search_database.get_search_data(search_query, result_container)
+        threading.Thread(
+            target=search_database.save,
+            args=(searchData, host),
+            name='save_search_' + str(searchData)
+        ).start()
+
+    ordered_plugin = request.user_plugins
+    plugins.call(ordered_plugin, 'post_search', request, searchData)
+
+    for result in searchData.results:
+        plugins.call(ordered_plugin, 'on_result', request, searchData, result)
+    return searchData
+
+
 class Search(object):
     """Search information container"""
 
diff --git a/searx/search_database.py b/searx/search_database.py
index c9f74e6bf..2ba7ab0c2 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -7,7 +7,6 @@ import redis
 from searx import settings
 from searx.plugins import plugins
 from searx.query import SearchQuery
-from searx.search import Search, get_search_query_from_webapp
 from searx.url_utils import urlparse
 
 
@@ -131,22 +130,6 @@ def get_search_data(q, r):
                       r.infoboxes, r.suggestions, r.unresponsive_engines)
 
 
-def search(request, host):
-    search_query = get_search_query_from_webapp(request.preferences, request.form)
-    searchData = read(search_query, host)
-    if searchData is None:
-        result_container = Search(search_query).search()
-        searchData = get_search_data(search_query, result_container)
-        threading.Thread(target=save, args=(searchData, host), name='save_search_' + str(searchData)).start()
-
-    ordered_plugin = request.user_plugins
-    plugins.call(ordered_plugin, 'post_search', request, searchData)
-
-    for result in searchData.results:
-        plugins.call(ordered_plugin, 'on_result', request, searchData, result)
-    return searchData
-
-
 def update(d, host):
     conn = redis.StrictRedis(host)
     key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
diff --git a/searx/webapp.py b/searx/webapp.py
index 58de17912..9a6dfd99a 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -68,7 +68,7 @@ from searx.utils import (
 )
 from searx.version import VERSION_STRING
 from searx.languages import language_codes as languages
-from searx.search import Search
+from searx.search import Search, search
 from searx.query import RawTextQuery
 from searx.autocomplete import searx_bang, backends as autocomplete_backends
 from searx.plugins import plugins
@@ -77,7 +77,7 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
 from searx.answerers import answerers
 from searx.url_utils import urlencode, urlparse, urljoin
 from searx.utils import new_hmac
-from searx.search_database import get_twenty_queries, search
+from searx.search_database import get_twenty_queries
 import threading
 
 # check if the pyopenssl package is installed.
-- 
GitLab


From 743ed248ae4e0394d6802b53eafee145ab81698b Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 10/32] Remove unused class SearchWithPlugins in search module

---
 searx/search.py | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index 195d836c7..018104382 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -31,7 +31,6 @@ from searx.engines import (
     categories, engines, settings
 )
 from searx.exceptions import SearxParameterException
-from searx.plugins import plugins
 from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
 from searx.results import ResultContainer
 from searx.utils import gen_useragent
@@ -377,25 +376,3 @@ class Search(object):
 
         # return results, suggestions, answers and infoboxes
         return self.result_container
-
-
-class SearchWithPlugins(Search):
-    """Similar to the Search class but call the plugins."""
-
-    def __init__(self, search_query, ordered_plugin_list, request):
-        super(SearchWithPlugins, self).__init__(search_query)
-        self.ordered_plugin_list = ordered_plugin_list
-        self.request = request
-
-    def search(self):
-        if plugins.call(self.ordered_plugin_list, 'pre_search', self.request, self):
-            super(SearchWithPlugins, self).search()
-
-        plugins.call(self.ordered_plugin_list, 'post_search', self.request, self)
-
-        results = self.result_container.get_ordered_results()
-
-        for result in results:
-            plugins.call(self.ordered_plugin_list, 'on_result', self.request, self, result)
-
-        return self.result_container
-- 
GitLab


From 00642228fc36c18417c515d6b0848b4ea6dd160d Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 11/32] Move SearchData in results module

---
 searx/results.py         | 20 ++++++++++++++++++++
 searx/search_database.py | 21 +--------------------
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/searx/results.py b/searx/results.py
index 6c5b52eaf..973e5bf82 100644
--- a/searx/results.py
+++ b/searx/results.py
@@ -317,3 +317,23 @@ class ResultContainer(object):
 
     def add_unresponsive_engine(self, engine_error):
         self.unresponsive_engines.add(engine_error)
+
+
+class SearchData(object):
+    def __init__(self, search_query, results, paging,
+                 results_number, answers, corrections, infoboxes, suggestions, unresponsive_engines):
+        self.categories = search_query.categories
+        self.query = search_query.query
+        self.pageno = search_query.pageno
+        self.safe_search = search_query.safesearch
+        self.language = search_query.lang
+        self.time_range = search_query.time_range
+        self.engines = search_query.engines
+        self.results = results
+        self.paging = paging
+        self.results_number = results_number
+        self.answers = answers
+        self.corrections = corrections
+        self.infoboxes = infoboxes
+        self.suggestions = suggestions
+        self.unresponsive_engines = unresponsive_engines
diff --git a/searx/search_database.py b/searx/search_database.py
index 2ba7ab0c2..4803ac03f 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -8,26 +8,7 @@ from searx import settings
 from searx.plugins import plugins
 from searx.query import SearchQuery
 from searx.url_utils import urlparse
-
-
-class SearchData(object):
-    def __init__(self, search_query, results, paging,
-                 results_number, answers, corrections, infoboxes, suggestions, unresponsive_engines):
-        self.categories = search_query.categories
-        self.query = search_query.query
-        self.pageno = search_query.pageno
-        self.safe_search = search_query.safesearch
-        self.language = search_query.lang
-        self.time_range = search_query.time_range
-        self.engines = search_query.engines
-        self.results = results
-        self.paging = paging
-        self.results_number = results_number
-        self.answers = answers
-        self.corrections = corrections
-        self.infoboxes = infoboxes
-        self.suggestions = suggestions
-        self.unresponsive_engines = unresponsive_engines
+from searx.results import SearchData
 
 
 def _get_connection(host):
-- 
GitLab


From f22fa470bdf815597a44d88c36e93a15baaa73f8 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 12/32] Move get_search_query_from_webapp in Search class

---
 searx/search.py           | 192 +++++++++++++++++++-------------------
 searx/webapp.py           |   3 +-
 utils/standalone_searx.py |   6 +-
 3 files changed, 99 insertions(+), 102 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index 018104382..49cb91164 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -185,95 +185,14 @@ def default_request_params():
     }
 
 
-def get_search_query_from_webapp(preferences, form):
-    # no text for the query ?
-    if not form.get('q'):
-        raise SearxParameterException('q', '')
-
-    # set blocked engines
-    disabled_engines = preferences.engines.get_disabled()
-
-    # parse query, if tags are set, which change
-    # the serch engine or search-language
-    raw_text_query = RawTextQuery(form['q'], disabled_engines)
-    raw_text_query.parse_query()
-
-    # set query
-    query = raw_text_query.getSearchQuery()
-
-    # get and check page number
-    pageno_param = form.get('pageno', '1')
-    if not pageno_param.isdigit() or int(pageno_param) < 1:
-        raise SearxParameterException('pageno', pageno_param)
-    query_pageno = int(pageno_param)
-
-    # get language
-    # set specific language if set on request, query or preferences
-    # TODO support search with multible languages
-    if len(raw_text_query.languages):
-        query_lang = raw_text_query.languages[-1]
-    elif 'language' in form:
-        query_lang = form.get('language')
-    else:
-        query_lang = preferences.get_value('language')
-
-    # check language
-    if not VALID_LANGUAGE_CODE.match(query_lang):
-        raise SearxParameterException('language', query_lang)
-
-    # get safesearch
-    if 'safesearch' in form:
-        query_safesearch = form.get('safesearch')
-        # first check safesearch
-        if not query_safesearch.isdigit():
-            raise SearxParameterException('safesearch', query_safesearch)
-        query_safesearch = int(query_safesearch)
-    else:
-        query_safesearch = preferences.get_value('safesearch')
-
-    # safesearch : second check
-    if query_safesearch < 0 or query_safesearch > 2:
-        raise SearxParameterException('safesearch', query_safesearch)
-
-    # get time_range
-    query_time_range = form.get('time_range')
-
-    # check time_range
-    if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
-        raise SearxParameterException('time_range', query_time_range)
-
-    # query_engines
-    query_engines = raw_text_query.engines
-
-    # we only need to check the categories parameter set by the caller (not by the user)
-    # so we can assume it contains a list of categories
-    query_categories = form.get('categories')
-
-    def append_to_engines(cat):
-        # protect agains custom category provided by the user
-        engines = categories.get(cat)
-        if engines is None:
-            return
-        for engine in engines:
-            if (engine.name, cat) not in disabled_engines:
-                query_engines.append({'category': cat, 'name': engine.name})
-
-    # on top of the category field we have query_categories, which will be the engines to
-    # use to perform the search.
-    for category in query_categories:
-        append_to_engines(category)
-
-    return SearchQuery(query, query_engines, query_categories, query_lang, query_safesearch, query_pageno,
-                       query_time_range)
-
-
 def search(request, host):
     """ Entry point to perform search request on engines
     """
-    search_query = get_search_query_from_webapp(request.preferences, request.form)
+    search = Search()
+    search_query = search.get_search_query_from_webapp(request.preferences, request.form)
     searchData = search_database.read(search_query, host)
     if searchData is None:
-        result_container = Search(search_query).search()
+        result_container = search.search(search_query)
         searchData = search_database.get_search_data(search_query, result_container)
         threading.Thread(
             target=search_database.save,
@@ -292,26 +211,24 @@ def search(request, host):
 class Search(object):
     """Search information container"""
 
-    def __init__(self, search_query):
-        # init vars
-        super(Search, self).__init__()
-        self.search_query = search_query
-        self.result_container = ResultContainer()
+    def search(self, search_query):
+        """ do search-request
 
-    # do search-request
-    def search(self):
+        Return a ResultContainer object
+        """
         global number_of_searches
+        result_container = ResultContainer()
 
         # start time
         start_time = time()
 
         # answeres ?
-        answerers_results = ask(self.search_query)
+        answerers_results = ask(search_query)
 
         if answerers_results:
             for results in answerers_results:
-                self.result_container.extend('answer', results)
-            return self.result_container
+                result_container.extend('answer', results)
+            return result_container
 
         # init vars
         requests = []
@@ -323,8 +240,6 @@ class Search(object):
         # user_agent = request.headers.get('User-Agent', '')
         user_agent = gen_useragent()
 
-        search_query = self.search_query
-
         # max of all selected engine timeout
         timeout_limit = 0
 
@@ -371,8 +286,89 @@ class Search(object):
 
         if requests:
             # send all search-request
-            search_multiple_requests(requests, self.result_container, start_time, timeout_limit)
+            search_multiple_requests(requests, result_container, start_time, timeout_limit)
             start_new_thread(gc.collect, tuple())
 
         # return results, suggestions, answers and infoboxes
-        return self.result_container
+        return result_container
+
+    def get_search_query_from_webapp(self, preferences, form):
+        # no text for the query ?
+        if not form.get('q'):
+            raise SearxParameterException('q', '')
+
+        # set blocked engines
+        disabled_engines = preferences.engines.get_disabled()
+
+        # parse query, if tags are set, which change
+        # the serch engine or search-language
+        raw_text_query = RawTextQuery(form['q'], disabled_engines)
+        raw_text_query.parse_query()
+
+        # set query
+        query = raw_text_query.getSearchQuery()
+
+        # get and check page number
+        pageno_param = form.get('pageno', '1')
+        if not pageno_param.isdigit() or int(pageno_param) < 1:
+            raise SearxParameterException('pageno', pageno_param)
+        query_pageno = int(pageno_param)
+
+        # get language
+        # set specific language if set on request, query or preferences
+        # TODO support search with multible languages
+        if len(raw_text_query.languages):
+            query_lang = raw_text_query.languages[-1]
+        elif 'language' in form:
+            query_lang = form.get('language')
+        else:
+            query_lang = preferences.get_value('language')
+
+        # check language
+        if not VALID_LANGUAGE_CODE.match(query_lang):
+            raise SearxParameterException('language', query_lang)
+
+        # get safesearch
+        if 'safesearch' in form:
+            query_safesearch = form.get('safesearch')
+            # first check safesearch
+            if not query_safesearch.isdigit():
+                raise SearxParameterException('safesearch', query_safesearch)
+            query_safesearch = int(query_safesearch)
+        else:
+            query_safesearch = preferences.get_value('safesearch')
+
+        # safesearch : second check
+        if query_safesearch < 0 or query_safesearch > 2:
+            raise SearxParameterException('safesearch', query_safesearch)
+
+        # get time_range
+        query_time_range = form.get('time_range')
+
+        # check time_range
+        if query_time_range not in ('None', None, '', 'day', 'week', 'month', 'year'):
+            raise SearxParameterException('time_range', query_time_range)
+
+        # query_engines
+        query_engines = raw_text_query.engines
+
+        # we only need to check the categories parameter set by the caller (not by the user)
+        # so we can assume it contains a list of categories
+        query_categories = form.get('categories')
+
+        def append_to_engines(cat):
+            # protect agains custom category provided by the user
+            engines = categories.get(cat)
+            if engines is None:
+                return
+            for engine in engines:
+                if (engine.name, cat) not in disabled_engines:
+                    query_engines.append({'category': cat, 'name': engine.name})
+
+        # on top of the category field we have query_categories, which will be the engines to
+        # use to perform the search.
+        for category in query_categories:
+            append_to_engines(category)
+
+        return SearchQuery(query, query_engines, query_categories, query_lang, query_safesearch, query_pageno,
+                           query_time_range)
diff --git a/searx/webapp.py b/searx/webapp.py
index 9a6dfd99a..60bd13116 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -842,13 +842,14 @@ def wait_updating(start_time):
 
 
 def update_results():
+    search = Search()
     start_time = time.time()
     x = 0
     while not running.is_set():
         host = settings['redis']['host']
         queries = get_twenty_queries(x, host)
         for query in queries:
-            result_container = Search(query).search()
+            result_container = search.search(query)
             searchData = search_database.get_search_data(query, result_container)
             search_database.update(searchData, host)
             if running.is_set():
diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py
index 299860569..9efdc556d 100755
--- a/utils/standalone_searx.py
+++ b/utils/standalone_searx.py
@@ -64,9 +64,9 @@ form = {
 preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])
 preferences.key_value_settings['safesearch'].parse(args.safesearch)
 
-search_query = searx.search.get_search_query_from_webapp(preferences, form)
-search = searx.search.Search(search_query)
-result_container = search.search()
+search = searx.Search()
+search_query = search.get_search_query_from_webapp(preferences, form)
+result_container = search.search(search_query)
 
 # output
 from datetime import datetime
-- 
GitLab


From 754ff97a59963ddbbda5af4361e19aeda1a4dcec Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 13/32] Create a dedicated handler about plugins hooks in
 Search class

---
 searx/search.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index 49cb91164..814776e68 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -200,14 +200,9 @@ def search(request, host):
             name='save_search_' + str(searchData)
         ).start()
 
-    ordered_plugin = request.user_plugins
-    plugins.call(ordered_plugin, 'post_search', request, searchData)
-
-    for result in searchData.results:
-        plugins.call(ordered_plugin, 'on_result', request, searchData, result)
+    search.search_with_plugins(request, searchData)
     return searchData
 
-
 class Search(object):
     """Search information container"""
 
@@ -292,6 +287,13 @@ class Search(object):
         # return results, suggestions, answers and infoboxes
         return result_container
 
+    def search_with_plugins(self, request, searchData):
+        ordered_plugin = request.user_plugins
+        plugins.call(ordered_plugin, 'post_search', request, searchData)
+
+        for result in searchData.results:
+            plugins.call(ordered_plugin, 'on_result', request, searchData, result)
+
     def get_search_query_from_webapp(self, preferences, form):
         # no text for the query ?
         if not form.get('q'):
-- 
GitLab


From 4f2517869e8f8fdff0d79c334e0fea53080216ec Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 14/32] Move get_search_data in Search class as
 create_search_data

---
 searx/search.py          | 22 ++++++++++++++++++++--
 searx/search_database.py | 18 ------------------
 searx/webapp.py          |  2 +-
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index 814776e68..7561acd90 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -32,7 +32,7 @@ from searx.engines import (
 )
 from searx.exceptions import SearxParameterException
 from searx.query import RawTextQuery, SearchQuery, VALID_LANGUAGE_CODE
-from searx.results import ResultContainer
+from searx.results import ResultContainer, SearchData
 from searx.utils import gen_useragent
 from searx.plugins import plugins
 
@@ -193,7 +193,7 @@ def search(request, host):
     searchData = search_database.read(search_query, host)
     if searchData is None:
         result_container = search.search(search_query)
-        searchData = search_database.get_search_data(search_query, result_container)
+        searchData = search.create_search_data(search_query, result_container)
         threading.Thread(
             target=search_database.save,
             args=(searchData, host),
@@ -203,6 +203,7 @@ def search(request, host):
     search.search_with_plugins(request, searchData)
     return searchData
 
+
 class Search(object):
     """Search information container"""
 
@@ -374,3 +375,20 @@ class Search(object):
 
         return SearchQuery(query, query_engines, query_categories, query_lang, query_safesearch, query_pageno,
                            query_time_range)
+
+    def create_search_data(self, q, r):
+        results_number = r.results_number()
+        if results_number < r.results_length():
+            results_number = 0
+        results = r.get_ordered_results()
+        for result in results:
+            if 'publishedDate' in result:
+                try:
+                    result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S')
+                finally:
+                    result['publishedDate'] = None
+        if q.time_range is None:
+            q.time_range = ""
+
+        return SearchData(q, results, r.paging, results_number, r.answers, r.corrections,
+                          r.infoboxes, r.suggestions, r.unresponsive_engines)
diff --git a/searx/search_database.py b/searx/search_database.py
index 4803ac03f..3b2070606 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -93,24 +93,6 @@ def jds(coded):
     return jd(coded)
 
 
-def get_search_data(q, r):
-    results_number = r.results_number()
-    if results_number < r.results_length():
-        results_number = 0
-    results = r.get_ordered_results()
-    for result in results:
-        if 'publishedDate' in result:
-            try:
-                result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S')
-            finally:
-                result['publishedDate'] = None
-    if q.time_range is None:
-        q.time_range = ""
-
-    return SearchData(q, results, r.paging, results_number, r.answers, r.corrections,
-                      r.infoboxes, r.suggestions, r.unresponsive_engines)
-
-
 def update(d, host):
     conn = redis.StrictRedis(host)
     key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
diff --git a/searx/webapp.py b/searx/webapp.py
index 60bd13116..fd333afb6 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -850,7 +850,7 @@ def update_results():
         queries = get_twenty_queries(x, host)
         for query in queries:
             result_container = search.search(query)
-            searchData = search_database.get_search_data(query, result_container)
+            searchData = search.create_search_data(query, result_container)
             search_database.update(searchData, host)
             if running.is_set():
                 return
-- 
GitLab


From d8c97adf5562b65c243cca503421777277ed7bf9 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 15/32] Remove host parameter of all redis handlers

---
 searx/search.py          |  6 +++---
 searx/search_database.py | 22 +++++++++++-----------
 searx/webapp.py          |  7 +++----
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/searx/search.py b/searx/search.py
index 7561acd90..2742c5195 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -185,18 +185,18 @@ def default_request_params():
     }
 
 
-def search(request, host):
+def search(request):
     """ Entry point to perform search request on engines
     """
     search = Search()
     search_query = search.get_search_query_from_webapp(request.preferences, request.form)
-    searchData = search_database.read(search_query, host)
+    searchData = search_database.read(search_query)
     if searchData is None:
         result_container = search.search(search_query)
         searchData = search.create_search_data(search_query, result_container)
         threading.Thread(
             target=search_database.save,
-            args=(searchData, host),
+            args=(searchData,),
             name='save_search_' + str(searchData)
         ).start()
 
diff --git a/searx/search_database.py b/searx/search_database.py
index 3b2070606..f9dd7d7e4 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -9,19 +9,19 @@ from searx.plugins import plugins
 from searx.query import SearchQuery
 from searx.url_utils import urlparse
 from searx.results import SearchData
+from searx import settings
 
 
-def _get_connection(host):
-    host = host if host else settings['redis']['host']
-    return redis.StrictRedis(host=host, decode_responses=True)
+def _get_connection(decode_responses=True):
+    return redis.StrictRedis(host=settings['redis']['host'], decode_responses=decode_responses)
 
 
-def read(q, host):
+def read(q):
     time_range = q.time_range
     if q.time_range is None:
         q.time_range = ""
 
-    conn = _get_connection(host)
+    conn = _get_connection()
     key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
         e(q.query), je(q.engines), q.categories[0], q.lang, q.safesearch, q.pageno, time_range)
     response = conn.hgetall(key)
@@ -35,8 +35,8 @@ def read(q, host):
                       jds(response['suggestions']), jds(response['unresponsive_engines']))
 
 
-def save(d, host):
-    conn = _get_connection(host)
+def save(d):
+    conn = _get_connection()
     key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
         e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
     mapping = {
@@ -50,10 +50,10 @@ def save(d, host):
     conn.hmset(key, mapping)
 
 
-def get_twenty_queries(x, host):
+def get_twenty_queries(x):
     result = []
 
-    conn = _get_connection(host)
+    conn = _get_connection()
     keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
     if not keys:
         return result
@@ -93,8 +93,8 @@ def jds(coded):
     return jd(coded)
 
 
-def update(d, host):
-    conn = redis.StrictRedis(host)
+def update(d):
+    conn = _get_connection(decode_responses=False)
     key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
         e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
     current = conn.hgetall(key)
diff --git a/searx/webapp.py b/searx/webapp.py
index fd333afb6..dddebe6d6 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -503,7 +503,7 @@ def index():
             request.form['categories'] = ['general', 'videos', 'images']
         else:
             request.form['categories'] = [selected_category]
-        search_data = search(request, settings['redis']['host'])
+        search_data = search(request)
 
     except Exception as e:
         # log exception
@@ -846,12 +846,11 @@ def update_results():
     start_time = time.time()
     x = 0
     while not running.is_set():
-        host = settings['redis']['host']
-        queries = get_twenty_queries(x, host)
+        queries = get_twenty_queries(x)
         for query in queries:
             result_container = search.search(query)
             searchData = search.create_search_data(query, result_container)
-            search_database.update(searchData, host)
+            search_database.update(searchData)
             if running.is_set():
                 return
         x += 20
-- 
GitLab


From 6d400c45c172080dc8d24db47307aa7169aafd5b Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Mon, 1 Apr 2019 23:53:35 +0200
Subject: [PATCH 16/32] Unify the redis key generation

SearchData is composed by SearchQuery field, so we use inheritance
in order to use the same field naming. In that way we can use the an
uniq way to build our redis key.
---
 searx/query.py           |  6 +++---
 searx/results.py         | 12 ++++--------
 searx/search.py          |  2 +-
 searx/search_database.py | 32 ++++++++++++++++++++------------
 4 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/searx/query.py b/searx/query.py
index fdba2e092..89fe0607b 100644
--- a/searx/query.py
+++ b/searx/query.py
@@ -158,14 +158,14 @@ class RawTextQuery(object):
         return ''.join(self.query_parts)
 
 
-class SearchQuery(object):
+class SearchQuery:
     """container for all the search parameters (query, language, etc...)"""
 
-    def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):
+    def __init__(self, query, engines, categories, language, safesearch, pageno, time_range):
         self.query = query
         self.engines = engines
         self.categories = categories
-        self.lang = lang
+        self.language = language
         self.safesearch = safesearch
         self.pageno = pageno
         self.time_range = time_range
diff --git a/searx/results.py b/searx/results.py
index 973e5bf82..f655544da 100644
--- a/searx/results.py
+++ b/searx/results.py
@@ -4,6 +4,7 @@ from operator import itemgetter
 from threading import RLock
 from searx.engines import engines
 from searx.url_utils import urlparse, unquote
+from searx.query import SearchQuery
 
 
 CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)
@@ -319,16 +320,11 @@ class ResultContainer(object):
         self.unresponsive_engines.add(engine_error)
 
 
-class SearchData(object):
+class SearchData(SearchQuery):
     def __init__(self, search_query, results, paging,
                  results_number, answers, corrections, infoboxes, suggestions, unresponsive_engines):
-        self.categories = search_query.categories
-        self.query = search_query.query
-        self.pageno = search_query.pageno
-        self.safe_search = search_query.safesearch
-        self.language = search_query.lang
-        self.time_range = search_query.time_range
-        self.engines = search_query.engines
+        super().__init__(**search_query.__dict__)
+
         self.results = results
         self.paging = paging
         self.results_number = results_number
diff --git a/searx/search.py b/searx/search.py
index 2742c5195..118eddf1f 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -268,7 +268,7 @@ class Search(object):
             if hasattr(engine, 'language') and engine.language:
                 request_params['language'] = engine.language
             else:
-                request_params['language'] = search_query.lang
+                request_params['language'] = search_query.language
 
             # 0 = None, 1 = Moderate, 2 = Strict
             request_params['safesearch'] = search_query.safesearch
diff --git a/searx/search_database.py b/searx/search_database.py
index f9dd7d7e4..7540b68b4 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -12,18 +12,28 @@ from searx.results import SearchData
 from searx import settings
 
 
+def make_key(q):
+    if q.time_range is None:
+        q.time_range = ""
+
+    return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
+        e(q.query),
+        je(q.engines),
+        q.categories[0],
+        q.language,
+        q.safesearch,
+        q.pageno,
+        q.time_range,
+    )
+
+
 def _get_connection(decode_responses=True):
     return redis.StrictRedis(host=settings['redis']['host'], decode_responses=decode_responses)
 
 
 def read(q):
-    time_range = q.time_range
-    if q.time_range is None:
-        q.time_range = ""
-
     conn = _get_connection()
-    key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
-        e(q.query), je(q.engines), q.categories[0], q.lang, q.safesearch, q.pageno, time_range)
+    key = make_key(q)
     response = conn.hgetall(key)
     if not response:
         return None
@@ -37,10 +47,9 @@ def read(q):
 
 def save(d):
     conn = _get_connection()
-    key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
-        e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
+    key = make_key(d)
     mapping = {
-        'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safe_search': d.safe_search,
+        'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safesearch': d.safesearch,
         'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),
         'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),
         'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),
@@ -64,7 +73,7 @@ def get_twenty_queries(x):
     output = pipe.execute()
     for row in output:
         result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],
-                                  int(row['safe_search']), int(row['pageno']), row['time_range']))
+                                  int(row['safesearch']), int(row['pageno']), row['time_range']))
 
     return result
 
@@ -95,8 +104,7 @@ def jds(coded):
 
 def update(d):
     conn = _get_connection(decode_responses=False)
-    key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
-        e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
+    key = make_key(d)
     current = conn.hgetall(key)
     current.update({
         'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,
-- 
GitLab


From 43c2b056131af3bbdd3ec4b876ea0dc6a195b33d Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Thu, 11 Apr 2019 22:22:37 +0200
Subject: [PATCH 17/32] Add dockerignore

---
 .dockerignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..2b29f2764
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+tests
-- 
GitLab


From 38118ff8bc97afc268dad9fedf3a1bf6d6bb1a6d Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Wed, 3 Apr 2019 09:21:22 +0200
Subject: [PATCH 18/32] Update redis package and use pickle for serialize data

The new redis version is more strict about the kind of object
to store, we cannot use the same way that with the version 2.X.
In redis we want to store the SearchData class. Instead of
store field one by one we store the full serialized object with pickle.

Pickle is not very secure but we dont care for now because the redis
storage will be deploy in the same host that spot service. In a second time
we can use another way to serialize SearchData with msgpack and mashumaro
for example.
---
 requirements.txt          |  2 +-
 searx/search_database.py  | 97 ++++++++++++++-------------------------
 searx/webapp.py           | 14 +++---
 tests/unit/test_webapp.py |  2 +-
 4 files changed, 43 insertions(+), 72 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 546b8b761..fc7eac005 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-redis==2.10.6
+redis==3.2.1
 certifi==2017.11.5
 flask==1.0.2
 flask-babel==0.11.2
diff --git a/searx/search_database.py b/searx/search_database.py
index 7540b68b4..29da205f4 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -1,15 +1,8 @@
-import json
-import threading
-import urllib.parse
-
 import redis
+import pickle
 
 from searx import settings
-from searx.plugins import plugins
 from searx.query import SearchQuery
-from searx.url_utils import urlparse
-from searx.results import SearchData
-from searx import settings
 
 
 def make_key(q):
@@ -17,8 +10,8 @@ def make_key(q):
         q.time_range = ""
 
     return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
-        e(q.query),
-        je(q.engines),
+        q.query,
+        q.engines,
         q.categories[0],
         q.language,
         q.safesearch,
@@ -27,36 +20,25 @@ def make_key(q):
     )
 
 
-def _get_connection(decode_responses=True):
-    return redis.StrictRedis(host=settings['redis']['host'], decode_responses=decode_responses)
+def _get_connection():
+    return redis.Redis(host=settings['redis']['host'])
 
 
 def read(q):
     conn = _get_connection()
     key = make_key(q)
-    response = conn.hgetall(key)
+    response = conn.get(key)
     if not response:
         return None
-    results = jd(response['results'])
-    for result in results:
-        result['parsed_url'] = urlparse(result['url'])
-    return SearchData(q, results, response['paging'], int(response['results_number']),
-                      jds(response['answers']), jds(response['corrections']), jd(response['infoboxes']),
-                      jds(response['suggestions']), jds(response['unresponsive_engines']))
+    return pickle.loads(response)
 
 
 def save(d):
     conn = _get_connection()
     key = make_key(d)
-    mapping = {
-        'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safesearch': d.safesearch,
-        'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),
-        'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),
-        'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),
-        'unresponsive_engines': jes(d.unresponsive_engines)
-    }
-    conn.zadd('SEARCH_HISTORY_KEYS', conn.incr('SEARCH_HISTORY_INDEX'), key)
-    conn.hmset(key, mapping)
+    history = conn.incr("SEARCH_HISTORY_INDEX")
+    conn.zadd("SEARCH_HISTORY_KEYS", {key: history})
+    conn.set(key, pickle.dumps(d, protocol=4))
 
 
 def get_twenty_queries(x):
@@ -69,46 +51,35 @@ def get_twenty_queries(x):
 
     pipe = conn.pipeline()
     for key in keys:
-        pipe.hgetall(key)
+        pipe.get(key)
     output = pipe.execute()
     for row in output:
-        result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],
-                                  int(row['safesearch']), int(row['pageno']), row['time_range']))
+        row = pickle.loads(row)
+        result.append(
+            SearchQuery(
+                row.query,
+                row.engines,
+                row.categories,
+                row.language,
+                row.safesearch,
+                row.pageno,
+                row.time_range,
+            )
+        )
 
     return result
 
 
-def e(obj):
-    return urllib.parse.quote_plus(obj)
-
-
-def d(coded):
-    return urllib.parse.unquote_plus(coded)
-
-
-def je(obj):
-    return e(json.dumps(obj))
-
-
-def jd(coded):
-    return json.loads(d(coded))
-
-
-def jes(set):
-    return je(list(set))
-
-
-def jds(coded):
-    return jd(coded)
-
-
 def update(d):
-    conn = _get_connection(decode_responses=False)
+    conn = _get_connection()
     key = make_key(d)
-    current = conn.hgetall(key)
-    current.update({
-        'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,
-        'answers': jes(d.answers), 'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes),
-        'suggestions': jes(d.suggestions), 'unresponsive_engines': jes(d.unresponsive_engines)
-    })
-    conn.hmset(key, current)
+    current = read(d)
+    current.results = d.results
+    current.paging = d.paging
+    current.results_number = d.results_number
+    current.answers = d.answers
+    current.corrections = d.corrections
+    current.infoboxes = d.infoboxes
+    current.suggestions = d.suggestions
+    current.unresponsive_engines = d.unresponsive_engines
+    conn.set(key, pickle.dumps(current, protocol=4))
diff --git a/searx/webapp.py b/searx/webapp.py
index dddebe6d6..f6f9c0063 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -515,27 +515,27 @@ def index():
         else:
             return index_error(e, output), 500
 
+    results_copy = copy.deepcopy(search_data.results)
     if is_general_first_page:
-        result_copy = copy.copy(search_data.results)
-        for res in result_copy:
+        for res in results_copy:
             if res.get('category') == 'images':
                 if len(images) < 5:
                     images.append(res)
-                search_data.results.remove(res)
+                results_copy.remove(res)
             elif res.get('category') == 'videos':
                 if len(videos) < 2:
                     videos.append(res)
-                search_data.results.remove(res)
+                results_copy.remove(res)
             elif res.get('category') is None:
-                search_data.results.remove(res)
+                results_copy.remove(res)
 
     # output
-    config_results(search_data.results, search_data.query)
+    config_results(results_copy, search_data.query)
     config_results(images, search_data.query)
     config_results(videos, search_data.query)
 
     response = dict(
-        results=search_data.results,
+        results=results_copy,
         q=search_data.query,
         selected_category=selected_category,
         pageno=search_data.pageno,
diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py
index 2b95f63e7..8198b5980 100644
--- a/tests/unit/test_webapp.py
+++ b/tests/unit/test_webapp.py
@@ -63,7 +63,7 @@ class ViewsTestCase(TestCase):
         self.assertEqual(result.status_code, 200)
         self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)
 
-    @patch('redis.StrictRedis', mock_strict_redis_client)
+    @patch('redis.Redis', mock_strict_redis_client)
     def test_index_html(self):
         result = self.app.post('/', data={'q': 'test'})
         self.assertIn(
-- 
GitLab


From 7721632c7fe154e90b7bee70ebe5a129d26000e1 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Thu, 11 Apr 2019 23:43:04 +0200
Subject: [PATCH 19/32] Add cache interface

In order to use another cache tool or to use spot without cache
system.
---
 searx/__init__.py        |   1 +
 searx/search.py          |  40 +++++-----
 searx/search_database.py | 158 ++++++++++++++++++++++-----------------
 searx/settings.yml       |   1 +
 searx/webapp.py          |  15 ++--
 5 files changed, 120 insertions(+), 95 deletions(-)

diff --git a/searx/__init__.py b/searx/__init__.py
index 9dc594431..7324c1e4b 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -89,6 +89,7 @@ if 'BASE_URL' in environ:
 if 'IMAGE_PROXY' in environ:
     settings['server']['image_proxy'] = environ['IMAGE_PROXY']
 if 'SEARX_REDIS_HOST' in environ:
+    settings['redis']['enable'] = True
     settings['redis']['host'] = environ['SEARX_REDIS_HOST']
 if 'HTTP_PROXY_URL' in environ:
     settings['proxies']['http'] = environ['HTTP_PROXY_URL']
diff --git a/searx/search.py b/searx/search.py
index 118eddf1f..b208908c5 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -185,27 +185,25 @@ def default_request_params():
     }
 
 
-def search(request):
-    """ Entry point to perform search request on engines
-    """
-    search = Search()
-    search_query = search.get_search_query_from_webapp(request.preferences, request.form)
-    searchData = search_database.read(search_query)
-    if searchData is None:
-        result_container = search.search(search_query)
-        searchData = search.create_search_data(search_query, result_container)
-        threading.Thread(
-            target=search_database.save,
-            args=(searchData,),
-            name='save_search_' + str(searchData)
-        ).start()
-
-    search.search_with_plugins(request, searchData)
-    return searchData
-
-
-class Search(object):
-    """Search information container"""
+class Search:
+    """Search information manager"""
+
+    def __init__(self, cachecls=search_database.CacheInterface):
+        self.cache = cachecls()
+
+    def __call__(self, request):
+        """ Entry point to perform search request on engines
+        """
+        search_query = self.get_search_query_from_webapp(request.preferences, request.form)
+        searchData = self.cache.read(search_query)
+
+        if searchData is None:
+            result_container = self.search(search_query)
+            searchData = self.create_search_data(search_query, result_container)
+            self.cache.save(searchData)
+
+        self.search_with_plugins(request, searchData)
+        return searchData
 
     def search(self, search_query):
         """ do search-request
diff --git a/searx/search_database.py b/searx/search_database.py
index 29da205f4..e50fbde24 100644
--- a/searx/search_database.py
+++ b/searx/search_database.py
@@ -1,3 +1,4 @@
+import threading
 import redis
 import pickle
 
@@ -5,81 +6,104 @@ from searx import settings
 from searx.query import SearchQuery
 
 
-def make_key(q):
-    if q.time_range is None:
-        q.time_range = ""
+class CacheInterface:
+    """ Cache interface to store SearchData object
+    """
+    def read(self, q):
+        pass
 
-    return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
-        q.query,
-        q.engines,
-        q.categories[0],
-        q.language,
-        q.safesearch,
-        q.pageno,
-        q.time_range,
-    )
+    def save(self, d):
+        pass
 
+    def update(self, d):
+        pass
 
-def _get_connection():
-    return redis.Redis(host=settings['redis']['host'])
+    def get_twenty_queries(self, x):
+        return []
 
 
-def read(q):
-    conn = _get_connection()
-    key = make_key(q)
-    response = conn.get(key)
-    if not response:
-        return None
-    return pickle.loads(response)
+class RedisCache(CacheInterface):
+    def __init__(self):
+        self.pool = redis.ConnectionPool(host=settings['redis']['host'])
+        self.running = threading.Event()
 
+    def make_key(self, q):
+        if q.time_range is None:
+            q.time_range = ""
 
-def save(d):
-    conn = _get_connection()
-    key = make_key(d)
-    history = conn.incr("SEARCH_HISTORY_INDEX")
-    conn.zadd("SEARCH_HISTORY_KEYS", {key: history})
-    conn.set(key, pickle.dumps(d, protocol=4))
-
+        return "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
+            q.query,
+            q.engines,
+            q.categories[0],
+            q.language,
+            q.safesearch,
+            q.pageno,
+            q.time_range,
+        )
 
-def get_twenty_queries(x):
-    result = []
+    def _get_connection(self):
+        return redis.Redis(connection_pool=self.pool)
+
+    def read(self, q):
+        conn = self._get_connection()
+        key = self.make_key(q)
+        response = conn.get(key)
+        if not response:
+            return None
+        return pickle.loads(response)
+
+    def _save(self, d):
+        conn = self._get_connection()
+        key = self.make_key(d)
+        history = conn.incr("SEARCH_HISTORY_INDEX")
+        conn.zadd("SEARCH_HISTORY_KEYS", {key: history})
+        conn.set(key, pickle.dumps(d, protocol=4))
+
+    def save(self, d):
+        threading.Thread(
+            target=self._save,
+            args=(d,),
+            name='save_search_' + str(d)
+        ).start()
+
+    def get_twenty_queries(self, x):
+        result = []
+
+        conn = self._get_connection()
+        keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
+        if not keys:
+            return result
+
+        pipe = conn.pipeline()
+        for key in keys:
+            pipe.get(key)
+        output = pipe.execute()
+        for row in output:
+            row = pickle.loads(row)
+            result.append(
+                SearchQuery(
+                    row.query,
+                    row.engines,
+                    row.categories,
+                    row.language,
+                    row.safesearch,
+                    row.pageno,
+                    row.time_range,
+                )
+            )
 
-    conn = _get_connection()
-    keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
-    if not keys:
         return result
 
-    pipe = conn.pipeline()
-    for key in keys:
-        pipe.get(key)
-    output = pipe.execute()
-    for row in output:
-        row = pickle.loads(row)
-        result.append(
-            SearchQuery(
-                row.query,
-                row.engines,
-                row.categories,
-                row.language,
-                row.safesearch,
-                row.pageno,
-                row.time_range,
-            )
-        )
-
-    return result
-
-
-def update(d):
-    conn = _get_connection()
-    key = make_key(d)
-    current = read(d)
-    current.results = d.results
-    current.paging = d.paging
-    current.results_number = d.results_number
-    current.answers = d.answers
-    current.corrections = d.corrections
-    current.infoboxes = d.infoboxes
-    current.suggestions = d.suggestions
-    current.unresponsive_engines = d.unresponsive_engines
-    conn.set(key, pickle.dumps(current, protocol=4))
+    def update(self, d):
+        conn = self._get_connection()
+        key = self.make_key(d)
+        current = self.read(d)
+        current.results = d.results
+        current.paging = d.paging
+        current.results_number = d.results_number
+        current.answers = d.answers
+        current.corrections = d.corrections
+        current.infoboxes = d.infoboxes
+        current.suggestions = d.suggestions
+        current.unresponsive_engines = d.unresponsive_engines
+        conn.set(key, pickle.dumps(current, protocol=4))
diff --git a/searx/settings.yml b/searx/settings.yml
index 929da6680..d511fa320 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -17,6 +17,7 @@ server:
     http_protocol_version : "1.0"  # 1.0 and 1.1 are supported
 
 redis:
+    enable: False
     host : "127.0.0.1"
     user : "searx"
     password : "password" # change this!
diff --git a/searx/webapp.py b/searx/webapp.py
index f6f9c0063..84f9a504e 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -33,7 +33,7 @@ import copy
 
 import requests
 
-from searx import logger, search_database
+from searx import logger
 
 logger = logger.getChild('webapp')
 
@@ -68,7 +68,8 @@ from searx.utils import (
 )
 from searx.version import VERSION_STRING
 from searx.languages import language_codes as languages
-from searx.search import Search, search
+from searx.search import Search
+from searx.search_database import RedisCache
 from searx.query import RawTextQuery
 from searx.autocomplete import searx_bang, backends as autocomplete_backends
 from searx.plugins import plugins
@@ -77,7 +78,6 @@ from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
 from searx.answerers import answerers
 from searx.url_utils import urlencode, urlparse, urljoin
 from searx.utils import new_hmac
-from searx.search_database import get_twenty_queries
 import threading
 
 # check if the pyopenssl package is installed.
@@ -131,6 +131,8 @@ if not searx_debug \
 
 babel = Babel(app)
 
+search = Search(RedisCache) if settings["redis"]["enable"] else Search()
+
 rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
                'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
 
@@ -842,18 +844,17 @@ def wait_updating(start_time):
 
 
 def update_results():
-    search = Search()
     start_time = time.time()
     x = 0
     while not running.is_set():
-        queries = get_twenty_queries(x)
+        queries = search.cache.get_twenty_queries(x)
         for query in queries:
             result_container = search.search(query)
             searchData = search.create_search_data(query, result_container)
-            search_database.update(searchData)
+            search.cache.update(searchData)
             if running.is_set():
                 return
-        x += 20
+        x += len(queries)
         if len(queries) < 20:
             x = 0
             wait_updating(start_time)
-- 
GitLab


From 51ca35c780af57d3d716d976c75bfafd5e38fcb5 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Fri, 12 Apr 2019 00:22:39 +0200
Subject: [PATCH 20/32] Fix docker-compose warning

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index ad66e647e..6e73da0f0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,5 +8,5 @@ services:
     build: .
     environment:
       SEARX_REDIS_HOST: redis
-      VIRTUAL_HOST: ${SPOT_HOSTNAME}
+      VIRTUAL_HOST: ${SPOT_HOSTNAME:-spot}
       SEARX_LOGGER: INFO
-- 
GitLab


From 1478a0e88067a8d2b905bb262190643d69eb7df4 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Sat, 30 Mar 2019 00:05:55 +0100
Subject: [PATCH 21/32] Update CI with docker job

---
 .gitlab-ci.yml | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 47b51fadf..287ad22bd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,12 +1,20 @@
 image: $CI_REGISTRY_IMAGE/env:latest
 
 stages:
+ - check
  - build
  - test
  - report
  - deploy
 
-front-end:
+python:
+  stage: check
+  before_script:
+    - ./manage.sh update_dev_packages
+  script:
+    - ./manage.sh pep8_check
+
+build:web:
   stage: build
   before_script:
     - ./manage.sh npm_packages
@@ -16,14 +24,18 @@ front-end:
     - ./manage.sh styles
     - ./manage.sh grunt_build
 
-coding-rules:
+build:docker:
   stage: build
   before_script:
-    - ./manage.sh update_dev_packages
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - ./manage.sh pep8_check
+    - docker build -t $CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-latest} .
+    - docker push $CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-latest}
+  only:
+    - master
+    - tags
 
-unit-test:
+test:unit:
   stage: test
   before_script:
     - ./manage.sh update_dev_packages
@@ -34,7 +46,7 @@ unit-test:
       - coverage
     expire_in: 1 hour
 
-functional-test:
+test:functional:
   stage: test
   image: docker:stable
   services:
@@ -57,11 +69,11 @@ coverage:
   script:
     - ./manage.sh coverage
   dependencies:
-    - unit-test
-    - functional-test
+    - test:unit
+    - test:functional
   coverage: '/TOTAL.*\s+(\d+%)$/'
 
-deploy-test:
+deploy:test:
   image: docker:stable
   stage: deploy
   only:
-- 
GitLab


From a9e2a3a01c19dfcb340c211a6590fe4948301803 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Sat, 30 Mar 2019 00:00:13 +0100
Subject: [PATCH 22/32] Cleanup and update dependencies

---
 Dockerfile        |  5 +----
 requirements.txt  | 15 ++++++---------
 searx/__init__.py | 15 +--------------
 searx/webapp.py   | 23 +++++------------------
 4 files changed, 13 insertions(+), 45 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 70fb0323b..40ac3eb32 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,9 +4,7 @@ RUN apk add \
  git \
  build-base \
  libxml2-dev \
- libxslt-dev \
- libffi-dev \
- openssl-dev
+ libxslt-dev
 
 # Only to use the docker cache and optimize the build time
 WORKDIR /src
@@ -24,7 +22,6 @@ LABEL description="A privacy-respecting, hackable metasearch engine."
 RUN apk add \
  ca-certificates \
  libxslt \
- openssl \
 && pip install coverage
 
 COPY --from=builder /install/ /usr/local/
diff --git a/requirements.txt b/requirements.txt
index fc7eac005..ba135f738 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,8 @@
-redis==3.2.1
-certifi==2017.11.5
 flask==1.0.2
 flask-babel==0.11.2
-lxml==4.2.3
-idna==2.7
-pygments==2.1.3
-pyopenssl==18.0.0
-python-dateutil==2.7.3
-pyyaml==3.13
-requests[socks]==2.19.1
+lxml==4.3.3
+pygments==2.3.1
+python-dateutil==2.8.0
+pyyaml==5.1
+requests[socks]==2.21.0
+redis==3.2.1
diff --git a/searx/__init__.py b/searx/__init__.py
index 7324c1e4b..d0bb7126d 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -15,18 +15,11 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-import certifi
 import logging
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
 from io import open
-from ssl import OPENSSL_VERSION_INFO, OPENSSL_VERSION
-try:
-    from yaml import load
-except ImportError:
-    from sys import exit, stderr
-    stderr.write('[E] install pyyaml\n')
-    exit(2)
+from yaml import load
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
@@ -73,12 +66,6 @@ logging.basicConfig(level=getattr(logging, searx_loglevel))
 
 logger = logging.getLogger('searx')
 logger.debug('read configuration from %s', settings_path)
-# Workaround for openssl versions <1.0.2
-# https://github.com/certifi/python-certifi/issues/26
-if OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):
-    if hasattr(certifi, 'old_where'):
-        environ['REQUESTS_CA_BUNDLE'] = certifi.old_where()
-    logger.warning('You are using an old openssl version({0}), please upgrade above 1.0.2!'.format(OPENSSL_VERSION))
 
 logger.info('Initialisation done')
 
diff --git a/searx/webapp.py b/searx/webapp.py
index 84f9a504e..d7e773829 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -37,16 +37,11 @@ from searx import logger
 
 logger = logger.getChild('webapp')
 
-try:
-    from pygments import highlight
-    from pygments.lexers import get_lexer_by_name
-    from pygments.util import ClassNotFound
-    from pygments.formatters import HtmlFormatter
-except ImportError:
-    logger.critical("cannot import dependency: pygments")
-    from sys import exit
-
-    exit(1)
+from pygments import highlight
+from pygments.lexers import get_lexer_by_name
+from pygments.util import ClassNotFound
+from pygments.formatters import HtmlFormatter
+
 from html import escape
 from datetime import datetime, timedelta
 from werkzeug.contrib.fixers import ProxyFix
@@ -80,14 +75,6 @@ from searx.url_utils import urlencode, urlparse, urljoin
 from searx.utils import new_hmac
 import threading
 
-# check if the pyopenssl package is installed.
-# It is needed for SSL connection without trouble, see #298
-try:
-    import OpenSSL.SSL  # NOQA
-except ImportError:
-    logger.critical("The pyopenssl package has to be installed.\n"
-                    "Some HTTPS connections will fail")
-
 from io import StringIO
 
 # serve pages with HTTP/1.1
-- 
GitLab


From 6e676cc0d25c38106e0256e406d9b3e9041f6dfd Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Sat, 30 Mar 2019 01:20:07 +0100
Subject: [PATCH 23/32] Introduce flake8 check for main module

---
 manage.sh                 |  1 +
 requirements-dev.txt      |  3 ++-
 searx/engines/arxiv.py    |  2 +-
 searx/engines/pubmed.py   |  2 +-
 searx/engines/wikidata.py |  2 +-
 searx/query.py            | 30 ++++++++++++++--------------
 searx/utils.py            | 11 ++++++-----
 searx/webapp.py           | 41 +++++++++++++++++----------------------
 8 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/manage.sh b/manage.sh
index 47da7b36e..1d21b5886 100755
--- a/manage.sh
+++ b/manage.sh
@@ -38,6 +38,7 @@ pep8_check() {
     #  W503 line break before binary operator
     #  E722 do not use bare 'except'
     pycodestyle --exclude=searx/static --max-line-length=120 --ignore "E402,W503,E722" "$SEARX_DIR" "$BASE_DIR/tests"
+    flake8 --ignore=E722 $SEARX_DIR/*.py
 }
 
 unit_tests() {
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 1de575509..695f9e183 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,6 +1,7 @@
 babel==2.3.4
 mock==2.0.0
-pycodestyle==2.4.0
+pycodestyle==2.5.0
+flake8==3.7.7
 mockredispy==2.9.3
 pytest==4.1.0
 pytest-cov==2.6.1
diff --git a/searx/engines/arxiv.py b/searx/engines/arxiv.py
index 5ef84f0c1..84e844fbb 100644
--- a/searx/engines/arxiv.py
+++ b/searx/engines/arxiv.py
@@ -61,7 +61,7 @@ def response(resp):
             content = content_string.format(doi_content="", abstract_content=abstract)
 
         if len(content) > 300:
-                    content = content[0:300] + "..."
+            content = content[0:300] + "..."
         # TODO: center snippet on query term
 
         publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ')
diff --git a/searx/engines/pubmed.py b/searx/engines/pubmed.py
index 055f09226..82a37a564 100644
--- a/searx/engines/pubmed.py
+++ b/searx/engines/pubmed.py
@@ -81,7 +81,7 @@ def response(resp):
             pass
 
         if len(content) > 300:
-                    content = content[0:300] + "..."
+            content = content[0:300] + "..."
         # TODO: center snippet on query term
 
         res_dict = {'url': url,
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index 047f1ba77..81cf262c5 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -374,7 +374,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
 
     # wiki links don't have property in wikidata page
     if link_type and 'wiki' in link_type:
-            links.append(get_wikilink(result, link_type))
+        links.append(get_wikilink(result, link_type))
     else:
         dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))
         if dom_element:
diff --git a/searx/query.py b/searx/query.py
index 89fe0607b..1dde7f329 100644
--- a/searx/query.py
+++ b/searx/query.py
@@ -22,7 +22,6 @@ from searx.engines import (
     categories, engines, engine_shortcuts
 )
 import re
-import sys
 
 
 VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')
@@ -78,20 +77,21 @@ class RawTextQuery(object):
 
                     # if correct language-code is found
                     # set it as new search-language
-                    if (lang == lang_id
-                        or lang == lang_name
-                        or lang == english_name
-                        or lang.replace('-', ' ') == country)\
-                       and lang not in self.languages:
-                            parse_next = True
-                            lang_parts = lang_id.split('-')
-                            if len(lang_parts) == 2:
-                                self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())
-                            else:
-                                self.languages.append(lang_id)
-                            # to ensure best match (first match is not necessarily the best one)
-                            if lang == lang_id:
-                                break
+                    if (
+                        lang in [lang_id, lang_name, english_name]
+                        or lang.replace("-", " ") == country  # noqa
+                    ) and lang not in self.languages:
+                        parse_next = True
+                        lang_parts = lang_id.split("-")
+                        if len(lang_parts) == 2:
+                            self.languages.append(
+                                lang_parts[0] + "-" + lang_parts[1].upper()
+                            )
+                        else:
+                            self.languages.append(lang_id)
+                        # to ensure best match (first match is not necessarily the best one)
+                        if lang == lang_id:
+                            break
 
                 # user may set a valid, yet not selectable language
                 if VALID_LANGUAGE_CODE.match(lang):
diff --git a/searx/utils.py b/searx/utils.py
index d1a28ea24..a0df43a7f 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -11,7 +11,7 @@ from codecs import getincrementalencoder
 from imp import load_source
 from numbers import Number
 from os.path import splitext, join
-from io import open
+from pathlib import Path
 from random import choice
 import sys
 import json
@@ -19,7 +19,6 @@ import json
 from searx import settings
 from searx.version import VERSION_STRING
 from searx.languages import language_codes
-from searx import settings
 from searx import logger
 
 from io import StringIO
@@ -31,8 +30,10 @@ logger = logger.getChild('utils')
 blocked_tags = ('script',
                 'style')
 
-useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
-                             + "/data/useragents.json", 'r', encoding='utf-8').read())
+useragents = json.load(open(
+    Path(__file__).parent / "data" / "useragents.json",
+    encoding='utf-8')
+)
 
 
 def searx_useragent():
@@ -161,7 +162,7 @@ def get_resources_directory(searx_directory, subdirectory, resources_directory):
     if not resources_directory:
         resources_directory = os.path.join(searx_directory, subdirectory)
     if not os.path.isdir(resources_directory):
-        raise Exception(directory + " is not a directory")
+        raise Exception(resources_directory + " is not a directory")
     return resources_directory
 
 
diff --git a/searx/webapp.py b/searx/webapp.py
index d7e773829..8d837a870 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -17,17 +17,10 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
 '''
 
-if __name__ == '__main__':
-    from sys import path
-    from os.path import realpath, dirname
-
-    path.append(realpath(dirname(realpath(__file__)) + '/../'))
-
 import hashlib
 import hmac
 import json
 import os
-import sys
 import time
 import copy
 
@@ -35,7 +28,6 @@ import requests
 
 from searx import logger
 
-logger = logger.getChild('webapp')
 
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
@@ -75,11 +67,11 @@ from searx.url_utils import urlencode, urlparse, urljoin
 from searx.utils import new_hmac
 import threading
 
-from io import StringIO
-
 # serve pages with HTTP/1.1
 from werkzeug.serving import WSGIRequestHandler
 
+logger = logger.getChild('webapp')
+
 WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
 
 # about static
@@ -306,10 +298,11 @@ def render(template_name, override_theme=None, **kwargs):
 
     if 'categories' not in kwargs:
         kwargs['categories'] = ['general']
-        kwargs['categories'].extend(x for x in
-                                    sorted(categories.keys())
-                                    if x != 'general'
-                                    and x in enabled_categories)
+        kwargs["categories"].extend(
+            x
+            for x in sorted(categories.keys())
+            if x != "general" and x in enabled_categories
+        )
 
     if 'all_categories' not in kwargs:
         kwargs['all_categories'] = ['general']
@@ -415,7 +408,7 @@ def pre_request():
     else:
         try:
             preferences.parse_dict(request.form)
-        except Exception as e:
+        except Exception:
             logger.exception('invalid settings')
             request.errors.append(gettext('Invalid settings'))
 
@@ -431,8 +424,9 @@ def pre_request():
     allowed_plugins = preferences.plugins.get_enabled()
     disabled_plugins = preferences.plugins.get_disabled()
     for plugin in plugins:
-        if ((plugin.default_on and plugin.id not in disabled_plugins)
-                or plugin.id in allowed_plugins):
+        if (
+            plugin.default_on and plugin.id not in disabled_plugins
+        ) or plugin.id in allowed_plugins:
             request.user_plugins.append(plugin)
 
 
@@ -631,7 +625,6 @@ def preferences():
 
     # render preferences
     image_proxy = request.preferences.get_value('image_proxy')
-    lang = request.preferences.get_value('language')
     disabled_engines = request.preferences.engines.get_disabled()
     allowed_plugins = request.preferences.plugins.get_enabled()
 
@@ -677,11 +670,13 @@ def preferences():
 
 
 def _is_selected_language_supported(engine, preferences):
-    language = preferences.get_value('language')
-    return (language == 'all'
-            or match_language(language,
-                              getattr(engine, 'supported_languages', []),
-                              getattr(engine, 'language_aliases', {}), None))
+    language = preferences.get_value("language")
+    return language == "all" or match_language(
+        language,
+        getattr(engine, "supported_languages", []),
+        getattr(engine, "language_aliases", {}),
+        None,
+    )
 
 
 @app.route('/image_proxy', methods=['GET'])
-- 
GitLab


From 2cc736bdea21aab0c7b1680622df9c1496526411 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Sat, 30 Mar 2019 09:14:08 +0100
Subject: [PATCH 24/32] Fix python -X dev warnings

---
 searx/__init__.py                 | 10 +++++++---
 searx/engines/__init__.py         | 10 +++++-----
 searx/engines/currency_convert.py | 11 +++++------
 searx/engines/json_engine.py      |  2 +-
 searx/utils.py                    | 10 ++++------
 searx/webapp.py                   |  2 +-
 6 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/searx/__init__.py b/searx/__init__.py
index d0bb7126d..5bddf9218 100644
--- a/searx/__init__.py
+++ b/searx/__init__.py
@@ -18,8 +18,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 import logging
 from os import environ
 from os.path import realpath, dirname, join, abspath, isfile
-from io import open
-from yaml import load
+
+import yaml
 
 searx_dir = abspath(dirname(__file__))
 engine_dir = dirname(realpath(__file__))
@@ -46,7 +46,11 @@ if not settings_path:
 
 # load settings
 with open(settings_path, 'r', encoding='utf-8') as settings_yaml:
-    settings = load(settings_yaml)
+    # XXX: docker-compose does not support yet yaml >= 5
+    if int(yaml.__version__.split('.')[0]) >= 5:
+        settings = yaml.load(settings_yaml, Loader=yaml.FullLoader)
+    else:
+        settings = yaml.load(settings_yaml)
 
 '''
 enable debug if
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index f32b57202..953fa8bf0 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -18,12 +18,11 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
 
 import sys
 import threading
-from os.path import realpath, dirname
-from io import open
+import json
+from pathlib import Path
 from babel.localedata import locale_identifiers
 from flask_babel import gettext
 from operator import itemgetter
-from json import loads
 from requests import get
 from searx import settings
 from searx import logger
@@ -32,13 +31,14 @@ from searx.utils import load_module, match_language
 
 logger = logger.getChild('engines')
 
-engine_dir = dirname(realpath(__file__))
+engine_dir = Path(__file__).parent
 
 engines = {}
 
 categories = {'general': []}
 
-languages = loads(open(engine_dir + '/../data/engines_languages.json', 'r', encoding='utf-8').read())
+with open(engine_dir.parent / "data" / "engines_languages.json", encoding='utf-8') as fd:
+    languages = json.load(fd)
 babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]
                for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())]
 
diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py
index 9424d7d5e..2a2a3c35d 100644
--- a/searx/engines/currency_convert.py
+++ b/searx/engines/currency_convert.py
@@ -1,10 +1,9 @@
 import json
 import re
-import os
 import sys
 import unicodedata
 
-from io import open
+from pathlib import Path
 from datetime import datetime
 
 
@@ -88,10 +87,10 @@ def response(resp):
 def load():
     global db
 
-    current_dir = os.path.dirname(os.path.realpath(__file__))
-    json_data = open(current_dir + "/../data/currencies.json", 'r', encoding='utf-8').read()
-
-    db = json.loads(json_data)
+    with open(
+        Path(__file__).parent.parent / "data" / "currencies.json", encoding='utf-8'
+    ) as fd:
+        db = json.load(fd)
 
 
 load()
diff --git a/searx/engines/json_engine.py b/searx/engines/json_engine.py
index 550947ab6..7a48bc8ce 100644
--- a/searx/engines/json_engine.py
+++ b/searx/engines/json_engine.py
@@ -1,4 +1,4 @@
-from collections import Iterable
+from collections.abc import Iterable
 from json import loads
 from searx.url_utils import urlencode
 from searx.utils import to_string
diff --git a/searx/utils.py b/searx/utils.py
index a0df43a7f..a48dc36bf 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -3,12 +3,12 @@ import hashlib
 import hmac
 import os
 import re
+from importlib.machinery import SourceFileLoader
 
 from babel.core import get_global
 from babel.dates import format_date
 from babel import UnknownLocaleError
 from codecs import getincrementalencoder
-from imp import load_source
 from numbers import Number
 from os.path import splitext, join
 from pathlib import Path
@@ -30,10 +30,8 @@ logger = logger.getChild('utils')
 blocked_tags = ('script',
                 'style')
 
-useragents = json.load(open(
-    Path(__file__).parent / "data" / "useragents.json",
-    encoding='utf-8')
-)
+with open(Path(__file__).parent / "data" / "useragents.json", encoding='utf-8') as fd:
+    useragents = json.load(fd)
 
 
 def searx_useragent():
@@ -357,7 +355,7 @@ def load_module(filename, module_dir):
     if modname in sys.modules:
         del sys.modules[modname]
     filepath = join(module_dir, filename)
-    module = load_source(modname, filepath)
+    module = SourceFileLoader(modname, filepath).load_module()
     module.name = modname
     return module
 
diff --git a/searx/webapp.py b/searx/webapp.py
index 8d837a870..22f8eb143 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -36,7 +36,7 @@ from pygments.formatters import HtmlFormatter
 
 from html import escape
 from datetime import datetime, timedelta
-from werkzeug.contrib.fixers import ProxyFix
+from werkzeug.middleware.proxy_fix import ProxyFix
 from flask import (
     Flask, request, render_template, url_for, Response, make_response,
     redirect, send_from_directory
-- 
GitLab


From 3e1c2153f75adbf3e5d9eab0b0d4e567893e8d64 Mon Sep 17 00:00:00 2001
From: Marc Abonce Seguin <marc-abonce@mailbox.org>
Date: Sat, 13 Apr 2019 00:17:32 -0500
Subject: [PATCH 25/32] [fix] duckduckgo images requests

---
 searx/engines/duckduckgo_images.py           | 11 +++++++----
 tests/unit/engines/test_duckduckgo_images.py |  2 +-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py
index 8c46ec92d..89924b71c 100644
--- a/searx/engines/duckduckgo_images.py
+++ b/searx/engines/duckduckgo_images.py
@@ -35,9 +35,12 @@ site_url = 'https://duckduckgo.com/?{query}&iar=images&iax=1&ia=images'
 
 # run query in site to get vqd number needed for requesting images
 # TODO: find a way to get this number without an extra request (is it a hash of the query?)
-def get_vqd(query):
-    res = get(site_url.format(query=urlencode({'q': query})))
+def get_vqd(query, headers):
+    query_url = site_url.format(query=urlencode({'q': query}))
+    res = get(query_url, headers=headers)
     content = res.text
+    if content.find('vqd=\'') == -1:
+        raise Exception('Request failed')
     vqd = content[content.find('vqd=\'') + 5:]
     vqd = vqd[:vqd.find('\'')]
     return vqd
@@ -47,7 +50,7 @@ def get_vqd(query):
 def request(query, params):
     # to avoid running actual external requests when testing
     if 'is_test' not in params:
-        vqd = get_vqd(query)
+        vqd = get_vqd(query, params['headers'])
     else:
         vqd = '12345'
 
@@ -74,7 +77,7 @@ def response(resp):
     try:
         res_json = loads(content)
     except:
-        return []
+        raise Exception('Cannot parse results')
 
     # parse results
     for result in res_json['results']:
diff --git a/tests/unit/engines/test_duckduckgo_images.py b/tests/unit/engines/test_duckduckgo_images.py
index 7a7657cd5..0d152bec1 100644
--- a/tests/unit/engines/test_duckduckgo_images.py
+++ b/tests/unit/engines/test_duckduckgo_images.py
@@ -41,7 +41,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):
         self.assertRaises(AttributeError, duckduckgo_images.response, '[]')
 
         response = mock.Mock(text='If this error persists, please let us know: ops@duckduckgo.com')
-        self.assertEqual(duckduckgo_images.response(response), [])
+        self.assertRaises(Exception, duckduckgo_images.response, response)
 
         json = u"""
         {
-- 
GitLab


From 1c6ab79b9fa79906909d068d51d33ffca1c6dc4f Mon Sep 17 00:00:00 2001
From: Nick Espig <nickespig@gmail.com>
Date: Fri, 12 Apr 2019 23:12:56 +0200
Subject: [PATCH 26/32] Fix google image search

- Because there is not full image url in the dom, we replace "image_url" with the same url as the "url" (url of source).
  See example HTML https://gist.github.com/Nachtalb/2dea8a4d2c723c49226ad9645838121f
- Remove unused import
- Fix google image search title
- Keep google image safe value up to date
---
 searx/engines/google_images.py | 38 +++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py
index 504831a10..66a543e57 100644
--- a/searx/engines/google_images.py
+++ b/searx/engines/google_images.py
@@ -11,7 +11,6 @@
 """
 
 from datetime import date, timedelta
-from json import loads
 from lxml import html
 from searx.url_utils import urlencode, urlparse, parse_qs
 
@@ -39,7 +38,6 @@ time_range_dict = {'day': 'd',
 # do search-request
 def request(query, params):
     search_options = {
-        'ijn': params['pageno'] - 1,
         'start': (params['pageno'] - 1) * number_of_results
     }
 
@@ -53,7 +51,7 @@ def request(query, params):
         search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
 
     if safesearch and params['safesearch']:
-        search_options['safe'] = 'on'
+        search_options['safe'] = 'active'
 
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       search_options=urlencode(search_options))
@@ -63,24 +61,30 @@ def request(query, params):
 
 # get response from search-request
 def response(resp):
+    dom = html.fromstring(resp.text)
+
     results = []
+    for element in dom.xpath('//div[@id="search"] //td'):
+        link = element.xpath('./a')[0]
 
-    dom = html.fromstring(resp.text)
+        google_url = urlparse(link.xpath('.//@href')[0])
+        query = parse_qs(google_url.query)
+        source_url = next(iter(query.get('q', [])), None)
 
-    # parse results
-    for img in dom.xpath('//a'):
-        r = {
-            'title': u' '.join(img.xpath('.//div[class="rg_ilmbg"]//text()')),
+        title_parts = element.xpath('./cite//following-sibling::*/text()')
+        title_parts.extend(element.xpath('./cite//following-sibling::text()')[:-1])
+
+        result = {
+            'title': ''.join(title_parts),
             'content': '',
             'template': 'images.html',
+            'url': source_url,
+            'img_src': source_url,
+            'thumbnail_src': next(iter(link.xpath('.//img //@src')), None)
         }
-        url = urlparse(img.xpath('.//@href')[0])
-        query = parse_qs(url.query)
-        r['url'] = query['imgrefurl'][0]
-        r['img_src'] = query['imgurl'][0]
-        r['thumbnail_src'] = r['img_src']
-        # append result
-        results.append(r)
-
-    # return results
+
+        if not source_url or not result['thumbnail_src']:
+            continue
+
+        results.append(result)
     return results
-- 
GitLab


From b44063755a44241c64b337287f7be537d8851d05 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Tue, 16 Apr 2019 09:35:03 +0200
Subject: [PATCH 27/32] Revert "Disable google image engine"

This reverts commit 63e8c8b10fca38581c82fab26987d82b5fd2b41d.
---
 searx/settings.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/searx/settings.yml b/searx/settings.yml
index edbe3ae18..25cfb13ce 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -294,7 +294,6 @@ engines:
   - name : google images
     engine : google_images
     shortcut : goi
-    disabled: True
 
   - name : google news
     engine : google_news
-- 
GitLab


From 812bac9d9649b5ba53a97a43c15b739692706afa Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Tue, 16 Apr 2019 09:35:03 +0200
Subject: [PATCH 28/32] Fix searx themes

---
 searx/webapp.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/searx/webapp.py b/searx/webapp.py
index 22f8eb143..23188fc83 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -312,7 +312,7 @@ def render(template_name, override_theme=None, **kwargs):
 
     if 'selected_categories' not in kwargs:
         kwargs['selected_categories'] = []
-        for arg in request.args:
+        for arg in request.form:
             if arg.startswith('category_'):
                 c = arg.split('_', 1)[1]
                 if c in categories:
@@ -473,6 +473,16 @@ def index():
             return jsonify({}), 204
         return render('index.html')
 
+    if request.form.get('category') is None:
+        category = None
+        for name, value in request.form.items():
+            if name.startswith('category_'):
+                category = name[9:]
+                if category in categories and value == "on":
+                    break
+        if category is not None:
+            request.form["category"] = category
+
     selected_category = request.form.get('category') or 'general'
     first_page = request.form.get('pageno')
     is_general_first_page = selected_category == 'general' and (first_page is None or first_page == u'1')
@@ -521,6 +531,7 @@ def index():
         results=results_copy,
         q=search_data.query,
         selected_category=selected_category,
+        selected_categories=[selected_category],
         pageno=search_data.pageno,
         time_range=search_data.time_range,
         number_of_results=format_decimal(search_data.results_number),
-- 
GitLab


From 116e3f9b1ef241668678ea81cf725f6bc41fb2a4 Mon Sep 17 00:00:00 2001
From: Nicolas Gelot <nicofonk@gmail.com>
Date: Thu, 18 Apr 2019 09:13:05 +0200
Subject: [PATCH 29/32] Add ability on eelo theme to switch to another

---
 searx/templates/eelo/preferences.html | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/searx/templates/eelo/preferences.html b/searx/templates/eelo/preferences.html
index 55b4eba47..e32160964 100644
--- a/searx/templates/eelo/preferences.html
+++ b/searx/templates/eelo/preferences.html
@@ -72,6 +72,24 @@
                     </select>
                 {% endcall %}
 
+                {% set theme_label = _('Themes') %}
+                {% set theme_info = _('Change searx layout') %}
+                {% call preferences_item(theme_info, theme_label, rtl) %}
+                    <select class="form-control custom-select" name="theme">
+                        {% for name in themes %}
+                        <option value="{{ name }}" {% if name == theme %}selected="selected"{% endif %}>{{ name }}</option>
+                        {% endfor %}
+                    </select>
+                {% endcall %}
+
+                {% call preferences_item(_('Choose style for this theme'), _('Style'), rtl) %}
+                    <select class="form-control custom-select" name='oscar-style'>
+                        <option value="logicodev" >Logicodev</option>
+                        <option value="pointhi" {% if preferences.get_value('oscar-style') == 'pointhi' %}selected="selected"{% endif %}>Pointhi</option>
+                        <option value="logicodev-dark" {% if preferences.get_value('oscar-style') == 'logicodev-dark' %}selected="selected"{% endif %}>Logicodev dark</option>
+                    </select>
+                {% endcall %}
+
                 {% set label = _('Results on new tabs') %}
                 {% set info = _('Open result links on new browser tabs') %}
                 {% call preferences_item(info, label, rtl) %}
-- 
GitLab


From 42560b7592b9d96a5e6d417e7d87b42117435c83 Mon Sep 17 00:00:00 2001
From: Romain Hunault <romain.hunault@e.email>
Date: Fri, 3 May 2019 09:03:57 +0000
Subject: [PATCH 30/32] Add support SSL on deployment

---
 .gitlab-ci.yml     | 3 ++-
 docker-compose.yml | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 287ad22bd..877b064fc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -94,6 +94,7 @@ deploy:test:
     - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
     - chmod 644 ~/.ssh/known_hosts
   script:
-    - ssh -2 $PUBLISH_USER@$PUBLISH_URL 'if [ ! "$(docker ps -q -f name=proxy)" ] ; then docker run -d -p 80:80 --net my-network my-spot_default -v /var/run/docker.sock:/tmp/docker.sock:ro --restart unless-stopped --name proxy jwilder/nginx-proxy ; elif [ ! "$(docker ps -q -f name=proxy -f status=exited)" ] ; then docker start proxy ; fi'
+    - ssh -2 $PUBLISH_USER@$PUBLISH_URL 'if [ ! "$(docker ps -q -f name=proxy)" ] ; then docker run -d -p 80:80 -p 443:443 --net my-spot_default -v /mnt/data/html:/usr/share/nginx/html -v /mnt/data/vhosts:/etc/nginx/vhost.d -v /mnt/data/certs:/etc/nginx/certs:ro -v /var/run/docker.sock:/tmp/docker.sock:ro --restart unless-stopped --name proxy jwilder/nginx-proxy ; elif [ ! "$(docker ps -q -f name=proxy -f status=exited)" ] ; then docker start proxy ; fi'
+    - ssh -2 $PUBLISH_USER@$PUBLISH_URL 'docker run --name letsencrypt-proxy --volumes-from proxy -v /mnt/data/certs:/etc/nginx/certs -v /var/run/docker.sock:/var/run/docker.sock:ro --net my-spot_default -d jrcs/letsencrypt-nginx-proxy-companion'
     - ssh -2 $PUBLISH_USER@$PUBLISH_URL "cd /root/my-spot/ && git fetch && git checkout $CI_COMMIT_SHA"
     - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME && cd /root/my-spot/ && docker-compose pull && docker-compose up -d --build --force-recreate"
diff --git a/docker-compose.yml b/docker-compose.yml
index 6e73da0f0..0e5197d14 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,4 +9,5 @@ services:
     environment:
       SEARX_REDIS_HOST: redis
       VIRTUAL_HOST: ${SPOT_HOSTNAME:-spot}
+      LETSENCRYPT_HOST: ${SPOT_HOSTNAME:-spot}
       SEARX_LOGGER: INFO
-- 
GitLab


From 485f07d867eb78c05a8c3c608532e4a6e6fb2b12 Mon Sep 17 00:00:00 2001
From: Romain Hunault <romain.hunault@e.email>
Date: Fri, 3 May 2019 12:03:10 +0000
Subject: [PATCH 31/32] Use docker-compose for SSL

---
 .gitlab-ci.yml     |  2 --
 docker-compose.yml | 27 +++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 877b064fc..8e2c3cedb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -94,7 +94,5 @@ deploy:test:
     - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
     - chmod 644 ~/.ssh/known_hosts
   script:
-    - ssh -2 $PUBLISH_USER@$PUBLISH_URL 'if [ ! "$(docker ps -q -f name=proxy)" ] ; then docker run -d -p 80:80 -p 443:443 --net my-spot_default -v /mnt/data/html:/usr/share/nginx/html -v /mnt/data/vhosts:/etc/nginx/vhost.d -v /mnt/data/certs:/etc/nginx/certs:ro -v /var/run/docker.sock:/tmp/docker.sock:ro --restart unless-stopped --name proxy jwilder/nginx-proxy ; elif [ ! "$(docker ps -q -f name=proxy -f status=exited)" ] ; then docker start proxy ; fi'
-    - ssh -2 $PUBLISH_USER@$PUBLISH_URL 'docker run --name letsencrypt-proxy --volumes-from proxy -v /mnt/data/certs:/etc/nginx/certs -v /var/run/docker.sock:/var/run/docker.sock:ro --net my-spot_default -d jrcs/letsencrypt-nginx-proxy-companion'
     - ssh -2 $PUBLISH_USER@$PUBLISH_URL "cd /root/my-spot/ && git fetch && git checkout $CI_COMMIT_SHA"
     - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME && cd /root/my-spot/ && docker-compose pull && docker-compose up -d --build --force-recreate"
diff --git a/docker-compose.yml b/docker-compose.yml
index 0e5197d14..a3a0da139 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,12 +2,39 @@ version: '3.6'
 
 services:
   redis:
+    restart: unless-stopped
     image: redis:5-alpine
 
   spot:
     build: .
+    restart: unless-stopped
     environment:
       SEARX_REDIS_HOST: redis
       VIRTUAL_HOST: ${SPOT_HOSTNAME:-spot}
       LETSENCRYPT_HOST: ${SPOT_HOSTNAME:-spot}
       SEARX_LOGGER: INFO
+
+  proxy:
+    image: jwilder/nginx-proxy:alpine
+    restart: unless-stopped
+    container_name: proxy
+    volumes:
+      - /mnt/data/html:/usr/share/nginx/html
+      - /mnt/data/vhosts:/etc/nginx/vhost.d
+      - /mnt/data/certs:/etc/nginx/certs:ro
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+
+  cert:
+    image: jrcs/letsencrypt-nginx-proxy-companion
+    restart: unless-stopped
+    environment:
+      NGINX_PROXY_CONTAINER: proxy
+    volumes:
+      - /mnt/data/html:/usr/share/nginx/html
+      - /mnt/data/vhosts:/etc/nginx/vhost.d
+      - /mnt/data/certs:/etc/nginx/certs
+      - /var/run/docker.sock:/var/run/docker.sock:ro
-- 
GitLab


From 03755fc707749649312d75133f8d48f1dcf56673 Mon Sep 17 00:00:00 2001
From: Romain Hunault <romain.hunault@eelo.io>
Date: Mon, 6 May 2019 14:37:08 +0200
Subject: [PATCH 32/32] Deploy in prod

---
 .gitlab-ci.yml | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8e2c3cedb..14426c6a5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -73,26 +73,43 @@ coverage:
     - test:functional
   coverage: '/TOTAL.*\s+(\d+%)$/'
 
-deploy:test:
+.deploy:template:
   image: docker:stable
   stage: deploy
   only:
     - branches
   when: manual
-  variables:
-    PUBLISH_USER: root
-    PUBLISH_URL: spot.test.ecloud.global
-    GIT_STRATEGY: none
-    SPOT_HOSTNAME: spot.test.ecloud.global
   dependencies: []
+  variables:
+    DEPLOY_FOLDER: /mnt/data
   before_script:
     - 'which ssh-agent || ( apk --update add openssh-client )'
     - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY_TEST" | tr -d '\r' | ssh-add - > /dev/null
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh
     - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
     - chmod 644 ~/.ssh/known_hosts
   script:
-    - ssh -2 $PUBLISH_USER@$PUBLISH_URL "cd /root/my-spot/ && git fetch && git checkout $CI_COMMIT_SHA"
-    - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME && cd /root/my-spot/ && docker-compose pull && docker-compose up -d --build --force-recreate"
+    - ssh -2 $PUBLISH_USER@$PUBLISH_URL "mkdir -p ${DEPLOY_FOLDER} && cd ${DEPLOY_FOLDER} && if [ ! -d ${CI_PROJECT_NAME} ] ; then git clone ${CI_PROJECT_URL}.git ; fi && cd ${CI_PROJECT_NAME} && git fetch && git checkout $CI_COMMIT_SHA"
+    - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME && cd ${DEPLOY_FOLDER}/${CI_PROJECT_NAME} && docker-compose pull && docker-compose up -d --build --force-recreate"
+
+test:
+  extends: .deploy:template
+  variables:
+    PUBLISH_USER: root
+    PUBLISH_URL: spot.test.ecloud.global
+    GIT_STRATEGY: none
+    SPOT_HOSTNAME: spot.test.ecloud.global
+    SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_TEST}
+
+prod:
+  extends: .deploy:template
+  only:
+    - master
+  variables:
+    PUBLISH_USER: root
+    PUBLISH_URL: spot.ecloud.global
+    GIT_STRATEGY: none
+    SPOT_HOSTNAME: spot.ecloud.global
+    SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_PROD}
-- 
GitLab