Merge branch 'nicofonk/gitlabci' into 'master'

fix tests and enable gitlab CI

See merge request e/cloud/my-spot!19

.gitlab-ci.yml

+image: $CI_REGISTRY_IMAGE/env:latest
+
+stages:
+ - build
+ - test
+
+front-end:
+  stage: build
+  before_script:
+    - ./manage.sh npm_packages
+    - ./manage.sh update_dev_packages
+  script:
+    - ./manage.sh locales
+    - ./manage.sh styles
+    - ./manage.sh grunt_build
+
+coding-rules:
+  stage: build
+  before_script:
+    - ./manage.sh update_dev_packages
+  script:
+    - ./manage.sh pep8_check
+
+unittest:
+  stage: test
+  before_script:
+    - ./manage.sh update_dev_packages
+  script:
+    - ./manage.sh unit_tests

Dockerfile.env

+FROM fedora
+
+RUN dnf install -y\
+ wget\
+ python2-pip\
+ npm\
+&& dnf groupinstall -y "Development Tools" \
+&& pip install pytest ipdb ipython \

manage.sh

@@ -110,7 +110,7 @@ tests() {
 #
 
 npm_path_setup() {
-    which npm || (printf 'Error: npm is not found\n'; exit 1)
+    which npm &>/dev/null || whereis npm &>/dev/null || (printf 'Error: npm is not found\n'; exit 1)
     export PATH="$(npm bin)":$PATH
 }
 
@@ -152,6 +152,7 @@ styles() {
 }
 
 grunt_build() {
+    npm_path_setup
     echo '[!] Grunt build : oscar theme'
     grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js"
     echo '[!] Grunt build : simple theme'

requirements-dev.txt

@@ -9,3 +9,4 @@ transifex-client==0.12.2
 unittest2==1.1.0
 zope.testrunner==4.5.1
 selenium==3.5.0
+mockredispy==2.9.3

searx/search_database.py

@@ -32,7 +32,8 @@ class SearchData(object):
 
 
 def _get_connection(host):
-    return redis.StrictRedis(host if host else settings['redis']['host'], decode_responses=True)
+    host = host if host else settings['redis']['host']
+    return redis.StrictRedis(host=host, decode_responses=True)
 
 
 def read(q, host):

tests/unit/test_plugins.py

@@ -6,8 +6,7 @@ from mock import Mock
 
 
 def get_search_mock(query, **kwargs):
-    return Mock(search_query=Mock(query=query, **kwargs),
-                result_container=Mock(answers=set()))
+    return Mock(query=query, answers=set(), **kwargs)
 
 
 class PluginStoreTest(SearxTestCase):
@@ -50,11 +49,11 @@ class SelfIPTest(SearxTestCase):
         request.headers.getlist.return_value = []
         search = get_search_mock(query=b'ip', pageno=1)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertTrue('127.0.0.1' in search.result_container.answers)
+        self.assertTrue('127.0.0.1' in search.answers)
 
         search = get_search_mock(query=b'ip', pageno=2)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertFalse('127.0.0.1' in search.result_container.answers)
+        self.assertFalse('127.0.0.1' in search.answers)
 
         # User agent test
         request = Mock(user_agent='Mock')
@@ -62,24 +61,24 @@ class SelfIPTest(SearxTestCase):
 
         search = get_search_mock(query=b'user-agent', pageno=1)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertTrue('Mock' in search.result_container.answers)
+        self.assertTrue('Mock' in search.answers)
 
         search = get_search_mock(query=b'user-agent', pageno=2)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertFalse('Mock' in search.result_container.answers)
+        self.assertFalse('Mock' in search.answers)
 
         search = get_search_mock(query=b'user-agent', pageno=1)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertTrue('Mock' in search.result_container.answers)
+        self.assertTrue('Mock' in search.answers)
 
         search = get_search_mock(query=b'user-agent', pageno=2)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertFalse('Mock' in search.result_container.answers)
+        self.assertFalse('Mock' in search.answers)
 
         search = get_search_mock(query=b'What is my User-Agent?', pageno=1)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertTrue('Mock' in search.result_container.answers)
+        self.assertTrue('Mock' in search.answers)
 
         search = get_search_mock(query=b'What is my User-Agent?', pageno=2)
         store.call(store.plugins, 'post_search', request, search)
-        self.assertFalse('Mock' in search.result_container.answers)
+        self.assertFalse('Mock' in search.answers)

tests/unit/test_webapp.py

 # -*- coding: utf-8 -*-
 
 import json
-from mock import Mock
+from mock import Mock, patch
+from mockredis import mock_strict_redis_client
 from searx import webapp
 from searx.testing import SearxTestCase
 from searx.search import Search
@@ -19,12 +20,14 @@ class ViewsTestCase(SearxTestCase):
             {
                 'content': 'first test content',
                 'title': 'First Test',
+                'category': 'general',
                 'url': 'http://first.test.xyz',
                 'engines': ['youtube', 'startpage'],
                 'engine': 'startpage',
                 'parsed_url': ParseResult(scheme='http', netloc='first.test.xyz', path='/', params='', query='', fragment=''),  # noqa
             }, {
                 'content': 'second test content',
+                'category': 'general',
                 'title': 'Second Test',
                 'url': 'http://second.test.xyz',
                 'engines': ['youtube', 'startpage'],
@@ -33,16 +36,16 @@ class ViewsTestCase(SearxTestCase):
             },
         ]
 
-        def search_mock(search_self, *args):
-            search_self.result_container = Mock(get_ordered_results=lambda: self.test_results,
-                                                answers=set(),
-                                                corrections=set(),
-                                                suggestions=set(),
-                                                infoboxes=[],
-                                                unresponsive_engines=set(),
-                                                results=self.test_results,
-                                                results_number=lambda: 3,
-                                                results_length=lambda: len(self.test_results))
+        def search_mock(*args):
+            return Mock(get_ordered_results=lambda: self.test_results,
+                        answers=set(),
+                        corrections=set(),
+                        suggestions=set(),
+                        infoboxes=[],
+                        unresponsive_engines=set(),
+                        results=self.test_results,
+                        results_number=lambda: 3,
+                        results_length=lambda: len(self.test_results))
 
         Search.search = search_mock
 
@@ -60,6 +63,7 @@ class ViewsTestCase(SearxTestCase):
         self.assertEqual(result.status_code, 200)
         self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)
 
+    @patch('redis.StrictRedis', mock_strict_redis_client)
     def test_index_html(self):
         result = self.app.post('/', data={'q': 'test'})
         self.assertIn(
@@ -71,53 +75,6 @@ class ViewsTestCase(SearxTestCase):
             result.data
         )
 
-    def test_index_json(self):
-        result = self.app.post('/', data={'q': 'test', 'format': 'json'})
-
-        result_dict = json.loads(result.data.decode('utf-8'))
-
-        self.assertEqual('test', result_dict['query'])
-        self.assertEqual(result_dict['results'][0]['content'], 'first test content')
-        self.assertEqual(result_dict['results'][0]['url'], 'http://first.test.xyz')
-
-    def test_index_csv(self):
-        result = self.app.post('/', data={'q': 'test', 'format': 'csv'})
-
-        self.assertEqual(
-            b'title,url,content,host,engine,score\r\n'
-            b'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,\r\n'  # noqa
-            b'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,\r\n',  # noqa
-            result.data
-        )
-
-    def test_index_rss(self):
-        result = self.app.post('/', data={'q': 'test', 'format': 'rss'})
-
-        self.assertIn(
-            b'<description>Search results for "test" - searx</description>',
-            result.data
-        )
-
-        self.assertIn(
-            b'<opensearch:totalResults>3</opensearch:totalResults>',
-            result.data
-        )
-
-        self.assertIn(
-            b'<title>First Test</title>',
-            result.data
-        )
-
-        self.assertIn(
-            b'<link>http://first.test.xyz</link>',
-            result.data
-        )
-
-        self.assertIn(
-            b'<description>first test content</description>',
-            result.data
-        )
-
     def test_about(self):
         result = self.app.get('/about')
         self.assertEqual(result.status_code, 200)