Revert "remove 'all' option from search languages"

This reverts commit 4d177039.

searx/engines/yahoo.py

@@ -73,16 +73,25 @@ def _get_url(query, offset, language, time_range):
                                         lang=language)
 
 
+def _get_language(params):
+    if params['language'] == 'all':
+        return 'en'
+
+    language = match_language(params['language'], supported_languages, language_aliases)
+    if language not in language_aliases.values():
+        language = language.split('-')[0]
+    language = language.replace('-', '_').lower()
+
+    return language
+
+
 # do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
         return params
 
     offset = (params['pageno'] - 1) * 10 + 1
-    language = match_language(params['language'], supported_languages, language_aliases)
-    if language not in language_aliases.values():
-        language = language.split('-')[0]
-    language = language.replace('-', '_').lower()
+    language = _get_language(params)
 
     params['url'] = _get_url(query, offset, language, params['time_range'])
 

searx/engines/yahoo_news.py

@@ -41,7 +41,10 @@ suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
 
     params['url'] = search_url.format(offset=offset,
                                       query=urlencode({'p': query}),

searx/engines/youtube_api.py

@@ -34,7 +34,9 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       api_key=api_key)
 
-    params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
+    # add language tag if specified
+    if params['language'] != 'all':
+        params['url'] += '&relevanceLanguage=' + params['language'].split('-')[0]
 
     return params
 

searx/preferences.py

@@ -12,6 +12,7 @@ if version[0] == '3':
 
 COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5  # 5 years
 LANGUAGE_CODES = [l[0] for l in languages]
+LANGUAGE_CODES.append('all')
 DISABLED = 0
 ENABLED = 1
 DOI_RESOLVERS = list(settings['doi_resolvers'])

searx/search.py

@@ -24,7 +24,7 @@ from flask_babel import gettext
 import requests.exceptions
 import searx.poolrequests as requests_lib
 from searx.engines import (
-    categories, engines, settings
+    categories, engines
 )
 from searx.answerers import ask
 from searx.utils import gen_useragent
@@ -221,10 +221,6 @@ def get_search_query_from_webapp(preferences, form):
     else:
         query_lang = preferences.get_value('language')
 
-    # provides backwards compatibility for requests using old language default
-    if query_lang == 'all':
-        query_lang = settings['search']['language']
-
     # check language
     if not VALID_LANGUAGE_CODE.match(query_lang):
         raise SearxParameterException('language', query_lang)

searx/settings_robot.yml

@@ -5,7 +5,7 @@ general:
 search:
     safe_search : 0
     autocomplete : ""
-    language: "en-US"
+    language: "all"
 
 server:
     port : 11111

searx/templates/courgette/preferences.html

@@ -13,6 +13,7 @@
         <legend>{{ _('Search language') }}</legend>
         <p>
             <select name='language'>
+                <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
                 {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
                 <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
                 {% endfor %}

searx/templates/legacy/preferences.html

@@ -14,6 +14,7 @@
         <legend>{{ _('Search language') }}</legend>
         <p>
         <select name='language'>
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
             {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
             <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
             {% endfor %}

searx/templates/oscar/languages.html

@@ -3,9 +3,10 @@
 {% else %}
 <select class="time_range custom-select form-control" id='language' name='language'>
 {% endif %}
-	{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
-	<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
-		{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
-	</option>
-	{% endfor %}
+	<option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
+		{% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
+		<option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>
+			{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}
+		</option>
+		{% endfor %}
 </select>

searx/templates/pix-art/preferences.html

@@ -9,6 +9,7 @@
         <legend>{{ _('Search language') }}</legend>
         <p>
         <select name='language'>
+            <option value="all" {% if current_language == 'all' %}selected="selected"{% endif %}>{{ _('Default language') }}</option>
             {% for lang_id,lang_name,country_name,english_name in language_codes | sort(attribute=1) %}
             <option value="{{ lang_id }}" {% if lang_id == current_language %}selected="selected"{% endif %}>{{ lang_name }} {% if country_name %}({{ country_name }}) {% endif %}- {{ lang_id }}</option>
             {% endfor %}

searx/webapp.py

@@ -637,8 +637,8 @@ def autocompleter():
     if len(raw_results) <= 3 and completer:
         # get language from cookie
         language = request.preferences.get_value('language')
-        if not language:
-            language = settings['search']['language']
+        if not language or language == 'all':
+            language = 'en'
         else:
             language = language.split('-')[0]
         # run autocompletion
@@ -691,10 +691,7 @@ def preferences():
                              'warn_time': False}
             if e.timeout > settings['outgoing']['request_timeout']:
                 stats[e.name]['warn_timeout'] = True
-            if match_language(request.preferences.get_value('language'),
-                              getattr(e, 'supported_languages', []),
-                              getattr(e, 'language_aliases', {}), None):
-                stats[e.name]['supports_selected_language'] = True
+            stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)
 
     # get first element [0], the engine time,
     # and then the second element [1] : the time (the first one is the label)
@@ -725,6 +722,14 @@ def preferences():
                   preferences=True)
 
 
+def _is_selected_language_supported(engine, preferences):
+    language = preferences.get_value('language')
+    return (language == 'all'
+            or match_language(language,
+                              getattr(engine, 'supported_languages', []),
+                              getattr(engine, 'language_aliases', {}), None))
+
+
 @app.route('/image_proxy', methods=['GET'])
 def image_proxy():
     url = request.args.get('url').encode('utf-8')

tests/unit/engines/test_archlinux.py

@@ -4,7 +4,7 @@ from searx.engines import archlinux
 from searx.testing import SearxTestCase
 
 domains = {
-    'en': 'https://wiki.archlinux.org',
+    'all': 'https://wiki.archlinux.org',
     'de': 'https://wiki.archlinux.de',
     'fr': 'https://wiki.archlinux.fr',
     'ja': 'https://wiki.archlinuxjp.org',

tests/unit/engines/test_bing.py

@@ -18,6 +18,10 @@ class TestBingEngine(SearxTestCase):
         self.assertTrue('language%3AFR' in params['url'])
         self.assertTrue('bing.com' in params['url'])
 
+        dicto['language'] = 'all'
+        params = bing.request(query.encode('utf-8'), dicto)
+        self.assertTrue('language' in params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, bing.response, None)
         self.assertRaises(AttributeError, bing.response, [])

tests/unit/engines/test_bing_news.py

@@ -20,6 +20,10 @@ class TestBingNewsEngine(SearxTestCase):
         self.assertIn('bing.com', params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = bing_news.request(query, dicto)
+        self.assertIn('en', params['url'])
+
     def test_no_url_in_request_year_time_range(self):
         dicto = defaultdict(dict)
         query = 'test_query'

tests/unit/engines/test_dailymotion.py

@@ -19,6 +19,10 @@ class TestDailymotionEngine(SearxTestCase):
         self.assertTrue('dailymotion.com' in params['url'])
         self.assertTrue('fr' in params['url'])
 
+        dicto['language'] = 'all'
+        params = dailymotion.request(query, dicto)
+        self.assertTrue('en' in params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, dailymotion.response, None)
         self.assertRaises(AttributeError, dailymotion.response, [])

tests/unit/engines/test_duckduckgo_images.py

@@ -14,7 +14,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase):
         dicto['is_test'] = True
         dicto['pageno'] = 1
         dicto['safesearch'] = 0
-        dicto['language'] = 'en-US'
+        dicto['language'] = 'all'
         params = duckduckgo_images.request(query, dicto)
         self.assertIn('url', params)
         self.assertIn(query, params['url'])

tests/unit/engines/test_faroo.py

@@ -20,6 +20,10 @@ class TestFarooEngine(SearxTestCase):
         self.assertIn('en', params['url'])
         self.assertIn('web', params['url'])
 
+        dicto['language'] = 'all'
+        params = faroo.request(query, dicto)
+        self.assertIn('en', params['url'])
+
         dicto['language'] = 'de-DE'
         params = faroo.request(query, dicto)
         self.assertIn('de', params['url'])

tests/unit/engines/test_gigablast.py

@@ -11,11 +11,16 @@ class TestGigablastEngine(SearxTestCase):
         dicto = defaultdict(dict)
         dicto['pageno'] = 0
         dicto['safesearch'] = 0
-        dicto['language'] = 'en-US'
+        dicto['language'] = 'all'
         params = gigablast.request(query, dicto)
         self.assertTrue('url' in params)
         self.assertTrue(query in params['url'])
         self.assertTrue('gigablast.com' in params['url'])
+        self.assertTrue('xx' in params['url'])
+
+        dicto['language'] = 'en-US'
+        params = gigablast.request(query, dicto)
+        self.assertTrue('en' in params['url'])
         self.assertFalse('en-US' in params['url'])
 
     def test_response(self):

tests/unit/engines/test_google_news.py

@@ -21,6 +21,11 @@ class TestGoogleNewsEngine(SearxTestCase):
         self.assertIn(query, params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = google_news.request(query, dicto)
+        self.assertIn('url', params)
+        self.assertNotIn('fr', params['url'])
+
     def test_response(self):
         self.assertRaises(AttributeError, google_news.response, None)
         self.assertRaises(AttributeError, google_news.response, [])

tests/unit/engines/test_mediawiki.py

@@ -18,6 +18,10 @@ class TestMediawikiEngine(SearxTestCase):
         self.assertIn('wikipedia.org', params['url'])
         self.assertIn('fr', params['url'])
 
+        dicto['language'] = 'all'
+        params = mediawiki.request(query, dicto)
+        self.assertIn('en', params['url'])
+
         mediawiki.base_url = "http://test.url/"
         mediawiki.search_url = mediawiki.base_url +\
                                  'w/api.php?action=query'\