Merge branch 'nicofonk/rebase-from-searx' into 'master'

rebase from upstream and fix unit tests about /stats endpoint

See merge request e/cloud/my-spot!26

searx/__init__.py

@@ -98,3 +98,7 @@ if 'IMAGE_PROXY' in environ:
     settings['server']['image_proxy'] = environ['IMAGE_PROXY']
 if 'SEARX_REDIS_HOST' in environ:
     settings['redis']['host'] = environ['SEARX_REDIS_HOST']
+if 'HTTP_PROXY_URL' in environ:
+    settings['proxies']['http'] = environ['HTTP_PROXY_URL']
+if 'HTTPS_PROXY_URL' in environ:
+    settings['proxies']['https'] = environ['HTTPS_PROXY_URL']

searx/engines/__init__.py

@@ -113,8 +113,7 @@ def load_engine(engine_data):
                iso_lang not in getattr(engine, 'supported_languages'):
                 language_aliases[iso_lang] = engine_lang
 
-        if language_aliases:
-            setattr(engine, 'language_aliases', language_aliases)
+        setattr(engine, 'language_aliases', language_aliases)
 
     # assign language fetching method if auxiliary method exists
     if hasattr(engine, '_fetch_supported_languages'):

searx/engines/archlinux.py

@@ -36,7 +36,7 @@ def locale_to_lang_code(locale):
 # wikis for some languages were moved off from the main site, we need to make
 # requests to correct URLs to be able to get results in those languages
 lang_urls = {
-    'en': {
+    'all': {
         'base': 'https://wiki.archlinux.org',
         'search': '/index.php?title=Special:Search&offset={offset}&{query}'
     },
@@ -67,7 +67,7 @@ lang_urls = {
 def get_lang_urls(language):
     if language in lang_urls:
         return lang_urls[language]
-    return lang_urls['en']
+    return lang_urls['all']
 
 
 # Language names to build search requests for

searx/engines/asksteem.py

-"""
- Asksteem (general)
-
- @website     https://asksteem.com/
- @provide-api yes
-
- @using-api   yes
- @results     JSON (https://github.com/Hoxly/asksteem-docs/wiki)
- @stable      yes
- @parse       url, title, content
-"""
-
-from json import loads
-from searx.url_utils import urlencode
-
-# engine dependent config
-categories = ['general']
-paging = True
-language_support = False
-disabled = True
-
-# search-url
-search_url = 'https://api.asksteem.com/search?{params}'
-result_url = 'https://steemit.com/@{author}/{title}'
-
-
-# do search-request
-def request(query, params):
-    url = search_url.format(params=urlencode({'q': query, 'pg': params['pageno']}))
-    params['url'] = url
-    return params
-
-
-# get response from search-request
-def response(resp):
-    json = loads(resp.text)
-
-    results = []
-
-    for result in json.get('results', []):
-        results.append({'url': result_url.format(author=result['author'], title=result['permlink']),
-                        'title': result['title'],
-                        'content': result['summary']})
-    return results

searx/engines/bing.py

@@ -34,7 +34,10 @@ search_string = 'search?{query}&first={offset}'
 def request(query, params):
     offset = (params['pageno'] - 1) * 10 + 1
 
-    lang = match_language(params['language'], supported_languages, language_aliases)
+    if params['language'] == 'all':
+        lang = 'EN'
+    else:
+        lang = match_language(params['language'], supported_languages, language_aliases)
 
     query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)
 

searx/engines/bing_images.py

@@ -55,7 +55,7 @@ def request(query, params):
         query=urlencode({'q': query}),
         offset=offset)
 
-    language = match_language(params['language'], supported_languages).lower()
+    language = match_language(params['language'], supported_languages, language_aliases).lower()
 
     params['cookies']['SRCHHPGUSR'] = \
         'ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')

searx/engines/bing_news.py

@@ -71,7 +71,10 @@ def request(query, params):
 
     offset = (params['pageno'] - 1) * 10 + 1
 
-    language = match_language(params['language'], supported_languages, language_aliases)
+    if params['language'] == 'all':
+        language = 'en-US'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases)
 
     params['url'] = _get_url(query, language, offset, params['time_range'])
 

searx/engines/dailymotion.py

@@ -33,7 +33,10 @@ supported_languages_url = 'https://api.dailymotion.com/languages'
 
 # do search-request
 def request(query, params):
-    locale = match_language(params['language'], supported_languages)
+    if params['language'] == 'all':
+        locale = 'en-US'
+    else:
+        locale = match_language(params['language'], supported_languages)
 
     params['url'] = search_url.format(
         query=urlencode({'search': query, 'localization': locale}),

searx/engines/duckduckgo.py

@@ -54,6 +54,9 @@ content_xpath = './/a[@class="result__snippet"]'
 
 # match query's language to a region code that duckduckgo will accept
 def get_region_code(lang, lang_list=[]):
+    if lang == 'all':
+        return None
+
     lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')
     lang_parts = lang_code.split('-')
 
@@ -61,7 +64,6 @@ def get_region_code(lang, lang_list=[]):
     return lang_parts[1].lower() + '-' + lang_parts[0].lower()
 
 
-# do search-request
 def request(query, params):
     if params['time_range'] and params['time_range'] not in time_range_dict:
         return params
@@ -69,8 +71,12 @@ def request(query, params):
     offset = (params['pageno'] - 1) * 30
 
     region_code = get_region_code(params['language'], supported_languages)
-    params['url'] = url.format(
-        query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
+    if region_code:
+        params['url'] = url.format(
+            query=urlencode({'q': query, 'kl': region_code}), offset=offset, dc_param=offset)
+    else:
+        params['url'] = url.format(
+            query=urlencode({'q': query}), offset=offset, dc_param=offset)
 
     if params['time_range'] in time_range_dict:
         params['url'] += time_range_url.format(range=time_range_dict[params['time_range']])

searx/engines/duckduckgo_images.py

@@ -56,8 +56,12 @@ def request(query, params):
     safesearch = params['safesearch'] - 1
 
     region_code = get_region_code(params['language'], lang_list=supported_languages)
-    params['url'] = images_url.format(
-        query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    if region_code:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query, 'l': region_code}), offset=offset, safesearch=safesearch, vqd=vqd)
+    else:
+        params['url'] = images_url.format(
+            query=urlencode({'q': query}), offset=offset, safesearch=safesearch, vqd=vqd)
 
     return params
 

searx/engines/faroo.py

@@ -40,7 +40,10 @@ def request(query, params):
     offset = (params['pageno'] - 1) * number_of_results + 1
     categorie = search_category.get(params['category'], 'web')
 
-    language = params['language'].split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('-')[0]
 
     # if language is not supported, put it in english
     if language != 'en' and\

searx/engines/gigablast.py

@@ -50,9 +50,12 @@ supported_languages_url = 'https://gigablast.com/search?&rxikd=1'
 def request(query, params):
     offset = (params['pageno'] - 1) * number_of_results
 
-    language = params['language'].replace('-', '_').lower()
-    if language.split('-')[0] != 'zh':
-        language = language.split('-')[0]
+    if params['language'] == 'all':
+        language = 'xx'
+    else:
+        language = params['language'].replace('-', '_').lower()
+        if language.split('-')[0] != 'zh':
+            language = language.split('-')[0]
 
     if params['safesearch'] >= 1:
         safesearch = 1

searx/engines/google.py

@@ -165,7 +165,11 @@ def extract_text_from_dom(result, xpath):
 def request(query, params):
     offset = (params['pageno'] - 1) * 10
 
-    language = match_language(params['language'], supported_languages)
+    if params['language'] == 'all' or params['language'] == 'en-US':
+        language = 'en-GB'
+    else:
+        language = match_language(params['language'], supported_languages, language_aliases)
+
     language_array = language.split('-')
     if params['language'].find('-') > 0:
         country = params['language'].split('-')[1]
@@ -380,10 +384,10 @@ def attributes_to_html(attributes):
 def _fetch_supported_languages(resp):
     supported_languages = {}
     dom = html.fromstring(resp.text)
-    options = dom.xpath('//table//td/font/label/span')
+    options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
     for option in options:
-        code = option.xpath('./@id')[0][1:]
-        name = option.text.title()
+        code = option.xpath('./@value')[0].split('_')[-1]
+        name = option.xpath('./@data-name')[0].title()
         supported_languages[code] = {"name": name}
 
     return supported_languages

searx/engines/google_news.py

@@ -51,9 +51,10 @@ def request(query, params):
     params['url'] = search_url.format(query=urlencode({'q': query}),
                                       search_options=urlencode(search_options))
 
-    language = match_language(params['language'], supported_languages).split('-')[0]
-    if language:
-        params['url'] += '&lr=lang_' + language
+    if params['language'] != 'all':
+        language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
+        if language:
+            params['url'] += '&lr=lang_' + language
 
     return params
 

searx/engines/mediawiki.py

@@ -45,7 +45,10 @@ def request(query, params):
 
     format_strings = list(Formatter().parse(base_url))
 
-    language = params['language'].split('-')[0]
+    if params['language'] == 'all':
+        language = 'en'
+    else:
+        language = params['language'].split('-')[0]
 
     # format_string [('https://', 'language', '', None), ('.wikipedia.org/', None, None, None)]
     if any(x[1] == 'language' for x in format_strings):

searx/engines/photon.py

@@ -35,9 +35,10 @@ def request(query, params):
         search_string.format(query=urlencode({'q': query}),
                              limit=number_of_results)
 
-    language = params['language'].split('-')[0]
-    if language in supported_languages:
-        params['url'] = params['url'] + "&lang=" + language
+    if params['language'] != 'all':
+        language = params['language'].split('_')[0]
+        if language in supported_languages:
+            params['url'] = params['url'] + "&lang=" + language
 
     # using searx User-Agent
     params['headers']['User-Agent'] = searx_useragent()

searx/engines/qwant.py

@@ -46,8 +46,9 @@ def request(query, params):
                                    offset=offset)
 
     # add language tag
-    language = match_language(params['language'], supported_languages)
-    params['url'] += '&locale=' + language.replace('-', '_').lower()
+    if params['language'] != 'all':
+        language = match_language(params['language'], supported_languages, language_aliases)
+        params['url'] += '&locale=' + language.replace('-', '_').lower()
 
     return params
 

searx/engines/startpage.py

@@ -46,8 +46,9 @@ def request(query, params):
     params['data'] = {'query': query,
                       'startat': offset}
 
-    # set language
-    params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
+    # set language if specified
+    if params['language'] != 'all':
+        params['data']['with_language'] = ('lang_' + params['language'].split('-')[0])
 
     return params
 

searx/engines/subtitleseeker.py

@@ -48,7 +48,7 @@ def response(resp):
         search_lang = 'Farsi'
     elif resp.search_params['language'] == 'pt-BR':
         search_lang = 'Brazilian'
-    else:
+    elif resp.search_params['language'] != 'all':
         search_lang = [lc[3]
                        for lc in language_codes
                        if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]