Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
e
infra
spot
Commits
5568f24d
Commit
5568f24d
authored
Nov 25, 2018
by
Marc Abonce Seguin
Browse files
[fix] check language aliases when setting search language
parent
3c95d64f
Changes
16
Show whitespace changes
Inline
Side-by-side
searx/engines/bing_images.py
View file @
5568f24d
...
...
@@ -55,7 +55,7 @@ def request(query, params):
query
=
urlencode
({
'q'
:
query
}),
offset
=
offset
)
language
=
match_language
(
params
[
'language'
],
supported_languages
).
lower
()
language
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
).
lower
()
params
[
'cookies'
][
'SRCHHPGUSR'
]
=
\
'ADLT='
+
safesearch_types
.
get
(
params
[
'safesearch'
],
'DEMOTE'
)
...
...
searx/engines/bing_videos.py
View file @
5568f24d
...
...
@@ -48,7 +48,7 @@ def request(query, params):
'ADLT='
+
safesearch_types
.
get
(
params
[
'safesearch'
],
'DEMOTE'
)
# language cookie
language
=
match_language
(
params
[
'language'
],
supported_languages
).
lower
()
language
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
).
lower
()
params
[
'cookies'
][
'_EDGE_S'
]
=
'mkt='
+
language
+
'&F=1'
# query and paging
...
...
searx/engines/google.py
View file @
5568f24d
...
...
@@ -166,7 +166,7 @@ def extract_text_from_dom(result, xpath):
def
request
(
query
,
params
):
offset
=
(
params
[
'pageno'
]
-
1
)
*
10
language
=
match_language
(
params
[
'language'
],
supported_languages
)
language
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
)
language_array
=
language
.
split
(
'-'
)
if
params
[
'language'
].
find
(
'-'
)
>
0
:
country
=
params
[
'language'
].
split
(
'-'
)[
1
]
...
...
searx/engines/google_news.py
View file @
5568f24d
...
...
@@ -51,7 +51,7 @@ def request(query, params):
params
[
'url'
]
=
search_url
.
format
(
query
=
urlencode
({
'q'
:
query
}),
search_options
=
urlencode
(
search_options
))
language
=
match_language
(
params
[
'language'
],
supported_languages
).
split
(
'-'
)[
0
]
language
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
).
split
(
'-'
)[
0
]
if
language
:
params
[
'url'
]
+=
'&lr=lang_'
+
language
...
...
searx/engines/qwant.py
View file @
5568f24d
...
...
@@ -46,7 +46,7 @@ def request(query, params):
offset
=
offset
)
# add language tag
language
=
match_language
(
params
[
'language'
],
supported_languages
)
language
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
)
params
[
'url'
]
+=
'&locale='
+
language
.
replace
(
'-'
,
'_'
).
lower
()
return
params
...
...
searx/engines/swisscows.py
View file @
5568f24d
...
...
@@ -36,7 +36,7 @@ regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
# do search-request
def
request
(
query
,
params
):
region
=
match_language
(
params
[
'language'
],
supported_languages
)
region
=
match_language
(
params
[
'language'
],
supported_languages
,
language_aliases
)
ui_language
=
region
.
split
(
'-'
)[
0
]
search_path
=
search_string
.
format
(
...
...
searx/engines/wikidata.py
View file @
5568f24d
...
...
@@ -68,7 +68,7 @@ def response(resp):
html
=
fromstring
(
resp
.
text
)
search_results
=
html
.
xpath
(
wikidata_ids_xpath
)
language
=
match_language
(
resp
.
search_params
[
'language'
],
supported_languages
).
split
(
'-'
)[
0
]
language
=
match_language
(
resp
.
search_params
[
'language'
],
supported_languages
,
language_aliases
).
split
(
'-'
)[
0
]
# TODO: make requests asynchronous to avoid timeout when result_count > 1
for
search_result
in
search_results
[:
result_count
]:
...
...
searx/engines/wikipedia.py
View file @
5568f24d
...
...
@@ -31,7 +31,7 @@ supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
# set language in base_url
def
url_lang
(
lang
):
return
match_language
(
lang
,
supported_languages
).
split
(
'-'
)[
0
]
return
match_language
(
lang
,
supported_languages
,
language_aliases
).
split
(
'-'
)[
0
]
# do search-request
...
...
tests/unit/engines/test_bing_images.py
View file @
5568f24d
...
...
@@ -9,6 +9,7 @@ class TestBingImagesEngine(SearxTestCase):
def
test_request
(
self
):
bing_images
.
supported_languages
=
[
'fr-FR'
,
'en-US'
]
bing_images
.
language_aliases
=
{}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
dicto
[
'pageno'
]
=
1
...
...
tests/unit/engines/test_bing_videos.py
View file @
5568f24d
...
...
@@ -9,6 +9,7 @@ class TestBingVideosEngine(SearxTestCase):
def
test_request
(
self
):
bing_videos
.
supported_languages
=
[
'fr-FR'
,
'en-US'
]
bing_videos
.
language_aliases
=
{}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
dicto
[
'pageno'
]
=
1
...
...
tests/unit/engines/test_google.py
View file @
5568f24d
...
...
@@ -15,7 +15,8 @@ class TestGoogleEngine(SearxTestCase):
return
response
def
test_request
(
self
):
google
.
supported_languages
=
[
'en'
,
'fr'
,
'zh-CN'
]
google
.
supported_languages
=
[
'en'
,
'fr'
,
'zh-CN'
,
'iw'
]
google
.
language_aliases
=
{
'he'
:
'iw'
}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
...
...
@@ -41,6 +42,12 @@ class TestGoogleEngine(SearxTestCase):
self
.
assertIn
(
'zh-CN'
,
params
[
'url'
])
self
.
assertIn
(
'zh-CN'
,
params
[
'headers'
][
'Accept-Language'
])
dicto
[
'language'
]
=
'he'
params
=
google
.
request
(
query
,
dicto
)
self
.
assertIn
(
'google.com'
,
params
[
'url'
])
self
.
assertIn
(
'iw'
,
params
[
'url'
])
self
.
assertIn
(
'iw'
,
params
[
'headers'
][
'Accept-Language'
])
def
test_response
(
self
):
self
.
assertRaises
(
AttributeError
,
google
.
response
,
None
)
self
.
assertRaises
(
AttributeError
,
google
.
response
,
[])
...
...
tests/unit/engines/test_google_news.py
View file @
5568f24d
...
...
@@ -10,6 +10,7 @@ class TestGoogleNewsEngine(SearxTestCase):
def
test_request
(
self
):
google_news
.
supported_languages
=
[
'en-US'
,
'fr-FR'
]
google_news
.
language_aliases
=
{}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
dicto
[
'pageno'
]
=
1
...
...
tests/unit/engines/test_qwant.py
View file @
5568f24d
...
...
@@ -8,6 +8,7 @@ class TestQwantEngine(SearxTestCase):
def
test_request
(
self
):
qwant
.
supported_languages
=
[
'en-US'
,
'fr-CA'
,
'fr-FR'
]
qwant
.
language_aliases
=
{}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
dicto
[
'pageno'
]
=
0
...
...
tests/unit/engines/test_swisscows.py
View file @
5568f24d
...
...
@@ -8,6 +8,7 @@ class TestSwisscowsEngine(SearxTestCase):
def
test_request
(
self
):
swisscows
.
supported_languages
=
[
'de-AT'
,
'de-DE'
]
swisscows
.
language_aliases
=
{}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
dicto
[
'pageno'
]
=
1
...
...
tests/unit/engines/test_wikidata.py
View file @
5568f24d
...
...
@@ -27,6 +27,7 @@ class TestWikidataEngine(SearxTestCase):
self
.
assertRaises
(
AttributeError
,
wikidata
.
response
,
'[]'
)
wikidata
.
supported_languages
=
[
'en'
,
'es'
]
wikidata
.
language_aliases
=
{}
response
=
mock
.
Mock
(
text
=
'<html></html>'
,
search_params
=
{
"language"
:
"en"
})
self
.
assertEqual
(
wikidata
.
response
(
response
),
[])
...
...
tests/unit/engines/test_wikipedia.py
View file @
5568f24d
...
...
@@ -8,7 +8,8 @@ from searx.testing import SearxTestCase
class
TestWikipediaEngine
(
SearxTestCase
):
def
test_request
(
self
):
wikipedia
.
supported_languages
=
[
'fr'
,
'en'
]
wikipedia
.
supported_languages
=
[
'fr'
,
'en'
,
'no'
]
wikipedia
.
language_aliases
=
{
'nb'
:
'no'
}
query
=
'test_query'
dicto
=
defaultdict
(
dict
)
...
...
@@ -25,9 +26,13 @@ class TestWikipediaEngine(SearxTestCase):
self
.
assertIn
(
'Test_Query'
,
params
[
'url'
])
self
.
assertNotIn
(
'test_query'
,
params
[
'url'
])
dicto
[
'language'
]
=
'nb'
params
=
wikipedia
.
request
(
query
,
dicto
)
self
.
assertIn
(
'no.wikipedia.org'
,
params
[
'url'
])
dicto
[
'language'
]
=
'xx'
params
=
wikipedia
.
request
(
query
,
dicto
)
self
.
assertIn
(
'en'
,
params
[
'url'
])
self
.
assertIn
(
'en
.wikipedia.org
'
,
params
[
'url'
])
def
test_response
(
self
):
dicto
=
defaultdict
(
dict
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment