Commit 89d0954b authored by Nicolas Gelot's avatar Nicolas Gelot

Merge remote-tracking branch 'asciimoo/master' into dev

parents 88b1ef77 4b733228
flask==1.0.2 flask==1.0.2
jinja2==2.10 jinja2==2.10.1
flask-babel==0.12.2 flask-babel==0.12.2
lxml==4.3.3 lxml==4.3.3
pygments==2.3.1 pygments==2.1.3
python-dateutil==2.8.0 python-dateutil==2.8.0
pyyaml==5.1 pyyaml==5.1
requests[socks]==2.21.0 requests[socks]==2.22.0
redis==3.2.1 redis==3.2.1
...@@ -11,9 +11,9 @@ ...@@ -11,9 +11,9 @@
""" """
from datetime import date, timedelta from datetime import date, timedelta
from json import loads
from lxml import html from lxml import html
from searx.url_utils import urlencode, urlparse, parse_qs from searx.url_utils import urlencode
# engine dependent config # engine dependent config
categories = ['images'] categories = ['images']
...@@ -25,8 +25,7 @@ number_of_results = 100 ...@@ -25,8 +25,7 @@ number_of_results = 100
search_url = 'https://www.google.com/search'\ search_url = 'https://www.google.com/search'\
'?{query}'\ '?{query}'\
'&tbm=isch'\ '&tbm=isch'\
'&gbv=1'\ '&yv=2'\
'&sa=G'\
'&{search_options}' '&{search_options}'
time_range_attr = "qdr:{range}" time_range_attr = "qdr:{range}"
time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}" time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
...@@ -38,6 +37,7 @@ time_range_dict = {'day': 'd', ...@@ -38,6 +37,7 @@ time_range_dict = {'day': 'd',
# do search-request # do search-request
def request(query, params): def request(query, params):
search_options = { search_options = {
'ijn': params['pageno'] - 1,
'start': (params['pageno'] - 1) * number_of_results 'start': (params['pageno'] - 1) * number_of_results
} }
...@@ -51,7 +51,7 @@ def request(query, params): ...@@ -51,7 +51,7 @@ def request(query, params):
search_options['tbs'] = time_range_custom_attr.format(start=start, end=end) search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
if safesearch and params['safesearch']: if safesearch and params['safesearch']:
search_options['safe'] = 'active' search_options['safe'] = 'on'
params['url'] = search_url.format(query=urlencode({'q': query}), params['url'] = search_url.format(query=urlencode({'q': query}),
search_options=urlencode(search_options)) search_options=urlencode(search_options))
...@@ -61,30 +61,27 @@ def request(query, params): ...@@ -61,30 +61,27 @@ def request(query, params):
# get response from search-request # get response from search-request
def response(resp): def response(resp):
results = []
dom = html.fromstring(resp.text) dom = html.fromstring(resp.text)
results = [] # parse results
for element in dom.xpath('//div[@id="search"] //td'): for result in dom.xpath('//div[contains(@class, "rg_meta")]/text()'):
link = element.xpath('./a')[0]
try:
google_url = urlparse(link.xpath('.//@href')[0]) metadata = loads(result)
query = parse_qs(google_url.query) img_format = "{0} {1}x{2}".format(metadata['ity'], str(metadata['ow']), str(metadata['oh']))
source_url = next(iter(query.get('q', [])), None) source = "{0} ({1})".format(metadata['st'], metadata['isu'])
results.append({'url': metadata['ru'],
title_parts = element.xpath('./cite//following-sibling::*/text()') 'title': metadata['pt'],
title_parts.extend(element.xpath('./cite//following-sibling::text()')[:-1]) 'content': metadata['s'],
'source': source,
result = { 'img_format': img_format,
'title': ''.join(title_parts), 'thumbnail_src': metadata['tu'],
'content': '', 'img_src': metadata['ou'],
'template': 'images.html', 'template': 'images.html'})
'url': source_url,
'img_src': source_url, except:
'thumbnail_src': next(iter(link.xpath('.//img //@src')), None)
}
if not source_url or not result['thumbnail_src']:
continue continue
results.append(result)
return results return results
# Seedpeer (Videos, Music, Files)
#
# @website http://seedpeer.eu
# @provide-api no (nothing found)
#
# @using-api no
# @results HTML (using search portal)
# @stable yes (HTML can change)
# @parse url, title, content, seed, leech, magnetlink
from lxml import html
from operator import itemgetter
from searx.url_utils import quote, urljoin
url = 'http://www.seedpeer.eu/'
search_url = url + 'search/{search_term}/7/{page_no}.html'
# specific xpath variables
torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
# do search-request
def request(query, params):
params['url'] = search_url.format(search_term=quote(query),
page_no=params['pageno'] - 1)
return params
# get response from search-request
def response(resp):
results = []
dom = html.fromstring(resp.text)
torrent_links = dom.xpath(torrent_xpath)
if len(torrent_links) > 0:
seeds = dom.xpath(seeds_xpath)
peers = dom.xpath(peers_xpath)
titles = dom.xpath(title_xpath)
sizes = dom.xpath(size_xpath)
ages = dom.xpath(age_xpath)
else: # under ~5 results uses a different xpath
torrent_links = dom.xpath(alternative_torrent_xpath)
seeds = dom.xpath(alternative_seeds_xpath)
peers = dom.xpath(alternative_peers_xpath)
titles = dom.xpath(alternative_title_xpath)
sizes = dom.xpath(alternative_size_xpath)
ages = dom.xpath(alternative_age_xpath)
# return empty array if nothing is found
if not torrent_links:
return []
# parse results
for index, result in enumerate(torrent_links):
link = result.attrib.get('href')
href = urljoin(url, link)
results.append({'url': href,
'title': titles[index].text_content(),
'content': '{}, {}'.format(sizes[index], ages[index]),
'seed': seeds[index],
'leech': peers[index],
'template': 'torrent.html'})
# return results sorted by seeder
return sorted(results, key=itemgetter('seed'), reverse=True)
"""
Subtitleseeker (Video)
@website http://www.subtitleseeker.com
@provide-api no
@using-api no
@results HTML
@stable no (HTML can change)
@parse url, title, content
"""
from lxml import html
from searx.languages import language_codes
from searx.engines.xpath import extract_text
from searx.url_utils import quote_plus
# engine dependent config
categories = ['videos']
paging = True
language = ""
# search-url
url = 'http://www.subtitleseeker.com/'
search_url = url + 'search/TITLES/{query}?p={pageno}'
# specific xpath variables
results_xpath = '//div[@class="boxRows"]'
# do search-request
def request(query, params):
params['url'] = search_url.format(query=quote_plus(query),
pageno=params['pageno'])
return params
# get response from search-request
def response(resp):
results = []
dom = html.fromstring(resp.text)
search_lang = ""
# dirty fix for languages named differenly in their site
if resp.search_params['language'][:2] == 'fa':
search_lang = 'Farsi'
elif resp.search_params['language'] == 'pt-BR':
search_lang = 'Brazilian'
elif resp.search_params['language'] != 'all':
search_lang = [lc[3]
for lc in language_codes
if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]
search_lang = search_lang[0].split(' (')[0]
# parse results
for result in dom.xpath(results_xpath):
link = result.xpath(".//a")[0]
href = link.attrib.get('href')
if language is not "":
href = href + language + '/'
elif search_lang:
href = href + search_lang + '/'
title = extract_text(link)
content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
content = content + " - "
text = extract_text(result.xpath('.//div[contains(@class,"grey-web")]')[0])
content = content + text
if result.xpath(".//span") != []:
content = content +\
" - (" +\
extract_text(result.xpath(".//span")) +\
")"
# append result
results.append({'url': href,
'title': title,
'content': content})
# return results
return results
"""
Swisscows (Web, Images)
@website https://swisscows.ch
@provide-api no
@using-api no
@results HTML (using search portal)
@stable no (HTML can change)
@parse url, title, content
"""
from json import loads
import re
from lxml.html import fromstring
from searx.url_utils import unquote, urlencode
from searx.utils import match_language
# engine dependent config
categories = ['general', 'images']
paging = True
language_support = True
# search-url
base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}'
supported_languages_url = base_url
# regex
regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile(r'^initialData:\s*')
regex_json_remove_end = re.compile(r',\s*environment$')
regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')
# do search-request
def request(query, params):
if params['language'] == 'all':
ui_language = 'browser'
region = 'browser'
else:
region = match_language(params['language'], supported_languages, language_aliases)
ui_language = region.split('-')[0]
search_path = search_string.format(
query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
page=params['pageno']
)
# image search query is something like 'image?{query}&page={page}'
if params['category'] == 'images':
search_path = 'image' + search_path
params['url'] = base_url + search_path
return params
# get response from search-request
def response(resp):
results = []
json_regex = regex_json.search(resp.text)
# check if results are returned
if not json_regex:
return []
json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
json = loads(json_raw)
# parse results
for result in json['Results'].get('items', []):
result_title = result['Title'].replace('\\uE000', '').replace('\\uE001', '')
# parse image results
if result.get('ContentType', '').startswith('image'):
img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
# append result
results.append({'url': result['SourceUrl'],
'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
# parse general results
else:
result_url = result['Url'].replace('\\uE000', '').replace('\\uE001', '')
result_content = result['Description'].replace('\\uE000', '').replace('\\uE001', '')
# append result
results.append({'url': result_url,
'title': result_title,
'content': result_content})
# parse images
for result in json.get('Images', []):
# decode image url
img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
# append result
results.append({'url': result['SourceUrl'],
'title': result['Title'],
'content': '',
'img_src': img_url,
'template': 'images.html'})
# return results
return results
# get supported languages from their site
def _fetch_supported_languages(resp):
supported_languages = []
dom = fromstring(resp.text)
options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
for option in options:
code = option.xpath('./@data-search-language')[0]
if code.startswith('nb-'):
code = code.replace('nb', 'no', 1)
supported_languages.append(code)
return supported_languages
...@@ -577,24 +577,12 @@ engines: ...@@ -577,24 +577,12 @@ engines:
engine : spotify engine : spotify
shortcut : stf shortcut : stf
- name : subtitleseeker
engine : subtitleseeker
shortcut : ss
# The language is an option. You can put any language written in english
# Examples : English, French, German, Hungarian, Chinese...
# language : English
- name : startpage - name : startpage
engine : startpage engine : startpage
shortcut : sp shortcut : sp
timeout : 6.0 timeout : 6.0
disabled : True disabled : True
- name : swisscows
engine : swisscows
shortcut : sw
disabled : True
- name : tokyotoshokan - name : tokyotoshokan
engine : tokyotoshokan engine : tokyotoshokan
shortcut : tt shortcut : tt
...@@ -673,12 +661,6 @@ engines: ...@@ -673,12 +661,6 @@ engines:
timeout: 6.0 timeout: 6.0
categories : science categories : science
- name : seedpeer
engine : seedpeer
shortcut: speu
categories: files, music, videos
disabled: True
- name : dictzone - name : dictzone
engine : dictzone engine : dictzone
shortcut : dc shortcut : dc
...@@ -718,10 +700,11 @@ engines: ...@@ -718,10 +700,11 @@ engines:
engine: xpath engine: xpath
paging : True paging : True
search_url : https://search.seznam.cz/?q={query}&count=10&from={pageno} search_url : https://search.seznam.cz/?q={query}&count=10&from={pageno}
results_xpath: //div[@class="Page-content"]//div[@class="Result "] results_xpath: //div[@class="Page-content"]//div[contains(@class, "Result ")]
url_xpath : ./h3/a/@href url_xpath : ./h3/a/@href
title_xpath : ./h3 title_xpath : ./h3
content_xpath : .//p[@class="Result-description"] content_xpath : .//p[@class="Result-description"]
suggestion_xpath: //div[@class="Related-container"]//div[@class="RelatedItem"]/div/span/a
first_page_num : 0 first_page_num : 0
page_size : 10 page_size : 10
disabled : True disabled : True
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
<meta name="viewport" content="width=device-width, maximum-scale=1.0, user-scalable=1"> <meta name="viewport" content="width=device-width, maximum-scale=1.0, user-scalable=1">
<meta name="HandheldFriendly" content="True"> <meta name="HandheldFriendly" content="True">
<meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1"> <meta http-equiv="X-UA-Compatible" content="IE=edge, chrome=1">
<title>{% block title %}{% endblock %}searx</title> <title>{% block title %}{% endblock %}{{ instance_name }}</title>
{% if rtl %} {% if rtl %}
<link rel="stylesheet" href="{{ url_for('static', filename='css/searx-rtl.min.css') }}" type="text/css" media="screen" /> <link rel="stylesheet" href="{{ url_for('static', filename='css/searx-rtl.min.css') }}" type="text/css" media="screen" />
{% else %} {% else %}
......
import mock
from collections import defaultdict
from searx.engines import seedpeer
from unittest import TestCase
from datetime import datetime
class TestSeedPeerEngine(TestCase):
html = ''
with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
html += fixture.read()
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
params = seedpeer.request(query, dicto)
self.assertIn('url', params)
self.assertIn(query, params['url'])
self.assertIn('seedpeer.eu', params['url'])
def test_response_raises_attr_error_on_empty_response(self):
self.assertRaises(AttributeError, seedpeer.response, None)
self.assertRaises(AttributeError, seedpeer.response, [])
self.assertRaises(AttributeError, seedpeer.response, '')
self.assertRaises(AttributeError, seedpeer.response, '[]')
def test_response_returns_empty_list(self):
response = mock.Mock(text='<html></html>')
self.assertEqual(seedpeer.response(response), [])
def test_response_returns_all_results(self):
response = mock.Mock(text=self.html)
results = seedpeer.response(response)
self.assertTrue(isinstance(results, list))
self.assertEqual(len(results), 2)
def test_response_returns_correct_results(self):
response = mock.Mock(text=self.html)
results = seedpeer.response(response)
self.assertEqual(
results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
)
self.assertEqual(
results[0]['url'],
'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
)
self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
self.assertEqual(results[0]['seed'], '861')
self.assertEqual(results[0]['leech'], '332')
from collections import defaultdict
import mock
from searx.engines import subtitleseeker
from unittest import TestCase
class TestSubtitleseekerEngine(TestCase):
def test_request(self):
query = 'test_query'
dicto = defaultdict(dict)
dicto['pageno'] = 1
dicto['language'] = 'fr-FR'
params = subtitleseeker.request(query, dicto)
self.assertTrue('url' in params)
self.assertTrue(query in params['url'])
self.assertTrue('subtitleseeker.com' in params['url'])
def test_response(self):
dicto = defaultdict(dict)
dicto['language'] = 'fr-FR'
response = mock.Mock(search_params=dicto)
self.assertRaises(AttributeError, subtitleseeker.response, None)
self.assertRaises(AttributeError, subtitleseeker.response, [])
self.assertRaises(AttributeError, subtitleseeker.response, '')
self.assertRaises(AttributeError, subtitleseeker.response, '[]')
response = mock.Mock(text='<html></html>', search_params=dicto)
self.assertEqual(subtitleseeker.response(response), [])
html = """
<div class="boxRows">
<div class="boxRowsInner" style="width:600px;">
<img src="http://static.subtitleseeker.com/images/movie.gif"
style="width:16px; height:16px;" class="icon">
<a href="http://this.is.the.url/"
class="blue" title="Title subtitle" >
This is the Title
</a>
<br><br>
<span class="f10b grey-dark arial" style="padding:0px 0px 5px 20px">
"Alternative Title"
</span>
</div>
<div class="boxRowsInner f12b red" style="width:70px;">
1998
</div>
<div class="boxRowsInner grey-web f12" style="width:120px;">
<img src="http://static.subtitleseeker.com/images/basket_put.png"
style="width:16px; height:16px;" class="icon">
1039 Subs
</div>
<div class="boxRowsInner grey-web f10" style="width:130px;">
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
style="width:16px; height:16px;" class="icon">
1 hours ago
</div>
<div class="clear"></div>
</div>
"""
response = mock.Mock(text=html, search_params=dicto)
results = subtitleseeker.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the Title')
self.assertEqual(results[0]['url'], 'http://this.is.the.url/French/')
self.assertIn('1998', results[0]['content'])
self.assertIn('1039 Subs', results[0]['content'])
self.assertIn('Alternative Title', results[0]['content'])
dicto['language'] = 'pt-BR'
results = subtitleseeker.response(response)
self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/')
html = """
<div class="boxRows">
<div class="boxRowsInner" style="width:600px;">
<img src="http://static.subtitleseeker.com/images/movie.gif"
style="width:16px; height:16px;" class="icon">
<a href="http://this.is.the.url/"
class="blue" title="Title subtitle" >
This is the Title
</a>
</div>
<div class="boxRowsInner f12b red" style="width:70px;">
1998
</div>
<div class="boxRowsInner grey-web f12" style="width:120px;">
<img src="http://static.subtitleseeker.com/images/basket_put.png"
style="width:16px; height:16px;" class="icon">
1039 Subs
</div>
<div class="boxRowsInner grey-web f10" style="width:130px;">
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
style="width:16px; height:16px;" class="icon">
1 hours ago
</div>
<div class="clear"></div>
</div>
"""
dicto['language'] = 'all'
response = mock.Mock(text=html, search_params=dicto)
results = subtitleseeker.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the Title')
self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
self.assertIn('1998', results[0]['content'])
self.assertIn('1039 Subs', results[0]['content'])
html = """
<div class="boxRows">
<div class="boxRowsInner" style="width:600px;">
<img src="http://static.subtitleseeker.com/images/movie.gif"
style="width:16px; height:16px;" class="icon">
<a href="http://this.is.the.url/"
class="blue" title="Title subtitle" >
This is the Title
</a>
</div>
<div class="boxRowsInner f12b red" style="width:70px;">
1998
</div>
<div class="boxRowsInner grey-web f12" style="width:120px;">
<img src="http://static.subtitleseeker.com/images/basket_put.png"
style="width:16px; height:16px;" class="icon">
1039 Subs
</div>
<div class="boxRowsInner grey-web f10" style="width:130px;">
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
style="width:16px; height:16px;" class="icon">
1 hours ago
</div>
<div class="clear"></div>
</div>
"""
subtitleseeker.language = 'English'
response = mock.Mock(text=html, search_params=dicto)
results = subtitleseeker.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['title'], 'This is the Title')
self.assertEqual(results[0]['url'], 'http://this.is.the.url/English/')
self.assertIn('1998', results[0]['content'])
self.assertIn('1039 Subs', results[0]['content'])
html = """
<div class="boxRowsInner" style="width:600px;">
<img src="http://static.subtitleseeker.com/images/movie.gif"
style="width:16px; height:16px;" class="icon">
<a href="http://this.is.the.url/"
class="blue" title="Title subtitle" >
This is the Title
</a>
</div>
<div class="boxRowsInner f12b red" style="width:70px;">
1998
</div>
<div class="boxRowsInner grey-web f12" style="width:120px;">
<img src="http://static.subtitleseeker.com/images/basket_put.png"
style="width:16px; height:16px;" class="icon">
1039 Subs
</div>
<div class="boxRowsInner grey-web f10" style="width:130px;">
<img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"
style="width:16px; height:16px;" class="icon">
1 hours ago
</div>
"""
response = mock.Mock(text=html, search_params=dicto)
results = subtitleseeker.response(response)
self.assertEqual(type(results), list)
self.assertEqual(len(results), 0)
from collections import defaultdict
import mock
from searx.engines import swisscows
from unittest import TestCase