Merge remote-tracking branch 'asciimoo/master' into dev (89d0954b) · Commits · e / infra / spot

requirements.txt

+3 −3

Original line number	Original line	Diff line number	Diff line
	flask==1.0.2		flask==1.0.2
	jinja2==2.10		jinja2==2.10.1
	flask-babel==0.12.2		flask-babel==0.12.2
	lxml==4.3.3		lxml==4.3.3
	pygments==2.3.1		pygments==2.1.3
	python-dateutil==2.8.0		python-dateutil==2.8.0
	pyyaml==5.1		pyyaml==5.1
	requests [socks]==2.21.0		requests [socks]==2.22.0
	redis==3.2.1		redis==3.2.1

searx/engines/google_images.py

+24 −27

Original line number	Original line	Diff line number	Diff line
	@@ -11,9 +11,9 @@
	"""		"""

	from datetime import date, timedelta		from datetime import date, timedelta
			from json import loads
	from lxml import html		from lxml import html
	from searx.url_utils import urlencode, urlparse, parse_qs		from searx.url_utils import urlencode


	# engine dependent config		# engine dependent config
	categories = ['images']		categories = ['images']
	@@ -25,8 +25,7 @@ number_of_results = 100
	search_url = 'https://www.google.com/search'\		search_url = 'https://www.google.com/search'\
	'?{query}'\		'?{query}'\
	'&tbm=isch'\		'&tbm=isch'\
	'&gbv=1'\		'&yv=2'\
	'&sa=G'\
	'&{search_options}'		'&{search_options}'
	time_range_attr = "qdr:{range}"		time_range_attr = "qdr:{range}"
	time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"		time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
	@@ -38,6 +37,7 @@ time_range_dict = {'day': 'd',
	# do search-request		# do search-request
	def request(query, params):		def request(query, params):
	search_options = {		search_options = {
			'ijn': params['pageno'] - 1,
	'start': (params['pageno'] - 1) * number_of_results		'start': (params['pageno'] - 1) * number_of_results
	}		}

	@@ -51,7 +51,7 @@ def request(query, params):
	search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)		search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)

	if safesearch and params['safesearch']:		if safesearch and params['safesearch']:
	search_options['safe'] = 'active'		search_options['safe'] = 'on'

	params['url'] = search_url.format(query=urlencode({'q': query}),		params['url'] = search_url.format(query=urlencode({'q': query}),
	search_options=urlencode(search_options))		search_options=urlencode(search_options))
	@@ -61,30 +61,27 @@ def request(query, params):

	# get response from search-request		# get response from search-request
	def response(resp):		def response(resp):
			results = []

	dom = html.fromstring(resp.text)		dom = html.fromstring(resp.text)

	results = []		# parse results
	for element in dom.xpath('//div[@id="search"] //td'):		for result in dom.xpath('//div[contains(@class, "rg_meta")]/text()'):
	link = element.xpath('./a')[0]
			try:
	google_url = urlparse(link.xpath('.//@href')[0])		metadata = loads(result)
	query = parse_qs(google_url.query)		img_format = "{0} {1}x{2}".format(metadata['ity'], str(metadata['ow']), str(metadata['oh']))
	source_url = next(iter(query.get('q', [])), None)		source = "{0} ({1})".format(metadata['st'], metadata['isu'])
			results.append({'url': metadata['ru'],
	title_parts = element.xpath('./cite//following-sibling::*/text()')		'title': metadata['pt'],
	title_parts.extend(element.xpath('./cite//following-sibling::text()')[:-1])		'content': metadata['s'],
			'source': source,
	result = {		'img_format': img_format,
	'title': ''.join(title_parts),		'thumbnail_src': metadata['tu'],
	'content': '',		'img_src': metadata['ou'],
	'template': 'images.html',		'template': 'images.html'})
	'url': source_url,
	'img_src': source_url,		except:
	'thumbnail_src': next(iter(link.xpath('.//img //@src')), None)
	}

	if not source_url or not result['thumbnail_src']:
	continue		continue

	results.append(result)
	return results		return results

searx/engines/seedpeer.py

deleted100644 → 0

+0 −75

Original line number	Original line	Diff line number	Diff line
	# Seedpeer (Videos, Music, Files)
	#
	# @website http://seedpeer.eu
	# @provide-api no (nothing found)
	#
	# @using-api no
	# @results HTML (using search portal)
	# @stable yes (HTML can change)
	# @parse url, title, content, seed, leech, magnetlink

	from lxml import html
	from operator import itemgetter
	from searx.url_utils import quote, urljoin


	url = 'http://www.seedpeer.eu/'
	search_url = url + 'search/{search_term}/7/{page_no}.html'
	# specific xpath variables
	torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
	alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
	title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
	alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
	seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
	alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
	peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
	alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
	age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
	alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
	size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
	alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'


	# do search-request
	def request(query, params):
	params['url'] = search_url.format(search_term=quote(query),
	page_no=params['pageno'] - 1)
	return params


	# get response from search-request
	def response(resp):
	results = []
	dom = html.fromstring(resp.text)
	torrent_links = dom.xpath(torrent_xpath)
	if len(torrent_links) > 0:
	seeds = dom.xpath(seeds_xpath)
	peers = dom.xpath(peers_xpath)
	titles = dom.xpath(title_xpath)
	sizes = dom.xpath(size_xpath)
	ages = dom.xpath(age_xpath)
	else: # under ~5 results uses a different xpath
	torrent_links = dom.xpath(alternative_torrent_xpath)
	seeds = dom.xpath(alternative_seeds_xpath)
	peers = dom.xpath(alternative_peers_xpath)
	titles = dom.xpath(alternative_title_xpath)
	sizes = dom.xpath(alternative_size_xpath)
	ages = dom.xpath(alternative_age_xpath)
	# return empty array if nothing is found
	if not torrent_links:
	return []

	# parse results
	for index, result in enumerate(torrent_links):
	link = result.attrib.get('href')
	href = urljoin(url, link)
	results.append({'url': href,
	'title': titles[index].text_content(),
	'content': '{}, {}'.format(sizes[index], ages[index]),
	'seed': seeds[index],
	'leech': peers[index],

	'template': 'torrent.html'})

	# return results sorted by seeder
	return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/subtitleseeker.py

deleted100644 → 0

+0 −86

Original line number	Original line	Diff line number	Diff line
	"""
	Subtitleseeker (Video)

	@website http://www.subtitleseeker.com
	@provide-api no

	@using-api no
	@results HTML
	@stable no (HTML can change)
	@parse url, title, content
	"""

	from lxml import html
	from searx.languages import language_codes
	from searx.engines.xpath import extract_text
	from searx.url_utils import quote_plus

	# engine dependent config
	categories = ['videos']
	paging = True
	language = ""

	# search-url
	url = 'http://www.subtitleseeker.com/'
	search_url = url + 'search/TITLES/{query}?p={pageno}'

	# specific xpath variables
	results_xpath = '//div[@class="boxRows"]'


	# do search-request
	def request(query, params):
	params['url'] = search_url.format(query=quote_plus(query),
	pageno=params['pageno'])
	return params


	# get response from search-request
	def response(resp):
	results = []

	dom = html.fromstring(resp.text)

	search_lang = ""

	# dirty fix for languages named differenly in their site
	if resp.search_params['language'][:2] == 'fa':
	search_lang = 'Farsi'
	elif resp.search_params['language'] == 'pt-BR':
	search_lang = 'Brazilian'
	elif resp.search_params['language'] != 'all':
	search_lang = [lc[3]
	for lc in language_codes
	if lc[0].split('-')[0] == resp.search_params['language'].split('-')[0]]
	search_lang = search_lang[0].split(' (')[0]

	# parse results
	for result in dom.xpath(results_xpath):
	link = result.xpath(".//a")[0]
	href = link.attrib.get('href')

	if language is not "":
	href = href + language + '/'
	elif search_lang:
	href = href + search_lang + '/'

	title = extract_text(link)

	content = extract_text(result.xpath('.//div[contains(@class,"red")]'))
	content = content + " - "
	text = extract_text(result.xpath('.//div[contains(@class,"grey-web")]')[0])
	content = content + text

	if result.xpath(".//span") != []:
	content = content +\
	" - (" +\
	extract_text(result.xpath(".//span")) +\
	")"

	# append result
	results.append({'url': href,
	'title': title,
	'content': content})

	# return results
	return results

searx/engines/swisscows.py

deleted100644 → 0

+0 −125

Original line number	Original line	Diff line number	Diff line
	"""
	Swisscows (Web, Images)

	@website https://swisscows.ch
	@provide-api no

	@using-api no
	@results HTML (using search portal)
	@stable no (HTML can change)
	@parse url, title, content
	"""

	from json import loads
	import re
	from lxml.html import fromstring
	from searx.url_utils import unquote, urlencode
	from searx.utils import match_language

	# engine dependent config
	categories = ['general', 'images']
	paging = True
	language_support = True

	# search-url
	base_url = 'https://swisscows.ch/'
	search_string = '?{query}&page={page}'

	supported_languages_url = base_url

	# regex
	regex_json = re.compile(r'initialData: {"Request":(.\|\n)},\senvironment')
	regex_json_remove_start = re.compile(r'^initialData:\s*')
	regex_json_remove_end = re.compile(r',\s*environment$')
	regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')


	# do search-request
	def request(query, params):
	if params['language'] == 'all':
	ui_language = 'browser'
	region = 'browser'
	else:
	region = match_language(params['language'], supported_languages, language_aliases)
	ui_language = region.split('-')[0]

	search_path = search_string.format(
	query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
	page=params['pageno']
	)

	# image search query is something like 'image?{query}&page={page}'
	if params['category'] == 'images':
	search_path = 'image' + search_path

	params['url'] = base_url + search_path

	return params


	# get response from search-request
	def response(resp):
	results = []

	json_regex = regex_json.search(resp.text)

	# check if results are returned
	if not json_regex:
	return []

	json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
	json = loads(json_raw)

	# parse results
	for result in json['Results'].get('items', []):
	result_title = result['Title'].replace('\\uE000', '').replace('\\uE001', '')

	# parse image results
	if result.get('ContentType', '').startswith('image'):
	img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

	# append result
	results.append({'url': result['SourceUrl'],
	'title': result['Title'],
	'content': '',
	'img_src': img_url,
	'template': 'images.html'})

	# parse general results
	else:
	result_url = result['Url'].replace('\\uE000', '').replace('\\uE001', '')
	result_content = result['Description'].replace('\\uE000', '').replace('\\uE001', '')

	# append result
	results.append({'url': result_url,
	'title': result_title,
	'content': result_content})

	# parse images
	for result in json.get('Images', []):
	# decode image url
	img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

	# append result
	results.append({'url': result['SourceUrl'],
	'title': result['Title'],
	'content': '',
	'img_src': img_url,
	'template': 'images.html'})

	# return results
	return results


	# get supported languages from their site
	def _fetch_supported_languages(resp):
	supported_languages = []
	dom = fromstring(resp.text)
	options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
	for option in options:
	code = option.xpath('./@data-search-language')[0]
	if code.startswith('nb-'):
	code = code.replace('nb', 'no', 1)
	supported_languages.append(code)

	return supported_languages