Merge pull request #2375 from dalf/raise_for_errors (9f8ebeca) · Commits · e / infra / spot

docs/dev/engine_overview.rst

+13 −13

Original line number	Diff line number	Diff line
		@@ -134,9 +134,9 @@ The function ``def request(query, params):`` always returns the ``params``
		variable. Inside searx, the following paramters can be used to specify a search
		request:

		================== =========== ==========================================================================
		=================== =========== ==========================================================================
		argument type information
		================== =========== ==========================================================================
		=================== =========== ==========================================================================
		url string requested url
		method string HTTP request method
		headers set HTTP header information
		@@ -145,8 +145,8 @@ cookies set HTTP cookies
		verify boolean Performing SSL-Validity check
		max_redirects int maximum redirects, hard limit
		soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
		raise_for_status bool True by default: raise an exception if the HTTP code of response is >= 300
		================== =========== ==========================================================================
		raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300
		=================== =========== ==========================================================================


		example code

searx/engines/init.py

+6 −2

Original line number	Diff line number	Diff line
		@@ -281,7 +281,11 @@ def initialize_engines(engine_list):
		load_engines(engine_list)

		def engine_init(engine_name, init_fn):
		try:
		init_fn(get_engine_from_settings(engine_name))
		except Exception:
		logger.exception('%s engine: Fail to initialize', engine_name)
		else:
		logger.debug('%s engine: Initialized', engine_name)

		for engine_name, engine in engines.items():

searx/engines/qwant.py

+16 −11

Original line number	Diff line number	Diff line
		@@ -14,6 +14,8 @@ from datetime import datetime
		from json import loads
		from urllib.parse import urlencode
		from searx.utils import html_to_text, match_language
		from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
		from searx.raise_for_httperror import raise_for_httperror


		# engine dependent config
		@@ -24,8 +26,7 @@ supported_languages_url = 'https://qwant.com/region'

		category_to_keyword = {'general': 'web',
		'images': 'images',
		'news': 'news',
		'social media': 'social'}
		'news': 'news'}

		# search-url
		url = 'https://api.qwant.com/api/search/{keyword}?count=10&offset={offset}&f=&{query}&t={keyword}&uiv=4'
		@@ -51,6 +52,7 @@ def request(query, params):
		params['url'] += '&locale=' + language.replace('-', '_').lower()

		params['headers']['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64; rv:69.0) Gecko/20100101 Firefox/69.0'
		params['raise_for_httperror'] = False
		return params


		@@ -58,8 +60,20 @@ def request(query, params):
		def response(resp):
		results = []

		# According to https://www.qwant.com/js/app.js
		if resp.status_code == 429:
		raise SearxEngineCaptchaException()

		# raise for other errors
		raise_for_httperror(resp)

		# load JSON result
		search_results = loads(resp.text)

		# check for an API error
		if search_results.get('status') != 'success':
		raise SearxEngineAPIException('API error ' + str(search_results.get('error', '')))

		# return empty array if there are no results
		if 'data' not in search_results:
		return []
		@@ -90,15 +104,6 @@ def response(resp):
		'thumbnail_src': thumbnail_src,
		'img_src': img_src})

		elif category_to_keyword.get(categories[0], '') == 'social':
		published_date = datetime.fromtimestamp(result['date'], None)
		img_src = result.get('img', None)
		results.append({'url': res_url,
		'title': title,
		'publishedDate': published_date,
		'content': content,
		'img_src': img_src})

		elif category_to_keyword.get(categories[0], '') == 'news':
		published_date = datetime.fromtimestamp(result['date'], None)
		media = result.get('media', [])

searx/engines/wikidata.py

+0 −3

Original line number	Diff line number	Diff line
		@@ -161,9 +161,6 @@ def request(query, params):

		def response(resp):
		results = []
		if resp.status_code != 200:
		logger.debug('SPARQL endpoint error %s', resp.content.decode())
		resp.raise_for_status()
		jsonresponse = loads(resp.content.decode())

		language = resp.search_params['language'].lower()

searx/engines/wikipedia.py

+3 −1

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@ from urllib.parse import quote
		from json import loads
		from lxml.html import fromstring
		from searx.utils import match_language, searx_useragent
		from searx.raise_for_httperror import raise_for_httperror

		# search-url
		search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
		@@ -37,7 +38,7 @@ def request(query, params):
		language=url_lang(params['language']))

		params['headers']['User-Agent'] = searx_useragent()
		params['raise_for_status'] = False
		params['raise_for_httperror'] = False
		params['soft_max_redirects'] = 2

		return params
		@@ -47,6 +48,7 @@ def request(query, params):
		def response(resp):
		if resp.status_code == 404:
		return []
		raise_for_httperror(resp)

		results = []
		api_result = loads(resp.text)