Merge pull request #2225 from dalf/processors (9b27935f) · Commits · e / infra / spot

docs/admin/engines.rst

+6 −6

Original line number	Diff line number	Diff line
		@@ -33,7 +33,7 @@ Engine .. Paging support P
		------------------------- -------------------- ------------
		Shortcut S Language support L
		Timeout TO Time range support TR
		Disabled D Offline O
		Disabled D Engine type ET
		------------- ----------- -------------------- ------------
		Safe search SS
		------------- ----------- ---------------------------------
		@@ -62,7 +62,7 @@ Show errors DE
		- SS
		- D
		- TR
		- O
		- ET
		- W
		- D
		- DE
		@@ -79,7 +79,7 @@ Show errors DE
		- {{(mod.safesearch and "y") or ""}}
		- {{(mod.disabled and "y") or ""}}
		- {{(mod.time_range_support and "y") or ""}}
		- {{(mod.offline and "y") or ""}}
		- {{mod.engine_type or ""}}
		- {{mod.weight or 1 }}
		- {{(mod.disabled and "y") or ""}}
		- {{(mod.display_error_messages and "y") or ""}}

docs/dev/engine_overview.rst

+47 −19

Original line number	Diff line number	Diff line
		@@ -37,15 +37,16 @@ settings. However, the standard way is the following:
		engine file
		-----------

		======================= =========== ===========================================
		======================= =========== ========================================================
		argument type information
		======================= =========== ===========================================
		======================= =========== ========================================================
		categories list pages, in which the engine is working
		paging boolean support multible pages
		language_support boolean support language choosing
		time_range_support boolean support search time range
		offline boolean engine runs offline
		======================= =========== ===========================================
		engine_type str ``online`` by default, other possibles values are
		``offline``, ``online_dictionnary``, ``online_currency``
		======================= =========== ========================================================

		.. _engine settings:

		@@ -111,21 +112,48 @@ passed arguments
		These arguments can be used to construct the search query. Furthermore,
		parameters with default value can be redefined for special purposes.

		====================== ============ ========================================================================
		If the ``engine_type`` is ``online```:

		====================== ============== ========================================================================
		argument type default-value, information
		====================== ============ ========================================================================
		url string ``''``
		method string ``'GET'``
		====================== ============== ========================================================================
		url str ``''``
		method str ``'GET'``
		headers set ``{}``
		data set ``{}``
		cookies set ``{}``
		verify boolean ``True``
		headers.User-Agent string a random User-Agent
		category string current category, like ``'general'``
		started datetime current date-time
		verify bool ``True``
		headers.User-Agent str a random User-Agent
		category str current category, like ``'general'``
		safesearch int ``0``, between ``0`` and ``2`` (normal, moderate, strict)
		time_range Optional[str] ``None``, can be ``day``, ``week``, ``month``, ``year``
		pageno int current pagenumber
		language string specific language code like ``'en_US'``, or ``'all'`` if unspecified
		language str specific language code like ``'en_US'``, or ``'all'`` if unspecified
		====================== ============== ========================================================================


		If the ``engine_type`` is ``online_dictionnary```, in addition to the ``online`` arguments:

		====================== ============ ========================================================================
		argument type default-value, information
		====================== ============ ========================================================================
		from_lang str specific language code like ``'en_US'``
		to_lang str specific language code like ``'en_US'``
		query str the text query without the languages
		====================== ============ ========================================================================

		If the ``engine_type`` is ``online_currency```, in addition to the ``online`` arguments:

		====================== ============ ========================================================================
		argument type default-value, information
		====================== ============ ========================================================================
		amount float the amount to convert
		from str ISO 4217 code
		to str ISO 4217 code
		from_name str currency name
		to_name str currency name
		====================== ============ ========================================================================


		parsed arguments
		----------------
		@@ -137,12 +165,12 @@ request:
		=================== =========== ==========================================================================
		argument type information
		=================== =========== ==========================================================================
		url string requested url
		method string HTTP request method
		url str requested url
		method str HTTP request method
		headers set HTTP header information
		data set HTTP data information (parsed if ``method != 'GET'``)
		data set HTTP data information
		cookies set HTTP cookies
		verify boolean Performing SSL-Validity check
		verify bool Performing SSL-Validity check
		max_redirects int maximum redirects, hard limit
		soft_max_redirects int maximum redirects, soft limit. Record an error but don't stop the engine
		raise_for_httperror bool True by default: raise an exception if the HTTP code of response is >= 300

searx/engines/init.py

+6 −4

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ engine_default_args = {'paging': False,
		'suspend_end_time': 0,
		'continuous_errors': 0,
		'time_range_support': False,
		'offline': False,
		'engine_type': 'online',
		'display_error_messages': True,
		'tokens': []}

		@@ -142,7 +142,9 @@ def load_engine(engine_data):
		'errors': 0
		}

		if not engine.offline:
		engine_type = getattr(engine, 'engine_type', 'online')

		if engine_type != 'offline':
		engine.stats['page_load_time'] = 0
		engine.stats['page_load_count'] = 0

		@@ -209,7 +211,7 @@ def get_engines_stats(preferences):
		else:
		score = score_per_result = 0.0

		if not engine.offline:
		if engine.engine_type != 'offline':
		load_times = 0
		if engine.stats['page_load_count'] != 0:
		load_times = engine.stats['page_load_time'] / float(engine.stats['page_load_count']) # noqa
		@@ -300,7 +302,7 @@ def initialize_engines(engine_list):

		def _set_https_support_for_engine(engine):
		# check HTTPS support if it is not disabled
		if not engine.offline and not hasattr(engine, 'https_support'):
		if engine.engine_type != 'offline' and not hasattr(engine, 'https_support'):
		params = engine.request('http_test', {
		'method': 'GET',
		'headers': {},

searx/engines/command.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -23,7 +23,7 @@ from threading import Thread
		from searx import logger


		offline = True
		engine_type = 'offline'
		paging = True
		command = []
		delimiter = {}

searx/engines/currency_convert.py

+2 −40

Original line number	Diff line number	Diff line
		import json
		import re
		import unicodedata
		from searx.data import CURRENCIES # NOQA


		engine_type = 'online_currency'
		categories = []
		url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'
		weight = 100

		parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in\|to) ([^.0-9]+)', re.I)
		https_support = True


		def normalize_name(name):
		name = name.lower().replace('-', ' ').rstrip('s')
		name = re.sub(' +', ' ', name)
		return unicodedata.normalize('NFKD', name).lower()


		def name_to_iso4217(name):
		global CURRENCIES

		name = normalize_name(name)
		currency = CURRENCIES['names'].get(name, [name])
		return currency[0]


		def iso4217_to_name(iso4217, language):
		global CURRENCIES

		return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)


		def request(query, params):
		m = parser_re.match(query)
		if not m:
		# wrong query
		return params
		amount, from_currency, to_currency = m.groups()
		amount = float(amount)
		from_currency = name_to_iso4217(from_currency.strip())
		to_currency = name_to_iso4217(to_currency.strip())

		params['url'] = url.format(from_currency, to_currency)
		params['amount'] = amount
		params['from'] = from_currency
		params['to'] = to_currency
		params['from_name'] = iso4217_to_name(from_currency, 'en')
		params['to_name'] = iso4217_to_name(to_currency, 'en')

		params['url'] = url.format(params['from'], params['to'])
		return params