[enh] settings.yml: add use_default_settings option (2nd version) (b4b81a5e) · Commits · e / infra / spot

Makefile

+1 −16

Original line number	Original line	Diff line number	Diff line
	@@ -266,19 +266,4 @@ test.clean:
	travis.codecov:		travis.codecov:
	$(Q)$(PY_ENV_BIN)/python -m pip install codecov		$(Q)$(PY_ENV_BIN)/python -m pip install codecov


	# user-settings
	# -------------

	PHONY += user-settings.create user-settings.update

	user-settings.update: pyenvinstall
	$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
	$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py ${SEARX_SETTINGS_PATH}

	user-settings.update.engines: pyenvinstall
	$(Q)$(PY_ENV_ACT); pip install ruamel.yaml
	$(Q)$(PY_ENV_ACT); python utils/update_user_settings.py --add-engines ${SEARX_SETTINGS_PATH}


	.PHONY: $(PHONY)		.PHONY: $(PHONY)

docs/admin/settings.rst

+25 −42

Original line number	Original line	Diff line number	Diff line
	@@ -235,68 +235,51 @@ In the following example, the actual settings are the default settings defined i

	.. code-block:: yaml		.. code-block:: yaml

	use_default_settings: true		use_default_settings: True
	server:		server:
	secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"		secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
	server:
	bind_address: "0.0.0.0"		bind_address: "0.0.0.0"

	With ``use_default_settings: True``, each settings can be override in a similar way with one exception, the ``engines`` section:		With ``use_default_settings: True``, each settings can be override in a similar way, the ``engines`` section is merged according to the engine ``name``.

	* If the ``engines`` section is not defined in the user settings, searx uses the engines from the default setttings (the above example).
	* If the ``engines`` section is defined then:

	* searx loads only the engines declare in the user setttings.
	* searx merges the configuration according to the engine name.

	In the following example, only three engines are available. Each engine configuration is merged with the default configuration.		In this example, searx will load all the engine and the arch linux wiki engine has a :ref:`token<private engines>`:

	.. code-block:: yaml		.. code-block:: yaml

	use_default_settings: true		use_default_settings: True
	server:		server:
	secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"		secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
	engines:		engines:
	- name: wikipedia		- name: arch linux wiki
	- name: wikidata		tokens: ['$ecretValue']
	- name: ddg definitions

	Another example where four engines are available. The arch linux wiki engine has a :ref:`token<private engines>`.		It is possible to remove some engines from the default settings. The following example is similar to the above one, but searx doesn't load the the google engine:

	.. code-block:: yaml		.. code-block:: yaml

	use_default_settings: true		use_default_settings:
			engines:
			remove:
			- google
	server:		server:
	secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"		secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
	engines:		engines:
	- name: arch linux wiki		- name: arch linux wiki
	tokens: ['$ecretValue']		tokens: ['$ecretValue']
	- name: wikipedia
	- name: wikidata
	- name: ddg definitions

	automatic update
	----------------

	The following comand creates or updates a minimal user settings (a secret key is defined if it is not already the case):

	.. code-block:: sh

	make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update

	Set ``SEARX_SETTINGS_PATH`` to your user settings path.

	As soon the user settings contains an ``engines`` section, it becomes difficult to keep the engine list updated.		As an alternative, it is possible to specify the engines to keep. In the following example, searx has only two engines:
	The following command creates or updates the user settings including the ``engines`` section:

	.. code-block:: sh		.. code-block:: yaml

	make SEARX_SETTINGS_PATH=/etc/searx/settings.yml user-settings.update.engines

	After that ``/etc/searx/settings.yml``

	* has a ``secret key``
	* has a ``engine`` section if it is not already the case, moreover the command:

	* has deleted engines that do not exist in the default settings.		use_default_settings:
	* has added engines that exist in the default settings but are not declare in the user settings.		engines:
			keep_only:
			- google
			- duckduckgo
			server:
			secret_key: "uvys6bRhKHUdFF5CqbJonSDSRN8H0sCBziNSrDGNVdpz7IeZhveVart3yvghoKHA"
			engines:
			- name: google
			tokens: ['$ecretValue']
			- name: duckduckgo
			tokens: ['$ecretValue']

searx/init.py

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -16,7 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
	'''		'''

	import logging		import logging
	import searx.settings		import searx.settings_loader
	from os import environ		from os import environ
	from os.path import realpath, dirname, join, abspath, isfile		from os.path import realpath, dirname, join, abspath, isfile

	@@ -24,7 +24,7 @@ from os.path import realpath, dirname, join, abspath, isfile
	searx_dir = abspath(dirname(__file__))		searx_dir = abspath(dirname(__file__))
	engine_dir = dirname(realpath(__file__))		engine_dir = dirname(realpath(__file__))
	static_path = abspath(join(dirname(__file__), 'static'))		static_path = abspath(join(dirname(__file__), 'static'))
	settings, settings_load_message = searx.settings.load_settings()		settings, settings_load_message = searx.settings_loader.load_settings()

	if settings['ui']['static_path']:		if settings['ui']['static_path']:
	static_path = settings['ui']['static_path']		static_path = settings['ui']['static_path']

searx/settings.py→searx/settings_loader.py

+129 −0

Original line number	Original line	Diff line number	Diff line
	import collections.abc		# SPDX-License-Identifier: AGPL-3.0-or-later

	import yaml
	from searx.exceptions import SearxSettingsException
	from os import environ		from os import environ
	from os.path import dirname, join, abspath, isfile		from os.path import dirname, join, abspath, isfile
			from collections.abc import Mapping
			from itertools import filterfalse

			import yaml

			from searx.exceptions import SearxSettingsException


	searx_dir = abspath(dirname(__file__))		searx_dir = abspath(dirname(__file__))
	@@ -12,17 +16,13 @@ searx_dir = abspath(dirname(__file__))
	def check_settings_yml(file_name):		def check_settings_yml(file_name):
	if isfile(file_name):		if isfile(file_name):
	return file_name		return file_name
	else:
	return None		return None


	def load_yaml(file_name):		def load_yaml(file_name):
	try:		try:
	with open(file_name, 'r', encoding='utf-8') as settings_yaml:		with open(file_name, 'r', encoding='utf-8') as settings_yaml:
	settings = yaml.safe_load(settings_yaml)		return yaml.safe_load(settings_yaml)
	if not isinstance(settings, dict) or len(settings) == 0:
	raise SearxSettingsException('Empty file', file_name)
	return settings
	except IOError as e:		except IOError as e:
	raise SearxSettingsException(e, file_name)		raise SearxSettingsException(e, file_name)
	except yaml.YAMLError as e:		except yaml.YAMLError as e:
	@@ -39,35 +39,73 @@ def get_user_settings_path():
	# if possible set path to settings using the		# if possible set path to settings using the
	# enviroment variable SEARX_SETTINGS_PATH		# enviroment variable SEARX_SETTINGS_PATH
	return check_settings_yml(environ['SEARX_SETTINGS_PATH'])		return check_settings_yml(environ['SEARX_SETTINGS_PATH'])
	else:
	# if not, get it from searx code base or last solution from /etc/searx		# if not, get it from searx code base or last solution from /etc/searx
	return check_settings_yml('/etc/searx/settings.yml')		return check_settings_yml('/etc/searx/settings.yml')


	def update_dict(d, u):		def update_dict(default_dict, user_dict):
	for k, v in u.items():		for k, v in user_dict.items():
	if isinstance(v, collections.abc.Mapping):		if isinstance(v, Mapping):
	d[k] = update_dict(d.get(k, {}), v)		default_dict[k] = update_dict(default_dict.get(k, {}), v)
	else:		else:
	d[k] = v		default_dict[k] = v
	return d		return default_dict


	def update_settings(default_settings, user_settings):		def update_settings(default_settings, user_settings):
			# merge everything except the engines
	for k, v in user_settings.items():		for k, v in user_settings.items():
	if k == 'use_default_settings':		if k not in ('use_default_settings', 'engines'):
	continue
	elif k == 'engines':
	default_engines = default_settings[k]
	default_engines_dict = dict((definition['name'], definition) for definition in default_engines)
	default_settings[k] = [update_dict(default_engines_dict[definition['name']], definition)
	for definition in v]
	else:
	update_dict(default_settings[k], v)		update_dict(default_settings[k], v)

			# parse the engines
			remove_engines = None
			keep_only_engines = None
			use_default_settings = user_settings.get('use_default_settings')
			if isinstance(use_default_settings, dict):
			remove_engines = use_default_settings.get('engines', {}).get('remove')
			keep_only_engines = use_default_settings.get('engines', {}).get('keep_only')

			if 'engines' in user_settings or remove_engines is not None or keep_only_engines is not None:
			engines = default_settings['engines']

			# parse "use_default_settings.engines.remove"
			if remove_engines is not None:
			engines = list(filterfalse(lambda engine: (engine.get('name')) in remove_engines, engines))

			# parse "use_default_settings.engines.keep_only"
			if keep_only_engines is not None:
			engines = list(filter(lambda engine: (engine.get('name')) in keep_only_engines, engines))

			# parse "engines"
			user_engines = user_settings.get('engines')
			if user_engines:
			engines_dict = dict((definition['name'], definition) for definition in engines)
			for user_engine in user_engines:
			default_engine = engines_dict.get(user_engine['name'])
			if default_engine:
			update_dict(default_engine, user_engine)
			else:
			engines.append(user_engine)

			# store the result
			default_settings['engines'] = engines

	return default_settings		return default_settings


			def is_use_default_settings(user_settings):
			use_default_settings = user_settings.get('use_default_settings')
			if use_default_settings is True:
			return True
			if isinstance(use_default_settings, dict):
			return True
			if use_default_settings is False or use_default_settings is None:
			return False
			raise ValueError('Invalid value for use_default_settings')


	def load_settings(load_user_setttings=True):		def load_settings(load_user_setttings=True):
	default_settings_path = get_default_settings_path()		default_settings_path = get_default_settings_path()
	user_settings_path = get_user_settings_path()		user_settings_path = get_user_settings_path()
	@@ -78,7 +116,7 @@ def load_settings(load_user_setttings=True):

	# user settings		# user settings
	user_settings = load_yaml(user_settings_path)		user_settings = load_yaml(user_settings_path)
	if user_settings.get('use_default_settings'):		if is_use_default_settings(user_settings):
	# the user settings are merged with the default configuration		# the user settings are merged with the default configuration
	default_settings = load_yaml(default_settings_path)		default_settings = load_yaml(default_settings_path)
	update_settings(default_settings, user_settings)		update_settings(default_settings, user_settings)

tests/unit/settings/empty_settings.yml

0 → 100644

+0 −0

Empty file added.