Merge branch 'nicofonk/sync-upstream' into 'master'

Sync upstream

See merge request e/cloud/my-spot!68

.github/workflows/data-update.yml

@@ -40,8 +40,7 @@ jobs:
         env:
           FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
         run: |
-          source local/py3/bin/activate
-          $FETCH_SCRIPT
+          V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
 
       - name: Create Pull Request
         id: cpr

.github/workflows/integration.yml

@@ -59,7 +59,7 @@ jobs:
     - name: Install node dependencies
       run: make V=1 node.env
     - name: Build themes
-      run: make V=1 themes
+      run: make V=1 themes.all
 
   documentation:
     name: Documentation
@@ -77,14 +77,15 @@ jobs:
         python-version: '3.9'
         architecture: 'x64'
     - name: Build documentation
-      run: SEARX_DEBUG=1 make V=1 ci-gh-pages
+      run: |
+        make V=1 docs.clean docs.html
     - name: Deploy
       if:  github.ref == 'refs/heads/master'
       uses: JamesIves/github-pages-deploy-action@3.7.1
       with:
         GITHUB_TOKEN: ${{ github.token }}
         BRANCH: gh-pages
-        FOLDER: build/gh-pages
+        FOLDER: dist/docs
         CLEAN: true # Automatically remove deleted files from the deploy branch
 
   dockers:

.gitlab-ci.yml

@@ -11,20 +11,17 @@ services:
 
 python:
   stage: check
-  before_script:
-    - ./manage.sh update_dev_packages
   script:
-    - make test.pep8
-    - make test.pylint
+    - ./manage test.pep8
+    - ./manage test.pylint
 
 build:web:
   stage: build
   before_script:
-    - ./manage.sh npm_packages
-    - ./manage.sh update_dev_packages
+    - ./manage node.env
   script:
-    - ./manage.sh locales
-    - make V=1 themes.etheme
+    - ./manage themes.etheme
+    - ./manage babel.compile
 
 .build:docker:
   stage: build
@@ -56,10 +53,8 @@ build:docker:tags:
 
 test:unit:
   stage: test
-  before_script:
-    - ./manage.sh update_dev_packages
   script:
-    - make test.coverage
+    - ./manage test.coverage
   coverage: '/TOTAL.*\s+(\d+%)$/'
 
 .deploy:template:

AUTHORS.rst

-Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Markus Heiser.
+Searx was created by Adam Tauber and is maintained by Adam Tauber, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Émilien Devos.
 
 Major contributing authors:
 
@@ -12,8 +12,9 @@ Major contributing authors:
 - @pofilo
 - Markus Heiser @return42
 - Émilien Devos @unixfox
+- Alexandre Flament
 
-People who have submitted patches/translates, reported bugs, consulted features or
+People who have submitted patches/translations, reported bugs, consulted features or
 generally made searx better:
 
 - Laszlo Hammerl
@@ -164,3 +165,4 @@ generally made searx better:
 - @jhigginbotham
 - @xenrox
 - @OliveiraHermogenes
+- Paul Alcock @Guilvareux

Dockerfile

@@ -3,7 +3,6 @@ FROM registry.gitlab.e.foundation:5000/e/cloud/my-spot/env as builder
 COPY . /src/
 RUN pip install --force-reinstall --prefix /install /src
 
-
 FROM python:3.9-slim
 LABEL maintainer="spot <https://gitlab.e.foundation/e/cloud/my-spot/>"
 LABEL description="A privacy-respecting, hackable metasearch engine."

Makefile

 # -*- coding: utf-8; mode: makefile-gmake -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
 .DEFAULT_GOAL=help
+export MTOOLS=./manage
 
 include utils/makefile.include
 
-PYOBJECTS = searx
-DOC       = docs
-PY_SETUP_EXTRAS ?= \[test\]
-PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,E1136,W0707,E0237
-PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases
-
-include utils/makefile.python
-include utils/makefile.sphinx
-
 all: clean install
 
-PHONY += help-min help-all help
-
-help: help-min
-	@echo  ''
-	@echo  'to get more help:  make help-all'
-
-help-min:
-	@echo  '  test      - run developer tests'
-	@echo  '  docs      - build documentation'
-	@echo  '  docs-live - autobuild HTML documentation while editing'
-	@echo  '  run       - run developer instance'
-	@echo  '  install   - developer install (./local)'
-	@echo  '  uninstall - uninstall (./local)'
-	@echo  '  gh-pages  - build docs & deploy on gh-pages branch'
-	@echo  '  clean     - drop builds and environments'
-	@echo  '  project   - re-build generic files of the searx project'
-	@echo  '  buildenv  - re-build environment files (aka brand)'
-	@echo  '  themes    - re-build build the source of the themes'
-	@echo  '  docker    - build Docker image'
-	@echo  '  node.env  - download & install npm dependencies locally'
-	@echo  ''
-	@$(MAKE) -e -s make-help
-
-help-all: help-min
-	@echo  ''
-	@$(MAKE) -e -s python-help
-	@echo  ''
-	@$(MAKE) -e -s docs-help
+PHONY += help
 
-PHONY += install
-install: buildenv pyenvinstall
+help:
+	@./manage --help
+	@echo '----'
+	@echo 'run            - run developer instance'
+	@echo 'install        - developer install of searx into virtualenv'
+	@echo 'uninstall      - uninstall developer installation'
+	@echo 'clean          - clean up working tree'
+	@echo 'search.checker - check search engines'
+	@echo 'test           - run shell & CI tests'
+	@echo 'test.sh        - test shell scripts'
+	@echo 'ci.test        - run CI tests'
 
-PHONY += uninstall
-uninstall: pyenvuninstall
-
-PHONY += clean
-clean: pyclean docs-clean node.clean test.clean
-	$(call cmd,common_clean)
 
 PHONY += run
-run:  buildenv pyenvinstall
+run:  install
 	$(Q) ( \
 	sleep 2 ; \
 	xdg-open http://127.0.0.1:8888/ ; \
 	) &
-	SEARX_DEBUG=1 $(PY_ENV)/bin/python ./searx/webapp.py
-
-# docs
-# ----
-
-sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
-
-PHONY += docs
-docs:  sphinx-doc-prebuilds
-	$(call cmd,sphinx,html,docs,docs)
-
-PHONY += docs-live
-docs-live:  sphinx-doc-prebuilds
-	$(call cmd,sphinx_autobuild,html,docs,docs)
-
-PHONY += prebuild-includes
-prebuild-includes:
-	$(Q)mkdir -p $(DOCS_BUILD)/includes
-	$(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
-	$(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
-	$(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
-
-
-$(GH_PAGES)::
-	@echo "doc available at --> $(DOCS_URL)"
-
-# update project files
-# --------------------
-
-PHONY += project engines.languages useragents.update buildenv
-
-project: buildenv useragents.update engines.languages
-
-engines.languages:  pyenvinstall
-	$(Q)echo "fetch languages .."
-	$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_languages.py
-	$(Q)echo "updated searx/data/engines_languages.json"
-	$(Q)echo "updated searx/languages.py"
-
-useragents.update:  pyenvinstall
-	$(Q)echo "fetch useragents .."
-	$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_firefox_version.py
-	$(Q)echo "updated searx/data/useragents.json with the most recent versions of Firefox."
-
-buildenv: pyenv
-	$(Q)$(PY_ENV_ACT); SEARX_DEBUG=1 python utils/build_env.py
-
-# node / npm
-# ----------
+	SEARX_DEBUG=1 ./manage pyenv.cmd python ./searx/webapp.py
 
-node.env: buildenv
-	$(Q)./manage.sh npm_packages
+PHONY += install uninstall
+install uninstall:
+	$(Q)./manage pyenv.$@
 
-node.clean:
-	$(Q)echo "CLEAN     locally installed npm dependencies"
-	$(Q)rm -rf \
-	  ./node_modules  \
-	  ./package-lock.json \
-	  ./searx/static/themes/etheme/package-lock.json \
-	  ./searx/static/themes/etheme/node_modules
-
-# build themes
-# ------------
-
-PHONY += themes themes.etheme
-themes: buildenv themes.etheme
-
-quiet_cmd_lessc = LESSC     $3
-      cmd_lessc = PATH="$$(npm bin):$$PATH" \
-	lessc --clean-css="--s1 --advanced --compatibility=ie9" "searx/static/$2" "searx/static/$3"
-
-quiet_cmd_grunt = GRUNT     $2
-      cmd_grunt = PATH="$$(npm bin):$$PATH" \
-	grunt --gruntfile  "$2"
-
-themes.etheme: node.env
-	$(Q)echo '[!] build etheme theme'
-	$(call cmd,grunt,searx/static/themes/etheme/gruntfile.js)
-
-# docker
-# ------
-
-PHONY += docker
-docker: buildenv
-	$(Q)./manage.sh docker_build
-
-docker.push: buildenv
-	$(Q)./manage.sh docker_build push
-
-# gecko
-# -----
-
-PHONY += gecko.driver
-gecko.driver:
-	$(PY_ENV_ACT); ./manage.sh install_geckodriver
-
-# search.checker
-# --------------
-
-search.checker: pyenvinstall
-	$(Q)$(PY_ENV_ACT); searx-checker -v
-
-ENGINE_TARGETS=$(patsubst searx/engines/%.py,search.checker.%,$(wildcard searx/engines/[!_]*.py))
-
-$(ENGINE_TARGETS): pyenvinstall
-	$(Q)$(PY_ENV_ACT); searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
-
-
-# test
-# ----
-
-PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
-test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
-
-PYLINT_FILES=\
-	searx/preferences.py \
-	searx/testing.py \
-	searx/engines/gigablast.py \
-	searx/engines/deviantart.py \
-	searx/engines/digg.py \
-	searx/engines/google.py \
-	searx/engines/google_news.py \
-	searx/engines/google_videos.py \
-	searx/engines/google_images.py \
-	searx/engines/mediathekviewweb.py \
-	searx/engines/solidtorrents.py \
-	searx/engines/solr.py \
-	searx/engines/google_scholar.py \
-	searx/engines/yahoo_news.py \
-	searx/engines/apkmirror.py \
-	searx_extra/update/update_external_bangs.py
-
-test.pylint: pyenvinstall
-	$(call cmd,pylint,$(PYLINT_FILES))
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \
-		searx/engines \
-	)
-	$(call cmd,pylint,\
-		--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
-		--ignore=searx/engines \
-		searx tests \
-	)
+PHONY += clean
+clean: py.clean docs.clean node.clean test.clean
+	$(Q)./manage build_msg CLEAN  "common files"
+	$(Q)find . -name '*.orig' -exec rm -f {} +
+	$(Q)find . -name '*.rej' -exec rm -f {} +
+	$(Q)find . -name '*~' -exec rm -f {} +
+	$(Q)find . -name '*.bak' -exec rm -f {} +
+
+PHONY += search.checker search.checker.%
+search.checker: install
+	$(Q)./manage pyenv.cmd searx-checker -v
+
+search.checker.%: install
+	$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
+
+PHONY += ci.test test test.sh
+ci.test: test.pep8 test.pylint test.unit test.robot
+test: ci.test
+test.sh:
+	$(Q)shellcheck -x -s bash \
+		utils/brand.env \
+		./manage \
+		utils/lib.sh \
+	        utils/filtron.sh \
+	        utils/searx.sh \
+	        utils/morty.sh \
+	        utils/lxc.sh \
+	        utils/lxc-searx.env \
+	        .config.sh
+	$(Q)./manage build_msg TEST "$@ OK"
 
-# ignored rules:
-#  E402 module level import not at top of file
-#  W503 line break before binary operator
 
-# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
-test.sh:
-	shellcheck -x -s bash utils/brand.env
-	shellcheck -x utils/lib.sh
-	shellcheck -x utils/filtron.sh
-	shellcheck -x utils/searx.sh
-	shellcheck -x utils/morty.sh
-	shellcheck -x utils/lxc.sh
-	shellcheck -x utils/lxc-searx.env
-	shellcheck -x .config.sh
+# wrap ./manage script
 
-test.pep8: pyenvinstall
-	@echo "TEST      pycodestyle (formerly pep8)"
-	$(Q)$(PY_ENV_ACT); pycodestyle --exclude='searx/static, searx/languages.py, $(foreach f,$(PYLINT_FILES),$(f),)' \
-        --max-line-length=120 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" searx tests
+MANAGE += buildenv
+MANAGE += babel.compile
+MANAGE += data.all data.languages data.useragents
+MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
+MANAGE += docker.build docker.push
+MANAGE += gecko.driver
+MANAGE += node.env node.clean
+MANAGE += py.build py.clean
+MANAGE += pyenv pyenv.install pyenv.uninstall
+MANAGE += pypi.upload pypi.upload.test
+MANAGE += test.pylint test.pep8 test.unit test.coverage test.robot test.clean
+MANAGE += themes.all themes.oscar themes.simple themes.bootstrap
 
-test.unit: pyenvinstall
-	@echo "TEST      tests/unit"
-	$(Q)$(PY_ENV_ACT); python -m nose2 -s tests/unit
+PHONY += $(MANAGE)
 
-test.coverage:  pyenvinstall
-	@echo "TEST      unit test coverage"
-	$(Q)$(PY_ENV_ACT); \
-	python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit \
-	&& coverage report \
-	&& coverage html \
+$(MANAGE):
+	$(Q)$(MTOOLS) $@
 
-test.robot: pyenvinstall gecko.driver
-	@echo "TEST      robot"
-	$(Q)$(PY_ENV_ACT); PYTHONPATH=. python searx/testing.py robot
+# deprecated
 
-test.clean:
-	@echo "CLEAN     intermediate test stuff"
-	$(Q)rm -rf geckodriver.log .coverage coverage/
+PHONY += docs docs-clean docs-live docker themes
 
+docs: docs.html
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.html
 
-# travis
-# ------
+docs-clean: docs.clean
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.clean
 
-PHONY += ci.test
-ci.test:
-	$(PY_ENV_BIN)/python -c "import yaml"  || make clean
-	$(MAKE) test
+docs-live: docs.live
+	$(Q)./manage build_msg WARN $@ is deprecated use docs.live
 
-travis.codecov:
-	$(Q)$(PY_ENV_BIN)/python -m pip install codecov
+docker:  docker.build
+	$(Q)./manage build_msg WARN $@ is deprecated use docker.build
 
-.PHONY: $(PHONY)
+themes: themes.all
+	$(Q)./manage build_msg WARN $@ is deprecated use themes.all

dockerfiles/docker-entrypoint.sh

@@ -24,9 +24,6 @@ if [ -z "${BIND_ADDRESS}" ]; then
     export BIND_ADDRESS="${DEFAULT_BIND_ADDRESS}"
 fi
 
-export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
-export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
-
 # Parse special command line
 # see docs/admin/installation-docker.rst
 # display the help message without the version

docs/admin/buildhosts.rst

@@ -49,9 +49,9 @@ Build docs
    - dvisvgm_
 
 Most of the sphinx requirements are installed from :origin:`setup.py` and the
-docs can be build from scratch with ``make docs``.  For better math and image
-processing additional packages are needed.  The XeTeX_ needed not only for PDF
-creation, its also needed for :ref:`math` when HTML output is build.
+docs can be build from scratch with ``make docs.html``.  For better math and
+image processing additional packages are needed.  The XeTeX_ needed not only for
+PDF creation, its also needed for :ref:`math` when HTML output is build.
 
 To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
 as images (``sphinx.ext.imgmath`` extension).
@@ -64,7 +64,7 @@ to ``imgmath``:
    :start-after: # sphinx.ext.imgmath setup
    :end-before: # sphinx.ext.imgmath setup END
 
-If your docs build (``make docs``) shows warnings like this::
+If your docs build (``make docs.html``) shows warnings like this::
 
    WARNING: dot(1) not found, for better output quality install \
             graphviz from https://www.graphviz.org

docs/admin/installation-docker.rst

@@ -51,7 +51,7 @@ It's also possible to build searx from the embedded Dockerfile.
 
    git clone https://github.com/searx/searx.git
    cd searx
-   make docker
+   make docker.build
 
 
 Public instance

docs/admin/installation-searx.rst

@@ -61,7 +61,7 @@ from the login (*~/.profile*):
 
 .. tip::
 
-   Open a second terminal for the configuration tasks and left the ``(searx)$``
+   Open a second terminal for the configuration tasks and leave the ``(searx)$``
    terminal open for the tasks below.
 
 

docs/admin/installation.rst

@@ -39,13 +39,18 @@ install from ``root``, take into account that the scripts are creating a
 these new created users do need read access to the clone of searx, which is not
 the case if you clone into a folder below ``/root``.
 
-
 .. code:: bash
 
    $ cd ~/Downloads
    $ git clone https://github.com/searx/searx searx
    $ cd searx
 
+.. sidebar:: further read
+
+   - :ref:`toolboxing`
+   - :ref:`update searx`
+   - :ref:`inspect searx`
+
 **Install** :ref:`searx service <searx.sh>`
 
 This installs searx as described in :ref:`installation basic`.

docs/admin/settings.rst

@@ -130,14 +130,12 @@ Global Settings
        request_timeout : 2.0        # default timeout in seconds, can be override by engine
        # max_request_timeout: 10.0  # the maximum timeout in seconds
        useragent_suffix : ""        # informations like an email address to the administrator
-       pool_connections : 100       # Number of different hosts
-       pool_maxsize : 10            # Number of simultaneous requests by host
+       pool_connections : 100       # Maximum number of allowable connections, or None for no limits. The default is 100.
+       pool_maxsize : 10            # Number of allowable keep-alive connections, or None to always allow. The default is 10.
+       enable_http2: True           # See https://www.python-httpx.org/http2/
    # uncomment below section if you want to use a proxy
    #    proxies:
-   #        http:
-   #            - http://proxy1:8080
-   #            - http://proxy2:8080
-   #        https:
+   #        all://:
    #            - http://proxy1:8080
    #            - http://proxy2:8080
    # uncomment below section only if you have more than one network interface
@@ -145,6 +143,7 @@ Global Settings
    #    source_ips:
    #        - 1.1.1.1
    #        - 1.1.1.2
+   #        - fe80::/126
 
 
 ``request_timeout`` :
@@ -157,20 +156,46 @@ Global Settings
   Suffix to the user-agent searx uses to send requests to others engines.  If an
   engine wish to block you, a contact info here may be useful to avoid that.
 
-.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies
-.. _PySocks: https://pypi.org/project/PySocks/
+``keepalive_expiry``:
+  Number of seconds to keep a connection in the pool. By default 5.0 seconds.
+
+.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
 
 ``proxies`` :
-  Define one or more proxies you wish to use, see `requests proxies`_.
+  Define one or more proxies you wish to use, see `httpx proxies`_.
   If there are more than one proxy for one protocol (http, https),
   requests to the engines are distributed in a round-robin fashion.
 
-  - Proxy: `see <https://2.python-requests.org/en/latest/user/advanced/#proxies>`__.
-  - SOCKS proxies are also supported: `see <https://2.python-requests.org/en/latest/user/advanced/#socks>`__
-
 ``source_ips`` :
   If you use multiple network interfaces, define from which IP the requests must
-  be made. This parameter is ignored when ``proxies`` is set.
+  be made. Example:
+
+  * ``0.0.0.0`` any local IPv4 address.
+  * ``::`` any local IPv6 address.
+  * ``192.168.0.1``
+  * ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses
+  * ``fe80::60a2:1691:e5a2:ee1f``
+  * ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.
+  * ``[ 192.168.0.1, fe80::/126 ]``
+
+``retries`` :
+  Number of retry in case of an HTTP error.
+  On each retry, searx uses an different proxy and source ip.
+
+``retry_on_http_error`` :
+  Retry request on some HTTP status code.
+
+  Example:
+
+  * ``true`` : on HTTP status code between 400 and 599.
+  * ``403`` : on HTTP status code 403.
+  * ``[403, 429]``: on HTTP status code 403 and 429.
+
+``enable_http2`` :
+  Enable by default. Set to ``False`` to disable HTTP/2.
+
+``max_redirects`` :
+  30 by default. Maximum redirect before it is an error.
 
 
 ``locales:``
@@ -216,6 +241,13 @@ Engine settings
      api_key : 'apikey'
      disabled : True
      language : en_US
+     #enable_http: False
+     #enable_http2: False
+     #retries: 1
+     #retry_on_http_error: True # or 403 or [404, 429]
+     #max_connections: 100
+     #max_keepalive_connections: 10
+     #keepalive_expiry: 5.0
      #proxies:
      #    http:
      #        - http://proxy1:8080
@@ -270,6 +302,12 @@ Engine settings
 ``display_error_messages`` : default ``True``
   When an engine returns an error, the message is displayed on the user interface.
 
+``network``: optional
+  Use the network configuration from another engine.
+  In addition, there are two default networks:
+  * ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)
+  * ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)
+
 .. note::
 
    A few more options are possible, but they are pretty specific to some

docs/admin/update-searx.rst

@@ -4,20 +4,56 @@
 How to update
 =============
 
+How to update depends on the :ref:`installation` method.  If you have used the
+:ref:`installation scripts`, use ``update`` command from the scripts.
+
+**Update** :ref:`searx service <searx.sh>`
+
+.. code:: sh
+
+    sudo -H ./utils/searx.sh update searx
+
+**Update** :ref:`filtron reverse proxy <filtron.sh>`
+
 .. code:: sh
 
-    sudo -H -u searx -i
-    (searx)$ git stash
-    (searx)$ git pull origin master
-    (searx)$ git stash apply
-    (searx)$ ./manage.sh update_packages
+    sudo -H ./utils/filtron.sh update filtron
+
+**Update** :ref:`result proxy <morty.sh>`