Commit b6a4da74 authored by Nicolas Gelot's avatar Nicolas Gelot
Browse files

Merge branch 'nicofonk/sync-upstream' into 'master'

Sync upstream

See merge request e/cloud/my-spot!68
parents 40c698b6 834e9b34
......@@ -40,8 +40,7 @@ jobs:
env:
FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}
run: |
source local/py3/bin/activate
$FETCH_SCRIPT
V=1 ./manage pyenv.cmd python "$FETCH_SCRIPT"
- name: Create Pull Request
id: cpr
......
......@@ -59,7 +59,7 @@ jobs:
- name: Install node dependencies
run: make V=1 node.env
- name: Build themes
run: make V=1 themes
run: make V=1 themes.all
documentation:
name: Documentation
......@@ -77,14 +77,15 @@ jobs:
python-version: '3.9'
architecture: 'x64'
- name: Build documentation
run: SEARX_DEBUG=1 make V=1 ci-gh-pages
run: |
make V=1 docs.clean docs.html
- name: Deploy
if: github.ref == 'refs/heads/master'
uses: JamesIves/github-pages-deploy-action@3.7.1
with:
GITHUB_TOKEN: ${{ github.token }}
BRANCH: gh-pages
FOLDER: build/gh-pages
FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch
dockers:
......
......@@ -11,20 +11,17 @@ services:
python:
stage: check
before_script:
- ./manage.sh update_dev_packages
script:
- make test.pep8
- make test.pylint
- ./manage test.pep8
- ./manage test.pylint
build:web:
stage: build
before_script:
- ./manage.sh npm_packages
- ./manage.sh update_dev_packages
- ./manage node.env
script:
- ./manage.sh locales
- make V=1 themes.etheme
- ./manage themes.etheme
- ./manage babel.compile
.build:docker:
stage: build
......@@ -56,10 +53,8 @@ build:docker:tags:
test:unit:
stage: test
before_script:
- ./manage.sh update_dev_packages
script:
- make test.coverage
- ./manage test.coverage
coverage: '/TOTAL.*\s+(\d+%)$/'
.deploy:template:
......
Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Markus Heiser.
Searx was created by Adam Tauber and is maintained by Adam Tauber, Noémi Ványi, @pofilo, Gaspard d'Hautefeuille and Émilien Devos.
Major contributing authors:
......@@ -12,8 +12,9 @@ Major contributing authors:
- @pofilo
- Markus Heiser @return42
- Émilien Devos @unixfox
- Alexandre Flament
People who have submitted patches/translates, reported bugs, consulted features or
People who have submitted patches/translations, reported bugs, consulted features or
generally made searx better:
- Laszlo Hammerl
......@@ -164,3 +165,4 @@ generally made searx better:
- @jhigginbotham
- @xenrox
- @OliveiraHermogenes
- Paul Alcock @Guilvareux
......@@ -3,7 +3,6 @@ FROM registry.gitlab.e.foundation:5000/e/cloud/my-spot/env as builder
COPY . /src/
RUN pip install --force-reinstall --prefix /install /src
FROM python:3.9-slim
LABEL maintainer="spot <https://gitlab.e.foundation/e/cloud/my-spot/>"
LABEL description="A privacy-respecting, hackable metasearch engine."
......
# -*- coding: utf-8; mode: makefile-gmake -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
.DEFAULT_GOAL=help
export MTOOLS=./manage
include utils/makefile.include
PYOBJECTS = searx
DOC = docs
PY_SETUP_EXTRAS ?= \[test\]
PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,E1136,W0707,E0237
PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases
include utils/makefile.python
include utils/makefile.sphinx
all: clean install
PHONY += help-min help-all help
help: help-min
@echo ''
@echo 'to get more help: make help-all'
help-min:
@echo ' test - run developer tests'
@echo ' docs - build documentation'
@echo ' docs-live - autobuild HTML documentation while editing'
@echo ' run - run developer instance'
@echo ' install - developer install (./local)'
@echo ' uninstall - uninstall (./local)'
@echo ' gh-pages - build docs & deploy on gh-pages branch'
@echo ' clean - drop builds and environments'
@echo ' project - re-build generic files of the searx project'
@echo ' buildenv - re-build environment files (aka brand)'
@echo ' themes - re-build build the source of the themes'
@echo ' docker - build Docker image'
@echo ' node.env - download & install npm dependencies locally'
@echo ''
@$(MAKE) -e -s make-help
help-all: help-min
@echo ''
@$(MAKE) -e -s python-help
@echo ''
@$(MAKE) -e -s docs-help
PHONY += help
PHONY += install
install: buildenv pyenvinstall
help:
@./manage --help
@echo '----'
@echo 'run - run developer instance'
@echo 'install - developer install of searx into virtualenv'
@echo 'uninstall - uninstall developer installation'
@echo 'clean - clean up working tree'
@echo 'search.checker - check search engines'
@echo 'test - run shell & CI tests'
@echo 'test.sh - test shell scripts'
@echo 'ci.test - run CI tests'
PHONY += uninstall
uninstall: pyenvuninstall
PHONY += clean
clean: pyclean docs-clean node.clean test.clean
$(call cmd,common_clean)
PHONY += run
run: buildenv pyenvinstall
run: install
$(Q) ( \
sleep 2 ; \
xdg-open http://127.0.0.1:8888/ ; \
) &
SEARX_DEBUG=1 $(PY_ENV)/bin/python ./searx/webapp.py
# docs
# ----
sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes
PHONY += docs
docs: sphinx-doc-prebuilds
$(call cmd,sphinx,html,docs,docs)
PHONY += docs-live
docs-live: sphinx-doc-prebuilds
$(call cmd,sphinx_autobuild,html,docs,docs)
PHONY += prebuild-includes
prebuild-includes:
$(Q)mkdir -p $(DOCS_BUILD)/includes
$(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst
$(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst
$(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst
$(GH_PAGES)::
@echo "doc available at --> $(DOCS_URL)"
# update project files
# --------------------
PHONY += project engines.languages useragents.update buildenv
project: buildenv useragents.update engines.languages
engines.languages: pyenvinstall
$(Q)echo "fetch languages .."
$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_languages.py
$(Q)echo "updated searx/data/engines_languages.json"
$(Q)echo "updated searx/languages.py"
useragents.update: pyenvinstall
$(Q)echo "fetch useragents .."
$(Q)$(PY_ENV_ACT); python ./searx_extra/update/update_firefox_version.py
$(Q)echo "updated searx/data/useragents.json with the most recent versions of Firefox."
buildenv: pyenv
$(Q)$(PY_ENV_ACT); SEARX_DEBUG=1 python utils/build_env.py
# node / npm
# ----------
SEARX_DEBUG=1 ./manage pyenv.cmd python ./searx/webapp.py
node.env: buildenv
$(Q)./manage.sh npm_packages
PHONY += install uninstall
install uninstall:
$(Q)./manage pyenv.$@
node.clean:
$(Q)echo "CLEAN locally installed npm dependencies"
$(Q)rm -rf \
./node_modules \
./package-lock.json \
./searx/static/themes/etheme/package-lock.json \
./searx/static/themes/etheme/node_modules
# build themes
# ------------
PHONY += themes themes.etheme
themes: buildenv themes.etheme
quiet_cmd_lessc = LESSC $3
cmd_lessc = PATH="$$(npm bin):$$PATH" \
lessc --clean-css="--s1 --advanced --compatibility=ie9" "searx/static/$2" "searx/static/$3"
quiet_cmd_grunt = GRUNT $2
cmd_grunt = PATH="$$(npm bin):$$PATH" \
grunt --gruntfile "$2"
themes.etheme: node.env
$(Q)echo '[!] build etheme theme'
$(call cmd,grunt,searx/static/themes/etheme/gruntfile.js)
# docker
# ------
PHONY += docker
docker: buildenv
$(Q)./manage.sh docker_build
docker.push: buildenv
$(Q)./manage.sh docker_build push
# gecko
# -----
PHONY += gecko.driver
gecko.driver:
$(PY_ENV_ACT); ./manage.sh install_geckodriver
# search.checker
# --------------
search.checker: pyenvinstall
$(Q)$(PY_ENV_ACT); searx-checker -v
ENGINE_TARGETS=$(patsubst searx/engines/%.py,search.checker.%,$(wildcard searx/engines/[!_]*.py))
$(ENGINE_TARGETS): pyenvinstall
$(Q)$(PY_ENV_ACT); searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
# test
# ----
PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot
test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot
PYLINT_FILES=\
searx/preferences.py \
searx/testing.py \
searx/engines/gigablast.py \
searx/engines/deviantart.py \
searx/engines/digg.py \
searx/engines/google.py \
searx/engines/google_news.py \
searx/engines/google_videos.py \
searx/engines/google_images.py \
searx/engines/mediathekviewweb.py \
searx/engines/solidtorrents.py \
searx/engines/solr.py \
searx/engines/google_scholar.py \
searx/engines/yahoo_news.py \
searx/engines/apkmirror.py \
searx_extra/update/update_external_bangs.py
test.pylint: pyenvinstall
$(call cmd,pylint,$(PYLINT_FILES))
$(call cmd,pylint,\
--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
--additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \
searx/engines \
)
$(call cmd,pylint,\
--disable=$(PYLINT_SEARX_DISABLE_OPTION) \
--ignore=searx/engines \
searx tests \
)
PHONY += clean
clean: py.clean docs.clean node.clean test.clean
$(Q)./manage build_msg CLEAN "common files"
$(Q)find . -name '*.orig' -exec rm -f {} +
$(Q)find . -name '*.rej' -exec rm -f {} +
$(Q)find . -name '*~' -exec rm -f {} +
$(Q)find . -name '*.bak' -exec rm -f {} +
PHONY += search.checker search.checker.%
search.checker: install
$(Q)./manage pyenv.cmd searx-checker -v
search.checker.%: install
$(Q)./manage pyenv.cmd searx-checker -v "$(subst _, ,$(patsubst search.checker.%,%,$@))"
PHONY += ci.test test test.sh
ci.test: test.pep8 test.pylint test.unit test.robot
test: ci.test
test.sh:
$(Q)shellcheck -x -s bash \
utils/brand.env \
./manage \
utils/lib.sh \
utils/filtron.sh \
utils/searx.sh \
utils/morty.sh \
utils/lxc.sh \
utils/lxc-searx.env \
.config.sh
$(Q)./manage build_msg TEST "$@ OK"
# ignored rules:
# E402 module level import not at top of file
# W503 line break before binary operator
# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported!
test.sh:
shellcheck -x -s bash utils/brand.env
shellcheck -x utils/lib.sh
shellcheck -x utils/filtron.sh
shellcheck -x utils/searx.sh
shellcheck -x utils/morty.sh
shellcheck -x utils/lxc.sh
shellcheck -x utils/lxc-searx.env
shellcheck -x .config.sh
# wrap ./manage script
test.pep8: pyenvinstall
@echo "TEST pycodestyle (formerly pep8)"
$(Q)$(PY_ENV_ACT); pycodestyle --exclude='searx/static, searx/languages.py, $(foreach f,$(PYLINT_FILES),$(f),)' \
--max-line-length=120 --ignore "E117,E252,E402,E722,E741,W503,W504,W605" searx tests
MANAGE += buildenv
MANAGE += babel.compile
MANAGE += data.all data.languages data.useragents
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push
MANAGE += gecko.driver
MANAGE += node.env node.clean
MANAGE += py.build py.clean
MANAGE += pyenv pyenv.install pyenv.uninstall
MANAGE += pypi.upload pypi.upload.test
MANAGE += test.pylint test.pep8 test.unit test.coverage test.robot test.clean
MANAGE += themes.all themes.oscar themes.simple themes.bootstrap
test.unit: pyenvinstall
@echo "TEST tests/unit"
$(Q)$(PY_ENV_ACT); python -m nose2 -s tests/unit
PHONY += $(MANAGE)
test.coverage: pyenvinstall
@echo "TEST unit test coverage"
$(Q)$(PY_ENV_ACT); \
python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit \
&& coverage report \
&& coverage html \
$(MANAGE):
$(Q)$(MTOOLS) $@
test.robot: pyenvinstall gecko.driver
@echo "TEST robot"
$(Q)$(PY_ENV_ACT); PYTHONPATH=. python searx/testing.py robot
# deprecated
test.clean:
@echo "CLEAN intermediate test stuff"
$(Q)rm -rf geckodriver.log .coverage coverage/
PHONY += docs docs-clean docs-live docker themes
docs: docs.html
$(Q)./manage build_msg WARN $@ is deprecated use docs.html
# travis
# ------
docs-clean: docs.clean
$(Q)./manage build_msg WARN $@ is deprecated use docs.clean
PHONY += ci.test
ci.test:
$(PY_ENV_BIN)/python -c "import yaml" || make clean
$(MAKE) test
docs-live: docs.live
$(Q)./manage build_msg WARN $@ is deprecated use docs.live
travis.codecov:
$(Q)$(PY_ENV_BIN)/python -m pip install codecov
docker: docker.build
$(Q)./manage build_msg WARN $@ is deprecated use docker.build
.PHONY: $(PHONY)
themes: themes.all
$(Q)./manage build_msg WARN $@ is deprecated use themes.all
......@@ -24,9 +24,6 @@ if [ -z "${BIND_ADDRESS}" ]; then
export BIND_ADDRESS="${DEFAULT_BIND_ADDRESS}"
fi
export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini
export SEARX_SETTINGS_PATH=/etc/searx/settings.yml
# Parse special command line
# see docs/admin/installation-docker.rst
# display the help message without the version
......
......@@ -49,9 +49,9 @@ Build docs
- dvisvgm_
Most of the sphinx requirements are installed from :origin:`setup.py` and the
docs can be build from scratch with ``make docs``. For better math and image
processing additional packages are needed. The XeTeX_ needed not only for PDF
creation, its also needed for :ref:`math` when HTML output is build.
docs can be build from scratch with ``make docs.html``. For better math and
image processing additional packages are needed. The XeTeX_ needed not only for
PDF creation, its also needed for :ref:`math` when HTML output is build.
To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered
as images (``sphinx.ext.imgmath`` extension).
......@@ -64,7 +64,7 @@ to ``imgmath``:
:start-after: # sphinx.ext.imgmath setup
:end-before: # sphinx.ext.imgmath setup END
If your docs build (``make docs``) shows warnings like this::
If your docs build (``make docs.html``) shows warnings like this::
WARNING: dot(1) not found, for better output quality install \
graphviz from https://www.graphviz.org
......
......@@ -51,7 +51,7 @@ It's also possible to build searx from the embedded Dockerfile.
git clone https://github.com/searx/searx.git
cd searx
make docker
make docker.build
Public instance
......
......@@ -61,7 +61,7 @@ from the login (*~/.profile*):
.. tip::
Open a second terminal for the configuration tasks and left the ``(searx)$``
Open a second terminal for the configuration tasks and leave the ``(searx)$``
terminal open for the tasks below.
......
......@@ -39,13 +39,18 @@ install from ``root``, take into account that the scripts are creating a
these new created users do need read access to the clone of searx, which is not
the case if you clone into a folder below ``/root``.
.. code:: bash
$ cd ~/Downloads
$ git clone https://github.com/searx/searx searx
$ cd searx
.. sidebar:: further read
- :ref:`toolboxing`
- :ref:`update searx`
- :ref:`inspect searx`
**Install** :ref:`searx service <searx.sh>`
This installs searx as described in :ref:`installation basic`.
......
......@@ -130,14 +130,12 @@ Global Settings
request_timeout : 2.0 # default timeout in seconds, can be override by engine
# max_request_timeout: 10.0 # the maximum timeout in seconds
useragent_suffix : "" # informations like an email address to the administrator
pool_connections : 100 # Number of different hosts
pool_maxsize : 10 # Number of simultaneous requests by host
pool_connections : 100 # Maximum number of allowable connections, or None for no limits. The default is 100.
pool_maxsize : 10 # Number of allowable keep-alive connections, or None to always allow. The default is 10.
enable_http2: True # See https://www.python-httpx.org/http2/
# uncomment below section if you want to use a proxy
# proxies:
# http:
# - http://proxy1:8080
# - http://proxy2:8080
# https:
# all://:
# - http://proxy1:8080
# - http://proxy2:8080
# uncomment below section only if you have more than one network interface
......@@ -145,6 +143,7 @@ Global Settings
# source_ips:
# - 1.1.1.1
# - 1.1.1.2
# - fe80::/126
``request_timeout`` :
......@@ -157,20 +156,46 @@ Global Settings
Suffix to the user-agent searx uses to send requests to others engines. If an
engine wish to block you, a contact info here may be useful to avoid that.
.. _requests proxies: https://requests.readthedocs.io/en/latest/user/advanced/#proxies
.. _PySocks: https://pypi.org/project/PySocks/
``keepalive_expiry``:
Number of seconds to keep a connection in the pool. By default 5.0 seconds.
.. _httpx proxies: https://www.python-httpx.org/advanced/#http-proxying
``proxies`` :
Define one or more proxies you wish to use, see `requests proxies`_.
Define one or more proxies you wish to use, see `httpx proxies`_.
If there are more than one proxy for one protocol (http, https),
requests to the engines are distributed in a round-robin fashion.
- Proxy: `see <https://2.python-requests.org/en/latest/user/advanced/#proxies>`__.
- SOCKS proxies are also supported: `see <https://2.python-requests.org/en/latest/user/advanced/#socks>`__
``source_ips`` :
If you use multiple network interfaces, define from which IP the requests must
be made. This parameter is ignored when ``proxies`` is set.
be made. Example:
* ``0.0.0.0`` any local IPv4 address.
* ``::`` any local IPv6 address.
* ``192.168.0.1``
* ``[ 192.168.0.1, 192.168.0.2 ]`` these two specific IP addresses
* ``fe80::60a2:1691:e5a2:ee1f``
* ``fe80::60a2:1691:e5a2:ee1f/126`` all IP addresses in this network.
* ``[ 192.168.0.1, fe80::/126 ]``
``retries`` :
Number of retry in case of an HTTP error.
On each retry, searx uses an different proxy and source ip.
``retry_on_http_error`` :
Retry request on some HTTP status code.
Example:
* ``true`` : on HTTP status code between 400 and 599.
* ``403`` : on HTTP status code 403.
* ``[403, 429]``: on HTTP status code 403 and 429.
``enable_http2`` :
Enable by default. Set to ``False`` to disable HTTP/2.
``max_redirects`` :
30 by default. Maximum redirect before it is an error.
``locales:``
......@@ -216,6 +241,13 @@ Engine settings
api_key : 'apikey'
disabled : True
language : en_US
#enable_http: False
#enable_http2: False
#retries: 1
#retry_on_http_error: True # or 403 or [404, 429]
#max_connections: 100
#max_keepalive_connections: 10
#keepalive_expiry: 5.0
#proxies:
# http:
# - http://proxy1:8080
......@@ -270,6 +302,12 @@ Engine settings
``display_error_messages`` : default ``True``
When an engine returns an error, the message is displayed on the user interface.
``network``: optional
Use the network configuration from another engine.
In addition, there are two default networks:
* ``ipv4`` set ``local_addresses`` to ``0.0.0.0`` (use only IPv4 local addresses)
* ``ipv6`` set ``local_addresses`` to ``::`` (use only IPv6 local addresses)
.. note::
A few more options are possible, but they are pretty specific to some
......
......@@ -4,20 +4,56 @@
How to update
=============
How to update depends on the :ref:`installation` method. If you have used the
:ref:`installation scripts`, use ``update`` command from the scripts.
**Update** :ref:`searx service <searx.sh>`
.. code:: sh
sudo -H ./utils/searx.sh update searx
**Update** :ref:`filtron reverse proxy <filtron.sh>`
.. code:: sh
sudo -H -u searx -i
(searx)$ git stash
(searx)$ git pull origin master
(searx)$ git stash apply
(searx)$ ./manage.sh update_packages
sudo -H ./utils/filtron.sh update filtron
**Update** :ref:`result proxy <morty.sh>`