Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 32e8c2cf authored by Alexandre Flament's avatar Alexandre Flament Committed by Alexandre FLAMENT
Browse files

searx.network: add "verify" option to the networks 

Each network can define a verify option:
* false to disable certificate verification
* a path to existing certificate.

SearXNG uses SSL_CERT_FILE and SSL_CERT_DIR when they are defined
see https://www.python-httpx.org/environment_variables/#ssl_cert_file
parent 72f6367e

docs/admin/engines/settings.rst

+32 −12
Original line number Diff line number Diff line
@@ -347,18 +347,27 @@ Communication with search engines. 
     pool_maxsize: 10           # Number of allowable keep-alive connections, or null

                                # to always allow. The default is 10.

     enable_http2: true         # See https://www.python-httpx.org/http2/

     # uncomment below section if you want to use a proxy

     # uncomment below section if you want to use a custom server certificate

     # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults

     # and https://www.python-httpx.org/compatibility/#ssl-configuration

     #  verify: ~/.mitmproxy/mitmproxy-ca-cert.cer

     #

     # uncomment below section if you want to use a proxyq see: SOCKS proxies

     #   https://2.python-requests.org/en/latest/user/advanced/#proxies

     # are also supported: see

     #   https://2.python-requests.org/en/latest/user/advanced/#socks

     #

     #  proxies:

     #    all://:

     #      - http://proxy1:8080

     #      - http://proxy2:8080

     # uncomment below section only if you have more than one network interface

     # which can be the source of outgoing search requests

     # source_ips:

     #   - 1.1.1.1

     #   - 1.1.1.2

     #   - fe80::/126



     #

     #  using_tor_proxy: true

     #

     # Extra seconds to add in order to account for the time taken by the proxy

     #

     #  extra_proxy_timeout: 10.0

     #



``request_timeout`` :

  Global timeout of the requests made to others engines in seconds.  A bigger
@@ -408,6 +417,17 @@ Communication with search engines. 
``enable_http2`` :

  Enable by default. Set to ``false`` to disable HTTP/2.



.. _httpx verification defaults: https://www.python-httpx.org/advanced/#changing-the-verification-defaults

.. _httpx ssl configuration: https://www.python-httpx.org/compatibility/#ssl-configuration



``verify``: : ``$SSL_CERT_FILE``, ``$SSL_CERT_DIR``

  Allow to specify a path to certificate.

  see `httpx verification defaults`_.



  In addition to ``verify``, SearXNG supports the ``$SSL_CERT_FILE`` (for a file) and

  ``$SSL_CERT_DIR`` (for a directory) OpenSSL variables.  

  see `httpx ssl configuration`_.



``max_redirects`` :

  30 by default. Maximum redirect before it is an error.

searx/network/client.py

+2 −7
Original line number Diff line number Diff line
@@ -26,9 +26,6 @@ else: 
logger = logger.getChild('searx.network.client')

LOOP = None

SSLCONTEXTS: Dict[Any, SSLContext] = {}

TRANSPORT_KWARGS = {

    'trust_env': False,

}





def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
@@ -74,7 +71,7 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit 
        rdns = True



    proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)

    verify = get_sslcontexts(proxy_url, None, True, False, http2) if verify is True else verify

    verify = get_sslcontexts(proxy_url, None, verify, True, http2) if verify is True else verify

    return AsyncProxyTransportFixed(

        proxy_type=proxy_type,

        proxy_host=proxy_host,
@@ -88,12 +85,11 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit 
        local_address=local_address,

        limits=limit,

        retries=retries,

        **TRANSPORT_KWARGS,

    )





def get_transport(verify, http2, local_address, proxy_url, limit, retries):

    verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify

    verify = get_sslcontexts(None, None, verify, True, http2) if verify is True else verify

    return httpx.AsyncHTTPTransport(

        # pylint: disable=protected-access

        verify=verify,
@@ -102,7 +98,6 @@ def get_transport(verify, http2, local_address, proxy_url, limit, retries): 
        proxy=httpx._config.Proxy(proxy_url) if proxy_url else None,

        local_address=local_address,

        retries=retries,

        **TRANSPORT_KWARGS,

    )

searx/network/network.py

+1 −1
Original line number Diff line number Diff line
@@ -334,7 +334,7 @@ def initialize(settings_engines=None, settings_outgoing=None): 
    # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121  # pylint: disable=line-too-long

    default_params = {

        'enable_http': False,

        'verify': True,

        'verify': settings_outgoing['verify'],

        'enable_http2': settings_outgoing['enable_http2'],

        'max_connections': settings_outgoing['pool_connections'],

        'max_keepalive_connections': settings_outgoing['pool_maxsize'],

searx/search/processors/online.py

+14 −4
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 


from timeit import default_timer

import asyncio

import ssl

import httpx



import searx.network
@@ -29,7 +30,6 @@ def default_request_params(): 
        'data': {},

        'url': '',

        'cookies': {},

        'verify': True,

        'auth': None

        # fmt: on

    }
@@ -76,9 +76,15 @@ class OnlineProcessor(EngineProcessor): 
    def _send_http_request(self, params):

        # create dictionary which contain all

        # information about the request

        request_args = dict(

            headers=params['headers'], cookies=params['cookies'], verify=params['verify'], auth=params['auth']

        )

        request_args = dict(headers=params['headers'], cookies=params['cookies'], auth=params['auth'])



        # verify

        # if not None, it overrides the verify value defined in the network.

        # use False to accept any server certificate

        # use a path to file to specify a server certificate

        verify = params.get('verify')

        if verify is not None:

            request_args['verify'] = params['verify']



        # max_redirects

        max_redirects = params.get('max_redirects')
@@ -153,6 +159,10 @@ class OnlineProcessor(EngineProcessor): 
            # send requests and parse the results

            search_results = self._search_basic(query, params)

            self.extend_container(result_container, start_time, search_results)

        except ssl.SSLError as e:

            # requests timeout (connect or read)

            self.handle_exception(result_container, e, suspend=True)

            self.logger.error("SSLError {}, verify={}".format(e, searx.network.get_network(self.engine_name).verify))

        except (httpx.TimeoutException, asyncio.TimeoutError) as e:

            # requests timeout (connect or read)

            self.handle_exception(result_container, e, suspend=True)

searx/settings.yml

+5 −0
Original line number Diff line number Diff line
@@ -145,6 +145,11 @@ outgoing: 
  pool_maxsize: 20

  # See https://www.python-httpx.org/http2/

  enable_http2: true

  # uncomment below section if you want to use a custom server certificate

  # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults

  # and https://www.python-httpx.org/compatibility/#ssl-configuration

  #  verify: ~/.mitmproxy/mitmproxy-ca-cert.cer

  #

  # uncomment below section if you want to use a proxyq see: SOCKS proxies

  #   https://2.python-requests.org/en/latest/user/advanced/#proxies

  # are also supported: see