Commit 4bcb8983 authored by Nivesh Krishna's avatar Nivesh Krishna
Browse files

Merge branch '4172-fix-crash' into 'master'

Updating lib version which fix TooManyStreams error

Closes e/backlog#4172

See merge request e/cloud/my-spot!96
parents b7f8aadc 5a3c3d8b
...@@ -7,11 +7,11 @@ lxml==4.6.3 ...@@ -7,11 +7,11 @@ lxml==4.6.3
pygments==2.8.0 pygments==2.8.0
python-dateutil==2.8.2 python-dateutil==2.8.2
pyyaml==5.4.1 pyyaml==5.4.1
httpx[http2]==0.19.0 httpx[http2]==0.21.2
Brotli==1.0.9 Brotli==1.0.9
uvloop==0.16.0; python_version >= '3.7' uvloop==0.16.0; python_version >= '3.7'
uvloop==0.14.0; python_version < '3.7' uvloop==0.14.0; python_version < '3.7'
httpx-socks[asyncio]==0.4.1 httpx-socks[asyncio]==0.7.2
langdetect==1.0.9 langdetect==1.0.9
setproctitle==1.2.2 setproctitle==1.2.2
redis==3.4.1 redis==3.4.1
......
...@@ -126,17 +126,17 @@ def request(method, url, **kwargs): ...@@ -126,17 +126,17 @@ def request(method, url, **kwargs):
if "redis_host" not in settings["server"]: if "redis_host" not in settings["server"]:
def get(url, **kwargs): def get(url, **kwargs):
kwargs.setdefault('allow_redirects', True) kwargs.setdefault('follow_redirects', True)
return request('get', url, **kwargs) return request('get', url, **kwargs)
def options(url, **kwargs): def options(url, **kwargs):
kwargs.setdefault('allow_redirects', True) kwargs.setdefault('follow_redirects', True)
return request('options', url, **kwargs) return request('options', url, **kwargs)
def head(url, **kwargs): def head(url, **kwargs):
kwargs.setdefault('allow_redirects', False) kwargs.setdefault('follow_redirects', False)
return request('head', url, **kwargs) return request('head', url, **kwargs)
...@@ -172,7 +172,7 @@ if "redis_host" in settings["server"]: ...@@ -172,7 +172,7 @@ if "redis_host" in settings["server"]:
async def stream_chunk_to_queue(network, q, method, url, **kwargs): async def stream_chunk_to_queue(network, q, method, url, **kwargs):
try: try:
async with network.stream(method, url, **kwargs) as response: async with await network.stream(method, url, **kwargs) as response:
q.put(response) q.put(response)
async for chunk in response.aiter_bytes(65536): async for chunk in response.aiter_bytes(65536):
if len(chunk) > 0: if len(chunk) > 0:
......
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=missing-module-docstring, global-statement
import asyncio import asyncio
import logging import logging
import threading import threading
import httpcore
import httpx import httpx
from httpx_socks import AsyncProxyTransport from httpx_socks import AsyncProxyTransport
from python_socks import ( from python_socks import parse_proxy_url, ProxyConnectionError, ProxyTimeoutError, ProxyError
parse_proxy_url,
ProxyConnectionError,
ProxyTimeoutError,
ProxyError
)
import python_socks._errors
from searx import logger from searx import logger
...@@ -26,29 +21,15 @@ else: ...@@ -26,29 +21,15 @@ else:
uvloop.install() uvloop.install()
logger = logger.getChild('searx.http.client') logger = logger.getChild('searx.network.client')
LOOP = None LOOP = None
SSLCONTEXTS = {} SSLCONTEXTS = {}
TRANSPORT_KWARGS = { TRANSPORT_KWARGS = {
'backend': 'asyncio',
'trust_env': False, 'trust_env': False,
} }
async def close_connections_for_url(connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL):
origin = httpcore._utils.url_to_origin(url)
logger.debug('Drop connections for %r', origin)
connections_to_close = connection_pool._connections_for_origin(origin)
for connection in connections_to_close:
await connection_pool._remove_from_pool(connection)
try:
await connection.aclose()
except httpx.NetworkError as e:
logger.warning('Error closing an existing connection', exc_info=e)
def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False): def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
global SSLCONTEXTS
key = (proxy_url, cert, verify, trust_env, http2) key = (proxy_url, cert, verify, trust_env, http2)
if key not in SSLCONTEXTS: if key not in SSLCONTEXTS:
SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env, http2) SSLCONTEXTS[key] = httpx.create_ssl_context(cert, verify, trust_env, http2)
...@@ -58,74 +39,25 @@ def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http ...@@ -58,74 +39,25 @@ def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http
class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport): class AsyncHTTPTransportNoHttp(httpx.AsyncHTTPTransport):
"""Block HTTP request""" """Block HTTP request"""
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None): async def handle_async_request(self, request):
raise httpx.UnsupportedProtocol("HTTP protocol is disabled") raise httpx.UnsupportedProtocol('HTTP protocol is disabled')
class AsyncProxyTransportFixed(AsyncProxyTransport): class AsyncProxyTransportFixed(AsyncProxyTransport):
"""Fix httpx_socks.AsyncProxyTransport """Fix httpx_socks.AsyncProxyTransport
Map python_socks exceptions to httpx.ProxyError Map python_socks exceptions to httpx.ProxyError exceptions
Map socket.gaierror to httpx.ConnectError
Note: keepalive_expiry is ignored, AsyncProxyTransport should call:
* self._keepalive_sweep()
* self._response_closed(self, connection)
Note: AsyncProxyTransport inherit from AsyncConnectionPool
""" """
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None): async def handle_async_request(self, request):
retry = 2 try:
while retry > 0: return await super().handle_async_request(request)
retry -= 1 except ProxyConnectionError as e:
try: raise httpx.ProxyError("ProxyConnectionError: " + e.strerror, request=request) from e
return await super().handle_async_request(method, url, headers, stream, extensions) except ProxyTimeoutError as e:
except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e: raise httpx.ProxyError("ProxyTimeoutError: " + e.args[0], request=request) from e
raise httpx.ProxyError(e) except ProxyError as e:
except OSError as e: raise httpx.ProxyError("ProxyError: " + e.args[0], request=request) from e
# socket.gaierror when DNS resolution fails
raise httpx.NetworkError(e)
except httpx.RemoteProtocolError as e:
# in case of httpx.RemoteProtocolError: Server disconnected
await close_connections_for_url(self, url)
logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
# retry
except (httpx.NetworkError, httpx.ProtocolError) as e:
# httpx.WriteError on HTTP/2 connection leaves a new opened stream
# then each new request creates a new stream and raise the same WriteError
await close_connections_for_url(self, url)
raise e
class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
"""Fix httpx.AsyncHTTPTransport"""
async def handle_async_request(self, method, url, headers=None, stream=None, extensions=None):
retry = 2
while retry > 0:
retry -= 1
try:
return await super().handle_async_request(method, url, headers, stream, extensions)
except OSError as e:
# socket.gaierror when DNS resolution fails
raise httpx.ConnectError(e)
except httpx.CloseError as e:
# httpx.CloseError: [Errno 104] Connection reset by peer
# raised by _keepalive_sweep()
# from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198 # noqa
await close_connections_for_url(self._pool, url)
logger.warning('httpx.CloseError: retry', exc_info=e)
# retry
except httpx.RemoteProtocolError as e:
# in case of httpx.RemoteProtocolError: Server disconnected
await close_connections_for_url(self._pool, url)
logger.warning('httpx.RemoteProtocolError: retry', exc_info=e)
# retry
except (httpx.ProtocolError, httpx.NetworkError) as e:
await close_connections_for_url(self._pool, url)
raise e
def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries): def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit, retries):
...@@ -141,56 +73,65 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit ...@@ -141,56 +73,65 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url) proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
verify = get_sslcontexts(proxy_url, None, True, False, http2) if verify is True else verify verify = get_sslcontexts(proxy_url, None, True, False, http2) if verify is True else verify
return AsyncProxyTransportFixed(proxy_type=proxy_type, proxy_host=proxy_host, proxy_port=proxy_port, return AsyncProxyTransportFixed(
username=proxy_username, password=proxy_password, proxy_type=proxy_type,
rdns=rdns, proxy_host=proxy_host,
loop=get_loop(), proxy_port=proxy_port,
verify=verify, username=proxy_username,
http2=http2, password=proxy_password,
local_address=local_address, rdns=rdns,
max_connections=limit.max_connections, loop=get_loop(),
max_keepalive_connections=limit.max_keepalive_connections, verify=verify,
keepalive_expiry=limit.keepalive_expiry, http2=http2,
retries=retries, local_address=local_address,
**TRANSPORT_KWARGS) limits=limit,
retries=retries,
**TRANSPORT_KWARGS,
)
def get_transport(verify, http2, local_address, proxy_url, limit, retries): def get_transport(verify, http2, local_address, proxy_url, limit, retries):
verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify
return AsyncHTTPTransportFixed(verify=verify, return httpx.AsyncHTTPTransport(
http2=http2, # pylint: disable=protected-access
local_address=local_address, verify=verify,
proxy=httpx._config.Proxy(proxy_url) if proxy_url else None, http2=http2,
limits=limit, limits=limit,
retries=retries, proxy=httpx._config.Proxy(proxy_url) if proxy_url else None,
**TRANSPORT_KWARGS) local_address=local_address,
retries=retries,
**TRANSPORT_KWARGS,
def iter_proxies(proxies): )
# https://www.python-httpx.org/compatibility/#proxy-keys
if isinstance(proxies, str):
yield 'all://', proxies def new_client(
elif isinstance(proxies, dict): # pylint: disable=too-many-arguments
for pattern, proxy_url in proxies.items(): enable_http,
yield pattern, proxy_url verify,
enable_http2,
max_connections,
def new_client(enable_http, verify, enable_http2, max_keepalive_connections,
max_connections, max_keepalive_connections, keepalive_expiry, keepalive_expiry,
proxies, local_address, retries, max_redirects): proxies,
limit = httpx.Limits(max_connections=max_connections, local_address,
max_keepalive_connections=max_keepalive_connections, retries,
keepalive_expiry=keepalive_expiry) max_redirects,
hook_log_response,
):
limit = httpx.Limits(
max_connections=max_connections,
max_keepalive_connections=max_keepalive_connections,
keepalive_expiry=keepalive_expiry,
)
# See https://www.python-httpx.org/advanced/#routing # See https://www.python-httpx.org/advanced/#routing
mounts = {} mounts = {}
for pattern, proxy_url in iter_proxies(proxies): for pattern, proxy_url in proxies.items():
if not enable_http and (pattern == 'http' or pattern.startswith('http://')): if not enable_http and pattern.startswith('http://'):
continue continue
if proxy_url.startswith('socks4://') \ if proxy_url.startswith('socks4://') or proxy_url.startswith('socks5://') or proxy_url.startswith('socks5h://'):
or proxy_url.startswith('socks5://') \ mounts[pattern] = get_transport_for_socks_proxy(
or proxy_url.startswith('socks5h://'): verify, enable_http2, local_address, proxy_url, limit, retries
mounts[pattern] = get_transport_for_socks_proxy(verify, enable_http2, local_address, proxy_url, limit, )
retries)
else: else:
mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries) mounts[pattern] = get_transport(verify, enable_http2, local_address, proxy_url, limit, retries)
...@@ -198,17 +139,26 @@ def new_client(enable_http, verify, enable_http2, ...@@ -198,17 +139,26 @@ def new_client(enable_http, verify, enable_http2,
mounts['http://'] = AsyncHTTPTransportNoHttp() mounts['http://'] = AsyncHTTPTransportNoHttp()
transport = get_transport(verify, enable_http2, local_address, None, limit, retries) transport = get_transport(verify, enable_http2, local_address, None, limit, retries)
return httpx.AsyncClient(transport=transport, mounts=mounts, max_redirects=max_redirects)
event_hooks = None
if hook_log_response:
event_hooks = {'response': [hook_log_response]}
return httpx.AsyncClient(
transport=transport,
mounts=mounts,
max_redirects=max_redirects,
event_hooks=event_hooks,
)
def get_loop(): def get_loop():
global LOOP
return LOOP return LOOP
def init(): def init():
# log # log
for logger_name in ('hpack.hpack', 'hpack.table'): for logger_name in ('hpack.hpack', 'hpack.table', 'httpx._client'):
logging.getLogger(logger_name).setLevel(logging.WARNING) logging.getLogger(logger_name).setLevel(logging.WARNING)
# loop # loop
...@@ -217,12 +167,12 @@ def init(): ...@@ -217,12 +167,12 @@ def init():
LOOP = asyncio.new_event_loop() LOOP = asyncio.new_event_loop()
LOOP.run_forever() LOOP.run_forever()
th = threading.Thread( thread = threading.Thread(
target=loop_thread, target=loop_thread,
name='asyncio_loop', name='asyncio_loop',
daemon=True, daemon=True,
) )
th.start() thread.start()
init() init()
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=global-statement
# pylint: disable=missing-module-docstring, missing-class-docstring
import atexit import atexit
import asyncio import asyncio
...@@ -7,9 +10,11 @@ from itertools import cycle ...@@ -7,9 +10,11 @@ from itertools import cycle
import httpx import httpx
from .client import new_client, get_loop from searx import logger, searx_debug
from .client import new_client, get_loop, AsyncHTTPTransportNoHttp
logger = logger.getChild('network')
DEFAULT_NAME = '__DEFAULT__' DEFAULT_NAME = '__DEFAULT__'
NETWORKS = {} NETWORKS = {}
# requests compatibility when reading proxy settings from settings.yml # requests compatibility when reading proxy settings from settings.yml
...@@ -26,31 +31,50 @@ PROXY_PATTERN_MAPPING = { ...@@ -26,31 +31,50 @@ PROXY_PATTERN_MAPPING = {
'socks5h:': 'socks5h://', 'socks5h:': 'socks5h://',
} }
ADDRESS_MAPPING = { ADDRESS_MAPPING = {'ipv4': '0.0.0.0', 'ipv6': '::'}
'ipv4': '0.0.0.0',
'ipv6': '::'
}
class Network: class Network:
__slots__ = ('enable_http', 'verify', 'enable_http2', __slots__ = (
'max_connections', 'max_keepalive_connections', 'keepalive_expiry', 'enable_http',
'local_addresses', 'proxies', 'max_redirects', 'retries', 'retry_on_http_error', 'verify',
'_local_addresses_cycle', '_proxies_cycle', '_clients') 'enable_http2',
'max_connections',
def __init__(self, 'max_keepalive_connections',
enable_http=True, 'keepalive_expiry',
verify=True, 'local_addresses',
enable_http2=False, 'proxies',
max_connections=None, 'using_tor_proxy',
max_keepalive_connections=None, 'max_redirects',
keepalive_expiry=None, 'retries',
proxies=None, 'retry_on_http_error',
local_addresses=None, '_local_addresses_cycle',
retries=0, '_proxies_cycle',
retry_on_http_error=None, '_clients',
max_redirects=30): '_logger',
)
_TOR_CHECK_RESULT = {}
def __init__(
# pylint: disable=too-many-arguments
self,
enable_http=True,
verify=True,
enable_http2=False,
max_connections=None,
max_keepalive_connections=None,
keepalive_expiry=None,
proxies=None,
using_tor_proxy=False,
local_addresses=None,
retries=0,
retry_on_http_error=None,
max_redirects=30,
logger_name=None,
):
self.enable_http = enable_http self.enable_http = enable_http
self.verify = verify self.verify = verify
self.enable_http2 = enable_http2 self.enable_http2 = enable_http2
...@@ -58,6 +82,7 @@ class Network: ...@@ -58,6 +82,7 @@ class Network:
self.max_keepalive_connections = max_keepalive_connections self.max_keepalive_connections = max_keepalive_connections
self.keepalive_expiry = keepalive_expiry self.keepalive_expiry = keepalive_expiry
self.proxies = proxies self.proxies = proxies
self.using_tor_proxy = using_tor_proxy
self.local_addresses = local_addresses self.local_addresses = local_addresses
self.retries = retries self.retries = retries
self.retry_on_http_error = retry_on_http_error self.retry_on_http_error = retry_on_http_error
...@@ -65,6 +90,7 @@ class Network: ...@@ -65,6 +90,7 @@ class Network:
self._local_addresses_cycle = self.get_ipaddress_cycle() self._local_addresses_cycle = self.get_ipaddress_cycle()
self._proxies_cycle = self.get_proxy_cycles() self._proxies_cycle = self.get_proxy_cycles()
self._clients = {} self._clients = {}
self._logger = logger.getChild(logger_name) if logger_name else logger
self.check_parameters() self.check_parameters()
def check_parameters(self): def check_parameters(self):
...@@ -81,7 +107,7 @@ class Network: ...@@ -81,7 +107,7 @@ class Network:
local_addresses = self.local_addresses local_addresses = self.local_addresses
if not local_addresses: if not local_addresses:
return return
elif isinstance(local_addresses, str): if isinstance(local_addresses, str):
local_addresses = [local_addresses] local_addresses = [local_addresses]
for address in local_addresses: for address in local_addresses:
yield address yield address
...@@ -119,25 +145,62 @@ class Network: ...@@ -119,25 +145,62 @@ class Network:
for pattern, proxy_urls in self.iter_proxies(): for pattern, proxy_urls in self.iter_proxies():
proxy_settings[pattern] = cycle(proxy_urls) proxy_settings[pattern] = cycle(proxy_urls)
while True: while True:
# pylint: disable=stop-iteration-return
yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items()) yield tuple((pattern, next(proxy_url_cycle)) for pattern, proxy_url_cycle in proxy_settings.items())
def get_client(self, verify=None, max_redirects=None): async def log_response(self, response: httpx.Response):
request = response.request
status = f"{response.status_code} {response.reason_phrase}"
response_line = f"{response.http_version} {status}"
content_type = response.headers.get("Content-Type")
content_type = f' ({content_type})' if content_type else ''
self._logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"{content_type}')
@staticmethod
async def check_tor_proxy(client: httpx.AsyncClient, proxies) -> bool:
if proxies in Network._TOR_CHECK_RESULT:
return Network._TOR_CHECK_RESULT[proxies]
result = True
# ignore client._transport because it is not used with all://
for transport in client._mounts.values(): # pylint: disable=protected-access
if isinstance(transport, AsyncHTTPTransportNoHttp):
continue
if not getattr(transport, '_rdns', False):
result = False
break
else:
response = await client.get('https://check.torproject.org/api/ip')
if not response.json()['IsTor']:
result = False
Network._TOR_CHECK_RESULT[proxies] = result
return result
async def get_client(self, verify=None, max_redirects=None):
verify = self.verify if verify is None else verify verify = self.verify if verify is None else verify
max_redirects = self.max_redirects if max_redirects is None else max_redirects max_redirects = self.max_redirects if max_redirects is None else max_redirects
local_address = next(self._local_addresses_cycle) local_address = next(self._local_addresses_cycle)
proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key proxies = next(self._proxies_cycle) # is a tuple so it can be part of the key
key = (verify, max_redirects, local_address, proxies) key = (verify, max_redirects, local_address, proxies)
hook_log_response = None
if key not in self._clients or self._clients[key].is_closed: if key not in self._clients or self._clients[key].is_closed:
self._clients[key] = new_client(self.enable_http, client = new_client(
verify, self.enable_http,
self.enable_http2, verify,
self.max_connections, self.enable_http2,
self.max_keepalive_connections, self.max_connections,