Merge pull request #192 from dalf/connection-pool (10891bde) · Commits · e / infra / spot

searx/autocomplete.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.


		from lxml import etree
		from requests import get
		from json import loads
		from urllib import urlencode
		from searx.languages import language_codes
		from searx.engines import (
		categories, engines, engine_shortcuts
		)
		from searx.poolrequests import get


		def searx_bang(full_query):

searx/engines/wikidata.py

+1 −1

Original line number	Diff line number	Diff line
		import json
		from requests import get
		from urllib import urlencode
		from searx.poolrequests import get
		from searx.utils import format_date_by_locale

		result_count = 1

searx/poolrequests.py

0 → 100644

+61 −0

Original line number	Diff line number	Diff line
		import requests


		the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
		the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)


		class SessionSinglePool(requests.Session):

		def __init__(self):
		global the_https_adapter, the_http_adapter
		super(SessionSinglePool, self).__init__()

		# reuse the same adapters
		self.adapters.clear()
		self.mount('https://', the_https_adapter)
		self.mount('http://', the_http_adapter)

		def close(self):
		"""Call super, but clear adapters since there are managed globaly"""
		self.adapters.clear()
		super(SessionSinglePool, self).close()


		def request(method, url, **kwargs):
		"""same as requests/requests/api.py request(...) except it use SessionSinglePool"""
		session = SessionSinglePool()
		response = session.request(method=method, url=url, **kwargs)
		session.close()
		return response


		def get(url, **kwargs):
		kwargs.setdefault('allow_redirects', True)
		return request('get', url, **kwargs)


		def options(url, **kwargs):
		kwargs.setdefault('allow_redirects', True)
		return request('options', url, **kwargs)


		def head(url, **kwargs):
		kwargs.setdefault('allow_redirects', False)
		return request('head', url, **kwargs)


		def post(url, data=None, json=None, **kwargs):
		return request('post', url, data=data, json=json, **kwargs)


		def put(url, data=None, **kwargs):
		return request('put', url, data=data, **kwargs)


		def patch(url, data=None, **kwargs):
		return request('patch', url, data=data, **kwargs)


		def delete(url, **kwargs):
		return request('delete', url, **kwargs)

searx/search.py

+1 −2

Original line number	Diff line number	Diff line
		@@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
		(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
		'''

		import requests as requests_lib
		import threading
		import re
		import searx.poolrequests as requests_lib
		from itertools import izip_longest, chain
		from operator import itemgetter
		from Queue import Queue
		@@ -31,7 +31,6 @@ from searx.utils import gen_useragent
		from searx.query import Query
		from searx import logger


		logger = logger.getChild('search')

		number_of_searches = 0

searx/webapp.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -28,7 +28,6 @@ import os
		import hashlib

		from datetime import datetime, timedelta
		from requests import get as http_get
		from itertools import chain
		from urllib import urlencode
		from flask import (
		@@ -37,6 +36,7 @@ from flask import (
		)
		from flask.ext.babel import Babel, gettext, format_date
		from searx import settings, searx_dir
		from searx.poolrequests import get as http_get
		from searx.engines import (
		categories, engines, get_engines_stats, engine_shortcuts
		)