Commit d959cb1c authored by Adam Tauber's avatar Adam Tauber

[enh] gevent/grequests changed to the built-in threading lib

parent b1b0b82a
......@@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
import grequests
import requests as requests_lib
import threading
import re
from itertools import izip_longest, chain
from datetime import datetime
......@@ -32,6 +33,18 @@ from searx.query import Query
number_of_searches = 0
def threaded_requests(requests):
for fn, url, request_args in requests:
th = threading.Thread(
target=fn, args=(url,), kwargs=request_args, name=url,
)
th.start()
for th in threading.enumerate():
if th.name.startswith('http'):
th.join()
# get default reqest parameter
def default_request_params():
return {
......@@ -471,9 +484,9 @@ class Search(object):
# specific type of request (GET or POST)
if request_params['method'] == 'GET':
req = grequests.get
req = requests_lib.get
else:
req = grequests.post
req = requests_lib.post
request_args['data'] = request_params['data']
# ignoring empty urls
......@@ -481,10 +494,10 @@ class Search(object):
continue
# append request to list
requests.append(req(request_params['url'], **request_args))
requests.append((req, request_params['url'], request_args))
# send all search-request
grequests.map(requests)
threaded_requests(requests)
# update engine-specific stats
for engine_name, engine_results in results.items():
......
......@@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''
from gevent import monkey
monkey.patch_all()
if __name__ == '__main__':
from sys import path
from os.path import realpath, dirname
......
......@@ -41,7 +41,7 @@ setup(
install_requires=[
'flask',
'flask-babel',
'grequests',
'requests',
'lxml',
'pyyaml',
'setuptools',
......
......@@ -36,18 +36,6 @@ zc.recipe.testrunner = 2.0.0
# WebTest==2.0.11
beautifulsoup4 = 4.3.2
# Required by:
# grequests==0.2.0
gevent = 1.0
# Required by:
# gevent==1.0
greenlet = 0.4.2
# Required by:
# searx==0.1
grequests = 0.2.0
# Required by:
# robotframework-httplibrary==0.4.2
jsonpatch = 1.3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment