diff options
| author | Adam Tauber <asciimoo@gmail.com> | 2014-12-05 19:24:11 +0100 |
|---|---|---|
| committer | Adam Tauber <asciimoo@gmail.com> | 2014-12-05 19:24:11 +0100 |
| commit | d959cb1c059008984554c129cb6e17b6c5394bfc (patch) | |
| tree | 34f7d8e780189ffa3afae60ec60e1f8bdddb2898 | |
| parent | b1b0b82a2a1d6aaa46799ad65cf75849895b8f07 (diff) | |
[enh] gevent/grequests changed to the built-in threading lib
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | searx/search.py | 23 | ||||
| -rw-r--r-- | searx/webapp.py | 4 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | versions.cfg | 12 |
5 files changed, 20 insertions, 23 deletions
diff --git a/requirements.txt b/requirements.txt index 88c1bc71..07b53d2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flask flask-babel -grequests +requests lxml pyyaml python-dateutil diff --git a/searx/search.py b/searx/search.py index 0e7aaed8..19469cc5 100644 --- a/searx/search.py +++ b/searx/search.py @@ -15,7 +15,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -import grequests +import requests as requests_lib +import threading import re from itertools import izip_longest, chain from datetime import datetime @@ -32,6 +33,18 @@ from searx.query import Query number_of_searches = 0 +def threaded_requests(requests): + for fn, url, request_args in requests: + th = threading.Thread( + target=fn, args=(url,), kwargs=request_args, name=url, + ) + th.start() + + for th in threading.enumerate(): + if th.name.startswith('http'): + th.join() + + # get default reqest parameter def default_request_params(): return { @@ -471,9 +484,9 @@ class Search(object): # specific type of request (GET or POST) if request_params['method'] == 'GET': - req = grequests.get + req = requests_lib.get else: - req = grequests.post + req = requests_lib.post request_args['data'] = request_params['data'] # ignoring empty urls @@ -481,10 +494,10 @@ class Search(object): continue # append request to list - requests.append(req(request_params['url'], **request_args)) + requests.append((req, request_params['url'], request_args)) # send all search-request - grequests.map(requests) + threaded_requests(requests) # update engine-specific stats for engine_name, engine_results in results.items(): diff --git a/searx/webapp.py b/searx/webapp.py index 46061913..57a16ff4 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -17,10 +17,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <asciimoo@gmail.com> ''' -from gevent import monkey -monkey.patch_all() - - if __name__ == '__main__': from sys import path from os.path import realpath, dirname @@ -41,7 +41,7 @@ setup( install_requires=[ 'flask', 'flask-babel', - 'grequests', + 'requests', 'lxml', 'pyyaml', 'setuptools', diff --git a/versions.cfg b/versions.cfg index e0d7f451..2f5dae8e 100644 --- a/versions.cfg +++ b/versions.cfg @@ -37,18 +37,6 @@ zc.recipe.testrunner = 2.0.0 beautifulsoup4 = 4.3.2 # Required by: -# grequests==0.2.0 -gevent = 1.0 - -# Required by: -# gevent==1.0 -greenlet = 0.4.2 - -# Required by: -# searx==0.1 -grequests = 0.2.0 - -# Required by: # robotframework-httplibrary==0.4.2 jsonpatch = 1.3 |