Commit 52e615de authored by Adam Tauber's avatar Adam Tauber
Browse files

[enh] py3 compatibility

parent 46a2c63f
......@@ -9,6 +9,7 @@ addons:
language: python
python:
- "2.7"
- "3.6"
before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
......@@ -24,9 +25,9 @@ script:
- ./manage.sh styles
- ./manage.sh grunt_build
- ./manage.sh tests
- ./manage.sh py_test_coverage
after_success:
coveralls
- ./manage.sh py_test_coverage
- coveralls
notifications:
irc:
channels:
......
......@@ -3,8 +3,7 @@ mock==2.0.0
nose2[coverage-plugin]
pep8==1.7.0
plone.testing==5.0.0
robotframework-selenium2library==1.8.0
robotsuite==1.7.0
splinter==0.7.5
transifex-client==0.12.2
unittest2==1.1.0
zope.testrunner==4.5.1
......
from os import listdir
from os.path import realpath, dirname, join, isdir
from sys import version_info
from searx.utils import load_module
from collections import defaultdict
if version_info[0] == 3:
unicode = str
answerers_dir = dirname(realpath(__file__))
......@@ -10,7 +14,7 @@ answerers_dir = dirname(realpath(__file__))
def load_answerers():
answerers = []
for filename in listdir(answerers_dir):
if not isdir(join(answerers_dir, filename)):
if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):
continue
module = load_module('answerer.py', join(answerers_dir, filename))
if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):
......@@ -30,12 +34,12 @@ def get_answerers_by_keywords(answerers):
def ask(query):
results = []
query_parts = filter(None, query.query.split())
query_parts = list(filter(None, query.query.split()))
if query_parts[0] not in answerers_by_keywords:
if query_parts[0].decode('utf-8') not in answerers_by_keywords:
return results
for answerer in answerers_by_keywords[query_parts[0]]:
for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:
result = answerer(query)
if result:
results.append(result)
......
import random
import string
import sys
from flask_babel import gettext
# required answerer attribute
......@@ -8,7 +9,11 @@ keywords = ('random',)
random_int_max = 2**31
random_string_letters = string.lowercase + string.digits + string.uppercase
if sys.version_info[0] == 2:
random_string_letters = string.lowercase + string.digits + string.uppercase
else:
unicode = str
random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase
def random_string():
......@@ -24,9 +29,9 @@ def random_int():
return unicode(random.randint(-random_int_max, random_int_max))
random_types = {u'string': random_string,
u'int': random_int,
u'float': random_float}
random_types = {b'string': random_string,
b'int': random_int,
b'float': random_float}
# required answerer function
......
from sys import version_info
from functools import reduce
from operator import mul
from flask_babel import gettext
if version_info[0] == 3:
unicode = str
keywords = ('min',
'max',
'avg',
......@@ -19,22 +23,22 @@ def answer(query):
return []
try:
args = map(float, parts[1:])
args = list(map(float, parts[1:]))
except:
return []
func = parts[0]
answer = None
if func == 'min':
if func == b'min':
answer = min(args)
elif func == 'max':
elif func == b'max':
answer = max(args)
elif func == 'avg':
elif func == b'avg':
answer = sum(args) / len(args)
elif func == 'sum':
elif func == b'sum':
answer = sum(args)
elif func == 'prod':
elif func == b'prod':
answer = reduce(mul, args, 1)
if answer is None:
......
......@@ -18,7 +18,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
from lxml import etree
from json import loads
from urllib import urlencode
from searx import settings
from searx.languages import language_codes
from searx.engines import (
......@@ -26,6 +25,11 @@ from searx.engines import (
)
from searx.poolrequests import get as http_get
try:
from urllib import urlencode
except:
from urllib.parse import urlencode
def get(*args, **kwargs):
if 'timeout' not in kwargs:
......
from urllib import quote
from lxml import html
from searx.engines.xpath import extract_text
from searx.utils import get_torrent_size
from urlparse import urljoin
from searx.url_utils import quote, urljoin
url = 'https://1337x.to/'
search_url = url + 'search/{search_term}/{pageno}/'
......
......@@ -72,12 +72,11 @@ def load_engine(engine_data):
if engine_data['categories'] == 'none':
engine.categories = []
else:
engine.categories = map(
str.strip, engine_data['categories'].split(','))
engine.categories = list(map(str.strip, engine_data['categories'].split(',')))
continue
setattr(engine, param_name, engine_data[param_name])
for arg_name, arg_value in engine_default_args.iteritems():
for arg_name, arg_value in engine_default_args.items():
if not hasattr(engine, arg_name):
setattr(engine, arg_name, arg_value)
......
......@@ -11,10 +11,9 @@
@parse url, title
"""
from urlparse import urljoin
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
from searx.url_utils import urlencode, urljoin
# engine dependent config
categories = ['it']
......
......@@ -14,10 +14,10 @@
"""
from lxml import etree
from urllib import urlencode
from searx.utils import searx_useragent
from datetime import datetime
import re
from searx.url_utils import urlencode
from searx.utils import searx_useragent
categories = ['science']
......@@ -73,7 +73,7 @@ def request(query, params):
def response(resp):
results = []
search_results = etree.XML(resp.content)
search_results = etree.XML(resp.text)
for entry in search_results.xpath('./result/doc'):
content = "No description available"
......
......@@ -13,9 +13,9 @@
@todo publishedDate
"""
from urllib import urlencode
from lxml import html
from searx.engines.xpath import extract_text
from searx.url_utils import urlencode
# engine dependent config
categories = ['general']
......
......@@ -15,11 +15,11 @@
limited response to 10 images
"""
from urllib import urlencode
from lxml import html
from json import loads
import re
from searx.engines.bing import _fetch_supported_languages, supported_languages_url
from searx.url_utils import urlencode
# engine dependent config
categories = ['images']
......
......@@ -11,13 +11,12 @@
@parse url, title, content, publishedDate, thumbnail
"""
from urllib import urlencode
from urlparse import urlparse, parse_qsl
from datetime import datetime
from dateutil import parser
from lxml import etree
from searx.utils import list_get
from searx.engines.bing import _fetch_supported_languages, supported_languages_url
from searx.url_utils import urlencode, urlparse, parse_qsl
# engine dependent config
categories = ['news']
......@@ -86,7 +85,7 @@ def request(query, params):
def response(resp):
results = []
rss = etree.fromstring(resp.content)
rss = etree.fromstring(resp.text)
ns = rss.nsmap
......
......@@ -11,7 +11,7 @@
"""
from json import loads
from urllib import urlencode
from searx.url_utils import urlencode
# engine dependent config
categories = ['images']
......
......@@ -10,11 +10,10 @@
@parse url, title, content, seed, leech, magnetlink
"""
from urlparse import urljoin
from urllib import quote
from lxml import html
from operator import itemgetter
from searx.engines.xpath import extract_text
from searx.url_utils import quote, urljoin
from searx.utils import get_torrent_size
# engine dependent config
......@@ -38,7 +37,7 @@ def request(query, params):
def response(resp):
results = []
dom = html.fromstring(resp.content)
dom = html.fromstring(resp.text)
search_res = dom.xpath('//div[@id="search_res"]/table/tr')
......
from datetime import datetime
import json
import re
import os
import json
import sys
import unicodedata
from datetime import datetime
if sys.version_info[0] == 3:
unicode = str
categories = []
url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'
weight = 100
parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) # noqa
parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)
db = 1
def normalize_name(name):
name = name.lower().replace('-', ' ').rstrip('s')
name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')
name = re.sub(' +', ' ', name)
return unicodedata.normalize('NFKD', name).lower()
......@@ -35,7 +39,7 @@ def iso4217_to_name(iso4217, language):
def request(query, params):
m = parser_re.match(unicode(query, 'utf8'))
m = parser_re.match(query)
if not m:
# wrong query
return params
......
......@@ -12,10 +12,9 @@
@todo set content-parameter with correct data
"""
from urllib import urlencode
from json import loads
from datetime import datetime
from requests import get
from searx.url_utils import urlencode
# engine dependent config
categories = ['videos']
......
......@@ -11,7 +11,7 @@
"""
from json import loads
from urllib import urlencode
from searx.url_utils import urlencode
# engine dependent config
categories = ['music']
......@@ -30,8 +30,7 @@ embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true"
def request(query, params):
offset = (params['pageno'] - 1) * 25
params['url'] = search_url.format(query=urlencode({'q': query}),
offset=offset)
params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
return params
......
......@@ -12,10 +12,10 @@
@todo rewrite to api
"""
from urllib import urlencode
from lxml import html
import re
from searx.engines.xpath import extract_text
from searx.url_utils import urlencode
# engine dependent config
categories = ['images']
......
......@@ -10,20 +10,20 @@
"""
import re
from urlparse import urljoin
from lxml import html
from searx.utils import is_valid_lang
from searx.url_utils import urljoin
categories = ['general']
url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'
weight = 100
parser_re = re.compile(u'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)
results_xpath = './/table[@id="r"]/tr'
def request(query, params):
m = parser_re.match(unicode(query, 'utf8'))
m = parser_re.match(query)
if not m:
return params
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment