webapp.py 35 KB
Newer Older
asciimoo's avatar
asciimoo committed
1 2
#!/usr/bin/env python

asciimoo's avatar
asciimoo committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.

(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''

asciimoo's avatar
asciimoo committed
20 21 22
if __name__ == '__main__':
    from sys import path
    from os.path import realpath, dirname
23
    path.append(realpath(dirname(realpath(__file__)) + '/../'))
asciimoo's avatar
asciimoo committed
24

25
import hashlib
26 27 28
import hmac
import json
import os
Adam Tauber's avatar
Adam Tauber committed
29 30
import sys

31
import requests
asciimoo's avatar
asciimoo committed
32

33 34 35 36 37 38 39 40 41 42 43
from searx import logger
logger = logger.getChild('webapp')

try:
    from pygments import highlight
    from pygments.lexers import get_lexer_by_name
    from pygments.formatters import HtmlFormatter
except:
    logger.critical("cannot import dependency: pygments")
    from sys import exit
    exit(1)
44 45 46 47
try:
    from cgi import escape
except:
    from html import escape
48
from datetime import datetime, timedelta
49
from time import time
50
from werkzeug.middleware.proxy_fix import ProxyFix
Gabor Nagy's avatar
Gabor Nagy committed
51 52 53 54
from flask import (
    Flask, request, render_template, url_for, Response, make_response,
    redirect, send_from_directory
)
55
from flask_babel import Babel, gettext, format_date, format_decimal
56
from flask.json import jsonify
57
from searx import settings, searx_dir, searx_debug
Adam Tauber's avatar
Adam Tauber committed
58
from searx.exceptions import SearxParameterException
Gabor Nagy's avatar
Gabor Nagy committed
59
from searx.engines import (
Adam Tauber's avatar
Adam Tauber committed
60
    categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
Gabor Nagy's avatar
Gabor Nagy committed
61
)
Matej Cotman's avatar
Matej Cotman committed
62
from searx.utils import (
63 64
    UnicodeWriter, highlight_content, html_to_text, get_resources_directory,
    get_static_files, get_result_templates, get_themes, gen_useragent,
65
    dict_subset, prettify_url, match_language
Matej Cotman's avatar
Matej Cotman committed
66
)
67
from searx.version import VERSION_STRING
68
from searx.languages import language_codes as languages
Adam Tauber's avatar
Adam Tauber committed
69 70
from searx.search import SearchWithPlugins, get_search_query_from_webapp
from searx.query import RawTextQuery
71
from searx.autocomplete import searx_bang, backends as autocomplete_backends
72
from searx.plugins import plugins
Noémi Ványi's avatar
Noémi Ványi committed
73
from searx.plugins.oa_doi_rewrite import get_doi_resolver
74
from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
75
from searx.answerers import answerers
Adam Tauber's avatar
Adam Tauber committed
76
from searx.url_utils import urlencode, urlparse, urljoin
77
from searx.utils import new_hmac
asciimoo's avatar
asciimoo committed
78

79 80
# check if the pyopenssl package is installed.
# It is needed for SSL connection without trouble, see #298
81 82 83
try:
    import OpenSSL.SSL  # NOQA
except ImportError:
84
    logger.critical("The pyopenssl package has to be installed.\n"
85
                    "Some HTTPS connections will fail")
86

Adam Tauber's avatar
Adam Tauber committed
87 88 89 90 91 92 93 94
try:
    from cStringIO import StringIO
except:
    from io import StringIO


if sys.version_info[0] == 3:
    unicode = str
95 96 97
    PY3 = True
else:
    PY3 = False
98 99
    logger.warning('\033[1;31m *** Deprecation Warning ***\033[0m')
    logger.warning('\033[1;31m Python2 is deprecated\033[0m')
Adam Tauber's avatar
Adam Tauber committed
100

Eig8phei's avatar
Eig8phei committed
101 102
# serve pages with HTTP/1.1
from werkzeug.serving import WSGIRequestHandler
103
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
asciimoo's avatar
asciimoo committed
104

105 106 107 108
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
109

110
# about templates
111
default_theme = settings['ui']['default_theme']
112 113
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
114
themes = get_themes(templates_path)
115 116 117 118 119 120 121
result_templates = get_result_templates(templates_path)
global_favicons = []
for indice, theme in enumerate(themes):
    global_favicons.append([])
    theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
    for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
        global_favicons[indice].extend(filenames)
Matej Cotman's avatar
Matej Cotman committed
122

123
# Flask app
124 125
app = Flask(
    __name__,
Matej Cotman's avatar
Matej Cotman committed
126 127
    static_folder=static_path,
    template_folder=templates_path
128 129
)

130 131
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
132
app.jinja_env.add_extension('jinja2.ext.loopcontrols')
133
app.secret_key = settings['server']['secret_key']
134

135 136 137
if not searx_debug \
   or os.environ.get("WERKZEUG_RUN_MAIN") == "true" \
   or os.environ.get('UWSGI_ORIGINAL_PROC_NAME') is not None:
138 139
    initialize_engines(settings['engines'])

140 141
babel = Babel(app)

142
rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
143
               'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
144

145
# used when translating category names
146 147 148 149 150 151 152 153
_category_names = (gettext('files'),
                   gettext('general'),
                   gettext('music'),
                   gettext('social media'),
                   gettext('images'),
                   gettext('videos'),
                   gettext('it'),
                   gettext('news'),
154
                   gettext('map'),
Thomas Pointhuber's avatar
Thomas Pointhuber committed
155
                   gettext('science'))
156

157
outgoing_proxies = settings['outgoing'].get('proxies') or None
asciimoo's avatar
asciimoo committed
158 159


160 161
@babel.localeselector
def get_locale():
162
    locale = "en-US"
asciimoo's avatar
asciimoo committed
163

164 165 166 167
    for lang in request.headers.get("Accept-Language", locale).split(","):
        locale = match_language(lang, settings['locales'].keys(), fallback=None)
        if locale is not None:
            break
asciimoo's avatar
asciimoo committed
168

Dalf's avatar
Dalf committed
169
    if request.preferences.get_value('locale') != '':
170
        locale = request.preferences.get_value('locale')
asciimoo's avatar
asciimoo committed
171

172 173 174
    logger.debug("selected locale is `%s`", locale)

    return locale
175 176


177 178 179 180 181 182
# code-highlighter
@app.template_filter('code_highlighter')
def code_highlighter(codelines, language=None):
    if not language:
        language = 'text'

183 184 185 186 187
    try:
        # find lexer by programing language
        lexer = get_lexer_by_name(language, stripall=True)
    except:
        # if lexer is not found, using default one
Adam Tauber's avatar
Adam Tauber committed
188
        logger.debug('highlighter cannot find lexer for {0}'.format(language))
189 190
        lexer = get_lexer_by_name('text', stripall=True)

191 192 193 194 195 196 197 198 199 200
    html_code = ''
    tmp_code = ''
    last_line = None

    # parse lines
    for line, code in codelines:
        if not last_line:
            line_code_start = line

        # new codeblock is detected
201 202
        if last_line is not None and\
           last_line + 1 != line:
203 204

            # highlight last codepart
205 206
            formatter = HtmlFormatter(linenos='inline',
                                      linenostart=line_code_start)
207
            html_code = html_code + highlight(tmp_code, lexer, formatter)
208

209 210 211 212 213 214
            # reset conditions for next codepart
            tmp_code = ''
            line_code_start = line

        # add codepart
        tmp_code += code + '\n'
215

216 217 218 219 220 221 222 223 224 225
        # update line
        last_line = line

    # highlight last codepart
    formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start)
    html_code = html_code + highlight(tmp_code, lexer, formatter)

    return html_code


Cqoicebordel's avatar
Cqoicebordel committed
226 227 228 229 230 231
# Extract domain from url
@app.template_filter('extract_domain')
def extract_domain(url):
    return urlparse(url)[1]


232
def get_base_url():
233 234
    if settings['server']['base_url']:
        hostname = settings['server']['base_url']
235 236 237 238 239 240 241 242
    else:
        scheme = 'http'
        if request.is_secure:
            scheme = 'https'
        hostname = url_for('index', _external=True, _scheme=scheme)
    return hostname


Matej Cotman's avatar
Matej Cotman committed
243 244 245 246 247 248 249 250
def get_current_theme_name(override=None):
    """Returns theme name.

    Checks in this order:
    1. override
    2. cookies
    3. settings"""

251
    if override and (override in themes or override == '__common__'):
Matej Cotman's avatar
Matej Cotman committed
252
        return override
Noemi Vanyi's avatar
Noemi Vanyi committed
253
    theme_name = request.args.get('theme', request.preferences.get_value('theme'))
Matej Cotman's avatar
Matej Cotman committed
254 255 256 257 258
    if theme_name not in themes:
        theme_name = default_theme
    return theme_name


259 260 261 262 263 264 265
def get_result_template(theme, template_name):
    themed_path = theme + '/result_templates/' + template_name
    if themed_path in result_templates:
        return themed_path
    return 'result_templates/' + template_name


Matej Cotman's avatar
Matej Cotman committed
266
def url_for_theme(endpoint, override_theme=None, **values):
267
    if endpoint == 'static' and values.get('filename'):
Matej Cotman's avatar
Matej Cotman committed
268
        theme_name = get_current_theme_name(override=override_theme)
269 270 271
        filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
        if filename_with_theme in static_files:
            values['filename'] = filename_with_theme
Matej Cotman's avatar
Matej Cotman committed
272 273 274
    return url_for(endpoint, **values)


275 276 277 278 279 280 281
def proxify(url):
    if url.startswith('//'):
        url = 'https:' + url

    if not settings.get('result_proxy'):
        return url

282 283 284 285 286 287
    url_params = dict(mortyurl=url.encode('utf-8'))

    if settings['result_proxy'].get('key'):
        url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
                                           url.encode('utf-8'),
                                           hashlib.sha256).hexdigest()
288 289

    return '{0}?{1}'.format(settings['result_proxy']['url'],
290
                            urlencode(url_params))
291 292


Adam Tauber's avatar
Adam Tauber committed
293 294 295 296 297
def image_proxify(url):

    if url.startswith('//'):
        url = 'https:' + url

298
    if not request.preferences.get_value('image_proxy'):
Adam Tauber's avatar
Adam Tauber committed
299 300
        return url

Venca24's avatar
Venca24 committed
301 302 303
    if url.startswith('data:image/jpeg;base64,'):
        return url

304 305 306
    if settings.get('result_proxy'):
        return proxify(url)

307
    h = new_hmac(settings['server']['secret_key'], url.encode('utf-8'))
dalf's avatar
dalf committed
308

Adam Tauber's avatar
Adam Tauber committed
309
    return '{0}?{1}'.format(url_for('image_proxy'),
Adam Tauber's avatar
Adam Tauber committed
310
                            urlencode(dict(url=url.encode('utf-8'), h=h)))
Adam Tauber's avatar
Adam Tauber committed
311 312


Matej Cotman's avatar
Matej Cotman committed
313
def render(template_name, override_theme=None, **kwargs):
314
    disabled_engines = request.preferences.engines.get_disabled()
315

316 317 318
    enabled_categories = set(category for engine_name in engines
                             for category in engines[engine_name].categories
                             if (engine_name, category) not in disabled_engines)
319

Adam Tauber's avatar
Adam Tauber committed
320
    if 'categories' not in kwargs:
Adam Tauber's avatar
Adam Tauber committed
321 322 323
        kwargs['categories'] = ['general']
        kwargs['categories'].extend(x for x in
                                    sorted(categories.keys())
Adam Tauber's avatar
Adam Tauber committed
324
                                    if x != 'general'
325
                                    and x in enabled_categories)
326

327 328 329 330 331 332
    if 'all_categories' not in kwargs:
        kwargs['all_categories'] = ['general']
        kwargs['all_categories'].extend(x for x in
                                        sorted(categories.keys())
                                        if x != 'general')

Adam Tauber's avatar
Adam Tauber committed
333
    if 'selected_categories' not in kwargs:
334
        kwargs['selected_categories'] = []
335 336 337 338 339
        for arg in request.args:
            if arg.startswith('category_'):
                c = arg.split('_', 1)[1]
                if c in categories:
                    kwargs['selected_categories'].append(c)
340

341
    if not kwargs['selected_categories']:
Noemi Vanyi's avatar
Noemi Vanyi committed
342
        cookie_categories = request.preferences.get_value('categories')
343
        for ccateg in cookie_categories:
344
            kwargs['selected_categories'].append(ccateg)
345

346 347
    if not kwargs['selected_categories']:
        kwargs['selected_categories'] = ['general']
348

Adam Tauber's avatar
Adam Tauber committed
349
    if 'autocomplete' not in kwargs:
350
        kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
351

352 353 354
    locale = request.preferences.get_value('locale')

    if locale in rtl_locales and 'rtl' not in kwargs:
355 356
        kwargs['rtl'] = True

357 358
    kwargs['searx_version'] = VERSION_STRING

Noemi Vanyi's avatar
Noemi Vanyi committed
359
    kwargs['method'] = request.preferences.get_value('method')
360

Noemi Vanyi's avatar
Noemi Vanyi committed
361
    kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
362

363
    kwargs['language_codes'] = languages
364
    if 'current_language' not in kwargs:
365 366
        kwargs['current_language'] = match_language(request.preferences.get_value('language'),
                                                    LANGUAGE_CODES,
367
                                                    fallback=locale)
368

Matej Cotman's avatar
Matej Cotman committed
369 370 371
    # override url_for function in templates
    kwargs['url_for'] = url_for_theme

Adam Tauber's avatar
Adam Tauber committed
372 373
    kwargs['image_proxify'] = image_proxify

374
    kwargs['proxify'] = proxify if settings.get('result_proxy', {}).get('url') else None
375

376 377
    kwargs['get_result_template'] = get_result_template

Matej Cotman's avatar
Matej Cotman committed
378
    kwargs['theme'] = get_current_theme_name(override=override_theme)
Adam Tauber's avatar
Adam Tauber committed
379

380
    kwargs['template_name'] = template_name
Matej Cotman's avatar
Matej Cotman committed
381

Cqoicebordel's avatar
Cqoicebordel committed
382 383
    kwargs['cookies'] = request.cookies

Adam Tauber's avatar
Adam Tauber committed
384 385
    kwargs['errors'] = request.errors

386 387
    kwargs['instance_name'] = settings['general']['instance_name']

388 389
    kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

Adam Tauber's avatar
Adam Tauber committed
390 391
    kwargs['unicode'] = unicode

392 393
    kwargs['preferences'] = request.preferences

394 395 396 397 398 399 400 401 402 403
    kwargs['scripts'] = set()
    for plugin in request.user_plugins:
        for script in plugin.js_dependencies:
            kwargs['scripts'].add(script)

    kwargs['styles'] = set()
    for plugin in request.user_plugins:
        for css in plugin.css_dependencies:
            kwargs['styles'].add(css)

Matej Cotman's avatar
Matej Cotman committed
404 405
    return render_template(
        '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
asciimoo's avatar
asciimoo committed
406

407

Adam Tauber's avatar
Adam Tauber committed
408 409
@app.before_request
def pre_request():
410 411
    request.start_time = time()
    request.timings = []
Adam Tauber's avatar
Adam Tauber committed
412 413
    request.errors = []

Adam Tauber's avatar
Adam Tauber committed
414
    preferences = Preferences(themes, list(categories.keys()), engines, plugins)
415
    request.preferences = preferences
416
    try:
417
        preferences.parse_dict(request.cookies)
418
    except:
Adam Tauber's avatar
Adam Tauber committed
419
        request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
420

Adam Tauber's avatar
Adam Tauber committed
421
    # merge GET, POST vars
dalf's avatar
dalf committed
422
    # request.form
Adam Tauber's avatar
Adam Tauber committed
423
    request.form = dict(request.form.items())
Adam Tauber's avatar
Adam Tauber committed
424
    for k, v in request.args.items():
Adam Tauber's avatar
Adam Tauber committed
425 426
        if k not in request.form:
            request.form[k] = v
427 428 429 430 431 432 433 434 435

    if request.form.get('preferences'):
        preferences.parse_encoded_data(request.form['preferences'])
    else:
        try:
            preferences.parse_dict(request.form)
        except Exception as e:
            logger.exception('invalid settings')
            request.errors.append(gettext('Invalid settings'))
Adam Tauber's avatar
Adam Tauber committed
436

437 438 439 440 441 442 443
    # init search language and locale
    locale = get_locale()
    if not preferences.get_value("language"):
        preferences.parse_dict({"language": locale})
    if not preferences.get_value("locale"):
        preferences.parse_dict({"locale": locale})

dalf's avatar
dalf committed
444
    # request.user_plugins
Adam Tauber's avatar
Adam Tauber committed
445
    request.user_plugins = []
Noemi Vanyi's avatar
Noemi Vanyi committed
446 447
    allowed_plugins = preferences.plugins.get_enabled()
    disabled_plugins = preferences.plugins.get_disabled()
Adam Tauber's avatar
Adam Tauber committed
448
    for plugin in plugins:
449 450
        if ((plugin.default_on and plugin.id not in disabled_plugins)
                or plugin.id in allowed_plugins):
Adam Tauber's avatar
Adam Tauber committed
451 452 453
            request.user_plugins.append(plugin)


454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
@app.after_request
def post_request(response):
    total_time = time() - request.start_time
    timings_all = ['total;dur=' + str(round(total_time * 1000, 3))]
    if len(request.timings) > 0:
        timings = sorted(request.timings, key=lambda v: v['total'])
        timings_total = ['total_' + str(i) + '_' + v['engine'] +
                         ';dur=' + str(round(v['total'] * 1000, 3)) for i, v in enumerate(timings)]
        timings_load = ['load_' + str(i) + '_' + v['engine'] +
                        ';dur=' + str(round(v['load'] * 1000, 3)) for i, v in enumerate(timings)]
        timings_all = timings_all + timings_total + timings_load
    response.headers.add('Server-Timing', ', '.join(timings_all))
    return response


469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
def index_error(output_format, error_message):
    if output_format == 'json':
        return Response(json.dumps({'error': error_message}),
                        mimetype='application/json')
    elif output_format == 'csv':
        response = Response('', mimetype='application/csv')
        cont_disp = 'attachment;Filename=searx.csv'
        response.headers.add('Content-Disposition', cont_disp)
        return response
    elif output_format == 'rss':
        response_rss = render(
            'opensearch_response_rss.xml',
            results=[],
            q=request.form['q'] if 'q' in request.form else '',
            number_of_results=0,
            base_url=get_base_url(),
485 486
            error_message=error_message,
            override_theme='__common__',
487 488 489 490 491 492 493 494 495 496
        )
        return Response(response_rss, mimetype='text/xml')
    else:
        # html
        request.errors.append(gettext('search error'))
        return render(
            'index.html',
        )


497
@app.route('/search', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
498
@app.route('/', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
499
def index():
Matej Cotman's avatar
Matej Cotman committed
500 501 502 503
    """Render index page.

    Supported outputs: html, json, csv, rss.
    """
504

505 506 507 508 509 510
    # output_format
    output_format = request.form.get('format', 'html')
    if output_format not in ['html', 'csv', 'json', 'rss']:
        output_format = 'html'

    # check if there is query
Adam Tauber's avatar
Adam Tauber committed
511
    if request.form.get('q') is None:
512 513 514 515 516 517
        if output_format == 'html':
            return render(
                'index.html',
            )
        else:
            return index_error(output_format, 'No query'), 400
asciimoo's avatar
asciimoo committed
518

dalf's avatar
dalf committed
519 520
    # search
    search_query = None
521
    raw_text_query = None
dalf's avatar
dalf committed
522
    result_container = None
523
    try:
524
        search_query, raw_text_query = get_search_query_from_webapp(request.preferences, request.form)
dalf's avatar
dalf committed
525
        # search = Search(search_query) #  without plugins
526
        search = SearchWithPlugins(search_query, request.user_plugins, request)
dalf's avatar
dalf committed
527
        result_container = search.search()
528 529
    except Exception as e:
        # log exception
530
        logger.exception('search error')
asciimoo's avatar
asciimoo committed
531

532 533 534 535 536 537 538
        # is it an invalid input parameter or something else ?
        if (issubclass(e.__class__, SearxParameterException)):
            return index_error(output_format, e.message), 400
        else:
            return index_error(output_format, gettext('search error')), 500

    # results
dalf's avatar
dalf committed
539
    results = result_container.get_ordered_results()
540 541 542
    number_of_results = result_container.results_number()
    if number_of_results < result_container.results_length():
        number_of_results = 0
543

dalf's avatar
dalf committed
544
    # UI
545
    advanced_search = request.form.get('advanced_search', None)
546

547 548 549
    # Server-Timing header
    request.timings = result_container.get_timings()

dalf's avatar
dalf committed
550
    # output
551
    for result in results:
dalf's avatar
dalf committed
552
        if output_format == 'html':
553
            if 'content' in result and result['content']:
Adam Tauber's avatar
Adam Tauber committed
554
                result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)
555 556
            if 'title' in result and result['title']:
                result['title'] = highlight_content(escape(result['title'] or u''), search_query.query)
asciimoo's avatar
asciimoo committed
557
        else:
558
            if result.get('content'):
asciimoo's avatar
asciimoo committed
559
                result['content'] = html_to_text(result['content']).strip()
560
            # removing html content and whitespace duplications
561
            result['title'] = ' '.join(html_to_text(result['title']).strip().split())
Adam Tauber's avatar
Adam Tauber committed
562

563 564
        if 'url' in result:
            result['pretty_url'] = prettify_url(result['url'])
asciimoo's avatar
asciimoo committed
565

566 567
        # TODO, check if timezone is calculated right
        if 'publishedDate' in result:
568 569 570 571
            try:  # test if publishedDate >= 1900 (datetime module bug)
                result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z')
            except ValueError:
                result['publishedDate'] = None
572
            else:
573 574 575 576 577
                if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):
                    timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)
                    minutes = int((timedifference.seconds / 60) % 60)
                    hours = int(timedifference.seconds / 60 / 60)
                    if hours == 0:
578
                        result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)
579
                    else:
580
                        result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes)  # noqa
581 582
                else:
                    result['publishedDate'] = format_date(result['publishedDate'])
583

dalf's avatar
dalf committed
584
    if output_format == 'json':
Adam Tauber's avatar
Adam Tauber committed
585
        return Response(json.dumps({'query': search_query.query.decode('utf-8'),
586
                                    'number_of_results': number_of_results,
587 588
                                    'results': results,
                                    'answers': list(result_container.answers),
589
                                    'corrections': list(result_container.corrections),
590
                                    'infoboxes': result_container.infoboxes,
591
                                    'suggestions': list(result_container.suggestions),
592 593
                                    'unresponsive_engines': list(result_container.unresponsive_engines)},
                                   default=lambda item: list(item) if isinstance(item, set) else item),
asciimoo's avatar
asciimoo committed
594
                        mimetype='application/json')
dalf's avatar
dalf committed
595
    elif output_format == 'csv':
Adam Tauber's avatar
Adam Tauber committed
596
        csv = UnicodeWriter(StringIO())
asciimoo's avatar
asciimoo committed
597
        keys = ('title', 'url', 'content', 'host', 'engine', 'score')
598
        csv.writerow(keys)
599
        for row in results:
600 601 602
            row['host'] = row['parsed_url'].netloc
            csv.writerow([row.get(key, '') for key in keys])
        csv.stream.seek(0)
asciimoo's avatar
asciimoo committed
603
        response = Response(csv.stream.read(), mimetype='application/csv')
Adam Tauber's avatar
Adam Tauber committed
604
        cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)
asciimoo's avatar
asciimoo committed
605
        response.headers.add('Content-Disposition', cont_disp)
asciimoo's avatar
asciimoo committed
606
        return response
dalf's avatar
dalf committed
607
    elif output_format == 'rss':
608 609
        response_rss = render(
            'opensearch_response_rss.xml',
610
            results=results,
611
            q=request.form['q'],
612
            number_of_results=number_of_results,
613 614
            base_url=get_base_url(),
            override_theme='__common__',
615
        )
asciimoo's avatar
asciimoo committed
616
        return Response(response_rss, mimetype='text/xml')
617

618 619 620
    # HTML output format

    # suggestions: use RawTextQuery to get the suggestion URLs with the same bang
621 622 623 624 625
    suggestion_urls = list(map(lambda suggestion: {
                               'url': raw_text_query.changeSearchQuery(suggestion).getFullQuery(),
                               'title': suggestion
                               },
                               result_container.suggestions))
626 627 628 629 630 631

    correction_urls = list(map(lambda correction: {
                               'url': raw_text_query.changeSearchQuery(correction).getFullQuery(),
                               'title': correction
                               },
                               result_container.corrections))
632
    #
633 634
    return render(
        'results.html',
635
        results=results,
636
        q=request.form['q'],
dalf's avatar
dalf committed
637 638 639
        selected_categories=search_query.categories,
        pageno=search_query.pageno,
        time_range=search_query.time_range,
640
        number_of_results=format_decimal(number_of_results),
dalf's avatar
dalf committed
641
        advanced_search=advanced_search,
642
        suggestions=suggestion_urls,
dalf's avatar
dalf committed
643
        answers=result_container.answers,
644
        corrections=correction_urls,
dalf's avatar
dalf committed
645 646
        infoboxes=result_container.infoboxes,
        paging=result_container.paging,
647
        unresponsive_engines=result_container.unresponsive_engines,
648 649
        current_language=match_language(search_query.lang,
                                        LANGUAGE_CODES,
650
                                        fallback=request.preferences.get_value("language")),
651
        base_url=get_base_url(),
652
        theme=get_current_theme_name(),
653 654
        favicons=global_favicons[themes.index(get_current_theme_name())],
        timeout_limit=request.form.get('timeout_limit', None)
655
    )
asciimoo's avatar
asciimoo committed
656

asciimoo's avatar
asciimoo committed
657

asciimoo's avatar
asciimoo committed
658 659
@app.route('/about', methods=['GET'])
def about():
Matej Cotman's avatar
Matej Cotman committed
660
    """Render about page"""
661 662 663
    return render(
        'about.html',
    )
asciimoo's avatar
asciimoo committed
664 665


666 667 668
@app.route('/autocompleter', methods=['GET', 'POST'])
def autocompleter():
    """Return autocompleter results"""
669

670
    # set blocked engines
671
    disabled_engines = request.preferences.engines.get_disabled()
672 673

    # parse query
674 675 676 677
    if PY3:
        raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines)
    else:
        raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)
dalf's avatar
dalf committed
678
    raw_text_query.parse_query()
679

680
    # check if search query is set
dalf's avatar
dalf committed
681
    if not raw_text_query.getSearchQuery():
682
        return '', 400
683

Noemi Vanyi's avatar
Noemi Vanyi committed
684 685
    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
686

687
    # parse searx specific autocompleter results like !bang
dalf's avatar
dalf committed
688
    raw_results = searx_bang(raw_text_query)
689

690 691 692 693 694
    # normal autocompletion results only appear if no inner results returned
    # and there is a query part besides the engine and language bangs
    if len(raw_results) == 0 and completer and (len(raw_text_query.query_parts) > 1 or
                                                (len(raw_text_query.languages) == 0 and
                                                 not raw_text_query.specific)):
a01200356's avatar
a01200356 committed
695
        # get language from cookie
696
        language = request.preferences.get_value('language')
697 698
        if not language or language == 'all':
            language = 'en'
a01200356's avatar
a01200356 committed
699
        else:
700
            language = language.split('-')[0]
701
        # run autocompletion
dalf's avatar
dalf committed
702
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
703 704 705 706

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
dalf's avatar
dalf committed
707
        raw_text_query.changeSearchQuery(result)
708 709

        # add parsed result
dalf's avatar
dalf committed
710
        results.append(raw_text_query.getFullQuery())
711

712
    # return autocompleter results
713
    if request.form.get('format') == 'x-suggestions':
dalf's avatar
dalf committed
714
        return Response(json.dumps([raw_text_query.query, results]),
715
                        mimetype='application/json')
716 717 718

    return Response(json.dumps(results),
                    mimetype='application/json')
719 720


asciimoo's avatar
asciimoo committed
721 722
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
Noemi Vanyi's avatar
Noemi Vanyi committed
723
    """Render preferences page && save user preferences"""
asciimoo's avatar
asciimoo committed
724

Noemi Vanyi's avatar
Noemi Vanyi committed
725 726 727 728 729 730
    # save preferences
    if request.method == 'POST':
        resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
        try:
            request.preferences.parse_form(request.form)
        except ValidationException:
Adam Tauber's avatar
Adam Tauber committed
731
            request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
732 733 734 735 736 737
            return resp
        return request.preferences.save(resp)

    # render preferences
    image_proxy = request.preferences.get_value('image_proxy')
    lang = request.preferences.get_value('language')
738
    disabled_engines = request.preferences.engines.get_disabled()
Noemi Vanyi's avatar
Noemi Vanyi committed
739
    allowed_plugins = request.preferences.plugins.get_enabled()
740 741 742 743 744 745 746 747 748

    # stats for preferences page
    stats = {}

    for c in categories:
        for e in categories[c]:
            stats[e.name] = {'time': None,
                             'warn_timeout': False,
                             'warn_time': False}
749
            if e.timeout > settings['outgoing']['request_timeout']:
750
                stats[e.name]['warn_timeout'] = True
751
            stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)
752

753 754
    # get first element [0], the engine time,
    # and then the second element [1] : the time (the first one is the label)
755 756
    for engine_stat in get_engines_stats()[0][1]:
        stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
757
        if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
758 759 760
            stats[engine_stat.get('name')]['warn_time'] = True
    # end of stats

asciimoo's avatar
asciimoo committed
761 762
    return render('preferences.html',
                  locales=settings['locales'],
763
                  current_locale=request.preferences.get_value("locale"),
Adam Tauber's avatar
Adam Tauber committed
764
                  image_proxy=image_proxy,
765
                  engines_by_category=categories,
766
                  stats=stats,
767
                  answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
768
                  disabled_engines=disabled_engines,
769
                  autocomplete_backends=autocomplete_backends,
Matej Cotman's avatar
Matej Cotman committed
770 771
                  shortcuts={y: x for x, y in engine_shortcuts.items()},
                  themes=themes,
772
                  plugins=plugins,
773
                  doi_resolvers=settings['doi_resolvers'],
Noémi Ványi's avatar
Noémi Ványi committed
774
                  current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
Noemi Vanyi's avatar
Noemi Vanyi committed
775
                  allowed_plugins=allowed_plugins,
776
                  theme=get_current_theme_name(),
777 778
                  preferences_url_params=request.preferences.get_as_url_params(),
                  base_url=get_base_url(),
779
                  preferences=True)
asciimoo's avatar
asciimoo committed
780 781


782 783 784 785 786 787 788 789
def _is_selected_language_supported(engine, preferences):
    language = preferences.get_value('language')
    return (language == 'all'
            or match_language(language,
                              getattr(engine, 'supported_languages', []),
                              getattr(engine, 'language_aliases', {}), None))


Adam Tauber's avatar
Adam Tauber committed
790 791
@app.route('/image_proxy', methods=['GET'])
def image_proxy():
792
    url = request.args.get('url').encode('utf-8')
Adam Tauber's avatar
Adam Tauber committed
793 794 795 796

    if not url:
        return '', 400

797
    h = new_hmac(settings['server']['secret_key'], url)
798 799 800 801 802 803 804

    if h != request.args.get('h'):
        return '', 400

    headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
    headers['User-Agent'] = gen_useragent()

805 806
    resp = requests.get(url,
                        stream=True,
807
                        timeout=settings['outgoing']['request_timeout'],
808 809
                        headers=headers,
                        proxies=outgoing_proxies)
810 811 812

    if resp.status_code == 304:
        return '', resp.status_code
Adam Tauber's avatar
Adam Tauber committed
813 814 815 816 817 818 819 820

    if resp.status_code != 200:
        logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
        if resp.status_code >= 400:
            return '', resp.status_code
        return '', 400

    if not resp.headers.get('content-type', '').startswith('image/'):
Adam Tauber's avatar
Adam Tauber committed
821
        logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
Adam Tauber's avatar
Adam Tauber committed
822 823
        return '', 400

824
    img = b''
Adam Tauber's avatar
Adam Tauber committed
825 826
    chunk_counter = 0

827
    for chunk in resp.iter_content(1024 * 1024):
Adam Tauber's avatar
Adam Tauber committed
828 829 830 831 832
        chunk_counter += 1
        if chunk_counter > 5:
            return '', 502  # Bad gateway - file is too big (>5M)
        img += chunk

833 834 835
    headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

    return Response(img, mimetype=resp.headers['content-type'], headers=headers)
Adam Tauber's avatar
Adam Tauber committed
836 837


asciimoo's avatar
asciimoo committed
838 839
@app.route('/stats', methods=['GET'])
def stats():
Matej Cotman's avatar
Matej Cotman committed
840
    """Render engine statistics page."""
asciimoo's avatar
asciimoo committed
841
    stats = get_engines_stats()
842 843 844 845
    return render(
        'stats.html',
        stats=stats,
    )
asciimoo's avatar
asciimoo committed
846

asciimoo's avatar
asciimoo committed
847

asciimoo's avatar
asciimoo committed
848 849 850 851 852 853
@app.route('/robots.txt', methods=['GET'])
def robots():
    return Response("""User-agent: *
Allow: /
Allow: /about
Disallow: /stats
asciimoo's avatar
asciimoo committed
854
Disallow: /preferences
855
Disallow: /*?*q=*
asciimoo's avatar
asciimoo committed
856 857
""", mimetype='text/plain')

asciimoo's avatar
asciimoo committed
858

asciimoo's avatar
asciimoo committed
859 860
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
asciimoo's avatar
asciimoo committed
861
    method = 'post'
Luc Didry's avatar
Luc Didry committed
862

Noemi Vanyi's avatar
Noemi Vanyi committed
863
    if request.preferences.get_value('method') == 'GET':
Luc Didry's avatar
Luc Didry committed
864 865
        method = 'get'

asciimoo's avatar
asciimoo committed
866
    # chrome/chromium only supports HTTP GET....
asciimoo's avatar
asciimoo committed
867 868
    if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
        method = 'get'
869 870

    ret = render('opensearch.xml',
871
                 opensearch_method=method,
872
                 host=get_base_url(),
873 874
                 urljoin=urljoin,
                 override_theme='__common__')
875

asciimoo's avatar
asciimoo committed
876
    resp = Response(response=ret,
asciimoo's avatar
asciimoo committed
877
                    status=200,
878
                    mimetype="text/xml")
asciimoo's avatar
asciimoo committed
879 880
    return resp

881

asciimoo's avatar
asciimoo committed
882 883
@app.route('/favicon.ico')
def favicon():
Matej Cotman's avatar
Matej Cotman committed
884
    return send_from_directory(os.path.join(app.root_path,
885 886
                                            static_path,
                                            'themes',
Matej Cotman's avatar
Matej Cotman committed
887 888
                                            get_current_theme_name(),
                                            'img'),
asciimoo's avatar
asciimoo committed
889 890
                               'favicon.png',
                               mimetype='image/vnd.microsoft.icon')
asciimoo's avatar
asciimoo committed
891 892


893 894
@app.route('/clear_cookies')
def clear_cookies():
895
    resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
896 897 898 899 900
    for cookie_name in request.cookies:
        resp.delete_cookie(cookie_name)
    return resp


901 902
@app.route('/config')
def config():
903
    return jsonify({'categories': list(categories.keys()),
904 905
                    'engines': [{'name': engine_name,
                                 'categories': engine.categories,
906
                                 'shortcut': engine.shortcut,
907 908 909 910
                                 'enabled': not engine.disabled,
                                 'paging': engine.paging,
                                 'language_support': engine.language_support,
                                 'supported_languages':
911
                                 list(engine.supported_languages.keys())
912 913 914 915 916
                                 if isinstance(engine.supported_languages, dict)
                                 else engine.supported_languages,
                                 'safesearch': engine.safesearch,
                                 'time_range_support': engine.time_range_support,
                                 'timeout': engine.timeout}
917 918 919 920 921 922 923 924 925
                                for engine_name, engine in engines.items()],
                    'plugins': [{'name': plugin.name,
                                 'enabled': plugin.default_on}
                                for plugin in plugins],
                    'instance_name': settings['general']['instance_name'],
                    'locales': settings['locales'],
                    'default_locale': settings['ui']['default_locale'],
                    'autocomplete': settings['search']['autocomplete'],
                    'safe_search': settings['search']['safe_search'],
926
                    'default_theme': settings['ui']['default_theme'],
927
                    'version': VERSION_STRING,
928
                    'doi_resolvers': [r for r in settings['doi_resolvers']],
929 930
                    'default_doi_resolver': settings['default_doi_resolver'],
                    })
931 932


Noemi Vanyi's avatar
Noemi Vanyi committed
933 934
@app.errorhandler(404)
def page_not_found(e):
935
    return render('404.html'), 404
Noemi Vanyi's avatar
Noemi Vanyi committed
936 937


938
def run():
939
    logger.debug('starting webserver on %s:%s', settings['server']['bind_address'], settings['server']['port'])
940
    app.run(
941 942
        debug=searx_debug,
        use_debugger=searx_debug,
943
        port=settings['server']['port'],
944 945
        host=settings['server']['bind_address'],
        threaded=True
946
    )
947 948


949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967
class ReverseProxyPathFix(object):
    '''Wrap the application in this middleware and configure the
    front-end server to add these headers, to let you quietly bind
    this to a URL other than / and to an HTTP scheme that is
    different than what is used locally.

    http://flask.pocoo.org/snippets/35/

    In nginx:
    location /myprefix {
        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Scheme $scheme;
        proxy_set_header X-Script-Name /myprefix;
        }

    :param app: the WSGI application
    '''
968