webapp.py 30.4 KB
Newer Older
asciimoo's avatar
asciimoo committed
1
2
#!/usr/bin/env python

asciimoo's avatar
asciimoo committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.

(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''

asciimoo's avatar
asciimoo committed
20
21
22
if __name__ == '__main__':
    from sys import path
    from os.path import realpath, dirname
23

24
    path.append(realpath(dirname(realpath(__file__)) + '/../'))
asciimoo's avatar
asciimoo committed
25

26
import hashlib
27
28
29
import hmac
import json
import os
Adam Tauber's avatar
Adam Tauber committed
30
import sys
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
31
import time
Omer's avatar
Omer committed
32
import copy
Adam Tauber's avatar
Adam Tauber committed
33

34
import requests
asciimoo's avatar
asciimoo committed
35

Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
36
from searx import logger, search_database
37

38
39
40
41
42
logger = logger.getChild('webapp')

try:
    from pygments import highlight
    from pygments.lexers import get_lexer_by_name
43
    from pygments.util import ClassNotFound
44
    from pygments.formatters import HtmlFormatter
45
except ImportError:
46
47
    logger.critical("cannot import dependency: pygments")
    from sys import exit
48

49
    exit(1)
Nicolas Gelot's avatar
Nicolas Gelot committed
50
from html import escape
51
from datetime import datetime, timedelta
52
from werkzeug.contrib.fixers import ProxyFix
Gabor Nagy's avatar
Gabor Nagy committed
53
54
55
56
from flask import (
    Flask, request, render_template, url_for, Response, make_response,
    redirect, send_from_directory
)
57
from flask_babel import Babel, gettext, format_date, format_decimal
58
from flask.json import jsonify
59
from searx import settings, searx_dir, searx_debug
Adam Tauber's avatar
Adam Tauber committed
60
from searx.exceptions import SearxParameterException
Gabor Nagy's avatar
Gabor Nagy committed
61
from searx.engines import (
Adam Tauber's avatar
Adam Tauber committed
62
    categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
Gabor Nagy's avatar
Gabor Nagy committed
63
)
Matej Cotman's avatar
Matej Cotman committed
64
from searx.utils import (
65
    highlight_content, get_resources_directory,
66
    get_static_files, get_result_templates, get_themes, gen_useragent,
67
    dict_subset, prettify_url, match_language
Matej Cotman's avatar
Matej Cotman committed
68
)
69
from searx.version import VERSION_STRING
70
from searx.languages import language_codes as languages
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
71
72
from searx.search import Search
from searx.query import RawTextQuery
73
from searx.autocomplete import searx_bang, backends as autocomplete_backends
74
from searx.plugins import plugins
Noémi Ványi's avatar
Noémi Ványi committed
75
from searx.plugins.oa_doi_rewrite import get_doi_resolver
76
from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
77
from searx.answerers import answerers
Adam Tauber's avatar
Adam Tauber committed
78
from searx.url_utils import urlencode, urlparse, urljoin
Noémi Ványi's avatar
Noémi Ványi committed
79
from searx.utils import new_hmac
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
80
81
from searx.search_database import get_twenty_queries, search
import threading
asciimoo's avatar
asciimoo committed
82

83
84
# check if the pyopenssl package is installed.
# It is needed for SSL connection without trouble, see #298
85
86
87
try:
    import OpenSSL.SSL  # NOQA
except ImportError:
88
    logger.critical("The pyopenssl package has to be installed.\n"
89
                    "Some HTTPS connections will fail")
90

Nicolas Gelot's avatar
Nicolas Gelot committed
91
from io import StringIO
Adam Tauber's avatar
Adam Tauber committed
92

Eig8phei's avatar
Eig8phei committed
93
94
# serve pages with HTTP/1.1
from werkzeug.serving import WSGIRequestHandler
95

96
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
asciimoo's avatar
asciimoo committed
97

98
99
100
101
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
Adam Tauber's avatar
Adam Tauber committed
102

103
# about templates
104
default_theme = settings['ui']['default_theme']
105
106
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
107
themes = get_themes(templates_path)
108
109
110
111
112
113
114
result_templates = get_result_templates(templates_path)
global_favicons = []
for indice, theme in enumerate(themes):
    global_favicons.append([])
    theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
    for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
        global_favicons[indice].extend(filenames)
Matej Cotman's avatar
Matej Cotman committed
115

116
# Flask app
117
118
app = Flask(
    __name__,
Matej Cotman's avatar
Matej Cotman committed
119
120
    static_folder=static_path,
    template_folder=templates_path
121
122
)

123
124
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
125
app.secret_key = settings['server']['secret_key']
126

127
if not searx_debug \
128
129
        or os.environ.get("WERKZEUG_RUN_MAIN") == "true" \
        or os.environ.get('UWSGI_ORIGINAL_PROC_NAME') is not None:
130
131
    initialize_engines(settings['engines'])

132
133
babel = Babel(app)

134
rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
135
               'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
136

137
# used when translating category names
138
139
140
141
142
143
144
145
_category_names = (gettext('files'),
                   gettext('general'),
                   gettext('music'),
                   gettext('social media'),
                   gettext('images'),
                   gettext('videos'),
                   gettext('it'),
                   gettext('news'),
146
                   gettext('map'),
Thomas Pointhuber's avatar
Thomas Pointhuber committed
147
                   gettext('science'))
148

149
outgoing_proxies = settings['outgoing'].get('proxies') or None
asciimoo's avatar
asciimoo committed
150
151


152
153
@babel.localeselector
def get_locale():
asciimoo's avatar
asciimoo committed
154
155
    locale = request.accept_languages.best_match(settings['locales'].keys())

156
157
    if request.preferences.get_value('locale') != '':
        locale = request.preferences.get_value('locale')
asciimoo's avatar
asciimoo committed
158

159
160
    if 'locale' in request.args \
            and request.args['locale'] in settings['locales']:
asciimoo's avatar
asciimoo committed
161
162
        locale = request.args['locale']

163
164
    if 'locale' in request.form \
            and request.form['locale'] in settings['locales']:
asciimoo's avatar
asciimoo committed
165
166
        locale = request.form['locale']

167
168
169
    if locale == 'zh_TW':
        locale = 'zh_Hant_TW'

asciimoo's avatar
asciimoo committed
170
    return locale
171
172


173
174
175
176
177
178
# code-highlighter
@app.template_filter('code_highlighter')
def code_highlighter(codelines, language=None):
    if not language:
        language = 'text'

179
180
181
    try:
        # find lexer by programing language
        lexer = get_lexer_by_name(language, stripall=True)
182
    except ClassNotFound:
183
        # if lexer is not found, using default one
Adam Tauber's avatar
Adam Tauber committed
184
        logger.debug('highlighter cannot find lexer for {0}'.format(language))
185
186
        lexer = get_lexer_by_name('text', stripall=True)

187
188
189
190
191
192
193
194
195
196
    html_code = ''
    tmp_code = ''
    last_line = None

    # parse lines
    for line, code in codelines:
        if not last_line:
            line_code_start = line

        # new codeblock is detected
197
198
        if last_line is not None and \
                last_line + 1 != line:
199
            # highlight last codepart
200
201
            formatter = HtmlFormatter(linenos='inline',
                                      linenostart=line_code_start)
202
            html_code = html_code + highlight(tmp_code, lexer, formatter)
203

204
205
206
207
208
209
            # reset conditions for next codepart
            tmp_code = ''
            line_code_start = line

        # add codepart
        tmp_code += code + '\n'
210

211
212
213
214
215
216
217
218
219
220
        # update line
        last_line = line

    # highlight last codepart
    formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start)
    html_code = html_code + highlight(tmp_code, lexer, formatter)

    return html_code


Cqoicebordel's avatar
Cqoicebordel committed
221
222
223
224
225
226
# Extract domain from url
@app.template_filter('extract_domain')
def extract_domain(url):
    return urlparse(url)[1]


227
def get_base_url():
228
229
    if settings['server']['base_url']:
        hostname = settings['server']['base_url']
230
231
232
233
234
235
236
237
    else:
        scheme = 'http'
        if request.is_secure:
            scheme = 'https'
        hostname = url_for('index', _external=True, _scheme=scheme)
    return hostname


Matej Cotman's avatar
Matej Cotman committed
238
239
240
241
242
243
244
245
def get_current_theme_name(override=None):
    """Returns theme name.

    Checks in this order:
    1. override
    2. cookies
    3. settings"""

246
    if override and (override in themes or override == '__common__'):
Matej Cotman's avatar
Matej Cotman committed
247
        return override
Noemi Vanyi's avatar
Noemi Vanyi committed
248
    theme_name = request.args.get('theme', request.preferences.get_value('theme'))
Matej Cotman's avatar
Matej Cotman committed
249
250
251
252
253
    if theme_name not in themes:
        theme_name = default_theme
    return theme_name


254
255
256
257
258
259
260
def get_result_template(theme, template_name):
    themed_path = theme + '/result_templates/' + template_name
    if themed_path in result_templates:
        return themed_path
    return 'result_templates/' + template_name


Matej Cotman's avatar
Matej Cotman committed
261
def url_for_theme(endpoint, override_theme=None, **values):
262
    if endpoint == 'static' and values.get('filename'):
Matej Cotman's avatar
Matej Cotman committed
263
        theme_name = get_current_theme_name(override=override_theme)
264
265
266
        filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
        if filename_with_theme in static_files:
            values['filename'] = filename_with_theme
Matej Cotman's avatar
Matej Cotman committed
267
268
269
    return url_for(endpoint, **values)


270
271
272
273
274
275
276
def proxify(url):
    if url.startswith('//'):
        url = 'https:' + url

    if not settings.get('result_proxy'):
        return url

Nicolas Gelot's avatar
Nicolas Gelot committed
277
    url_params = dict(mortyurl=url)
278
279
280

    if settings['result_proxy'].get('key'):
        url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
Nicolas Gelot's avatar
Nicolas Gelot committed
281
                                           url,
282
                                           hashlib.sha256).hexdigest()
283
284

    return '{0}?{1}'.format(settings['result_proxy']['url'],
285
                            urlencode(url_params))
286
287


Adam Tauber's avatar
Adam Tauber committed
288
289
290
291
def image_proxify(url):
    if url.startswith('//'):
        url = 'https:' + url

292
    if not request.preferences.get_value('image_proxy'):
Adam Tauber's avatar
Adam Tauber committed
293
294
        return url

295
296
297
    if settings.get('result_proxy'):
        return proxify(url)

Nicolas Gelot's avatar
Nicolas Gelot committed
298
    h = new_hmac(settings['server']['secret_key'], url)
dalf's avatar
dalf committed
299

Adam Tauber's avatar
Adam Tauber committed
300
    return '{0}?{1}'.format(url_for('image_proxy'),
Nicolas Gelot's avatar
Nicolas Gelot committed
301
                            urlencode(dict(url=url, h=h)))
Adam Tauber's avatar
Adam Tauber committed
302
303


Matej Cotman's avatar
Matej Cotman committed
304
def render(template_name, override_theme=None, **kwargs):
305
    disabled_engines = request.preferences.engines.get_disabled()
306

307
308
309
    enabled_categories = set(category for engine_name in engines
                             for category in engines[engine_name].categories
                             if (engine_name, category) not in disabled_engines)
310

Adam Tauber's avatar
Adam Tauber committed
311
    if 'categories' not in kwargs:
Adam Tauber's avatar
Adam Tauber committed
312
313
314
        kwargs['categories'] = ['general']
        kwargs['categories'].extend(x for x in
                                    sorted(categories.keys())
Adam Tauber's avatar
Adam Tauber committed
315
                                    if x != 'general'
316
                                    and x in enabled_categories)
317

318
319
320
321
322
323
    if 'all_categories' not in kwargs:
        kwargs['all_categories'] = ['general']
        kwargs['all_categories'].extend(x for x in
                                        sorted(categories.keys())
                                        if x != 'general')

Adam Tauber's avatar
Adam Tauber committed
324
    if 'selected_categories' not in kwargs:
325
        kwargs['selected_categories'] = []
326
327
328
329
330
        for arg in request.args:
            if arg.startswith('category_'):
                c = arg.split('_', 1)[1]
                if c in categories:
                    kwargs['selected_categories'].append(c)
331

332
    if not kwargs['selected_categories']:
Noemi Vanyi's avatar
Noemi Vanyi committed
333
        cookie_categories = request.preferences.get_value('categories')
334
        for ccateg in cookie_categories:
335
            kwargs['selected_categories'].append(ccateg)
336

337
338
    if not kwargs['selected_categories']:
        kwargs['selected_categories'] = ['general']
339

Adam Tauber's avatar
Adam Tauber committed
340
    if 'autocomplete' not in kwargs:
341
        kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
342

343
344
345
    if get_locale() in rtl_locales and 'rtl' not in kwargs:
        kwargs['rtl'] = True

346
347
    kwargs['searx_version'] = VERSION_STRING

Noemi Vanyi's avatar
Noemi Vanyi committed
348
    kwargs['method'] = request.preferences.get_value('method')
349

Noemi Vanyi's avatar
Noemi Vanyi committed
350
    kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
351

352
    kwargs['language_codes'] = languages
353
    if 'current_language' not in kwargs:
354
355
356
        kwargs['current_language'] = match_language(request.preferences.get_value('language'),
                                                    LANGUAGE_CODES,
                                                    fallback=settings['search']['language'])
357

Matej Cotman's avatar
Matej Cotman committed
358
359
360
    # override url_for function in templates
    kwargs['url_for'] = url_for_theme

Adam Tauber's avatar
Adam Tauber committed
361
362
    kwargs['image_proxify'] = image_proxify

363
    kwargs['proxify'] = proxify if settings.get('result_proxy', {}).get('url') else None
364

365
366
    kwargs['get_result_template'] = get_result_template

Matej Cotman's avatar
Matej Cotman committed
367
    kwargs['theme'] = get_current_theme_name(override=override_theme)
Adam Tauber's avatar
Adam Tauber committed
368

369
    kwargs['template_name'] = template_name
Matej Cotman's avatar
Matej Cotman committed
370

Cqoicebordel's avatar
Cqoicebordel committed
371
372
    kwargs['cookies'] = request.cookies

Adam Tauber's avatar
Adam Tauber committed
373
374
    kwargs['errors'] = request.errors

375
376
    kwargs['instance_name'] = settings['general']['instance_name']

377
378
    kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

Nicolas Gelot's avatar
Nicolas Gelot committed
379
    kwargs['unicode'] = str
Adam Tauber's avatar
Adam Tauber committed
380

381
382
    kwargs['preferences'] = request.preferences

383
384
385
386
387
388
389
390
391
392
    kwargs['scripts'] = set()
    for plugin in request.user_plugins:
        for script in plugin.js_dependencies:
            kwargs['scripts'].add(script)

    kwargs['styles'] = set()
    for plugin in request.user_plugins:
        for css in plugin.css_dependencies:
            kwargs['styles'].add(css)

Matej Cotman's avatar
Matej Cotman committed
393
394
    return render_template(
        '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
asciimoo's avatar
asciimoo committed
395

396

Adam Tauber's avatar
Adam Tauber committed
397
398
@app.before_request
def pre_request():
Adam Tauber's avatar
Adam Tauber committed
399
400
    request.errors = []

Adam Tauber's avatar
Adam Tauber committed
401
    preferences = Preferences(themes, list(categories.keys()), engines, plugins)
402
    request.preferences = preferences
403
    try:
404
        preferences.parse_dict(request.cookies)
405
    except:
Adam Tauber's avatar
Adam Tauber committed
406
        request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
407

Adam Tauber's avatar
Adam Tauber committed
408
    # merge GET, POST vars
dalf's avatar
dalf committed
409
    # request.form
Adam Tauber's avatar
Adam Tauber committed
410
    request.form = dict(request.form.items())
Adam Tauber's avatar
Adam Tauber committed
411
    for k, v in request.args.items():
Adam Tauber's avatar
Adam Tauber committed
412
413
        if k not in request.form:
            request.form[k] = v
414
415
416
417
418
419
420
421
422

    if request.form.get('preferences'):
        preferences.parse_encoded_data(request.form['preferences'])
    else:
        try:
            preferences.parse_dict(request.form)
        except Exception as e:
            logger.exception('invalid settings')
            request.errors.append(gettext('Invalid settings'))
Adam Tauber's avatar
Adam Tauber committed
423

dalf's avatar
dalf committed
424
    # request.user_plugins
Adam Tauber's avatar
Adam Tauber committed
425
    request.user_plugins = []
Noemi Vanyi's avatar
Noemi Vanyi committed
426
427
    allowed_plugins = preferences.plugins.get_enabled()
    disabled_plugins = preferences.plugins.get_disabled()
Adam Tauber's avatar
Adam Tauber committed
428
    for plugin in plugins:
429
430
        if ((plugin.default_on and plugin.id not in disabled_plugins)
                or plugin.id in allowed_plugins):
Adam Tauber's avatar
Adam Tauber committed
431
432
            request.user_plugins.append(plugin)

433

434
435
436
437
def config_results(results, query):
    for result in results:
        if 'content' in result and result['content']:
            result['content'] = highlight_content(escape(result['content'][:1024]), query)
Nicolas Gelot's avatar
Nicolas Gelot committed
438
        result['title'] = highlight_content(escape(result['title'] or ''), query)
439
        result['pretty_url'] = prettify_url(result['url'])
Adam Tauber's avatar
Adam Tauber committed
440

Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
441
442
443
444
445
446
447
        if 'pubdate' in result:
            publishedDate = datetime.strptime(result['pubdate'], '%Y-%m-%d %H:%M:%S')
            if publishedDate >= datetime.now() - timedelta(days=1):
                timedifference = datetime.now() - publishedDate
                minutes = int((timedifference.seconds / 60) % 60)
                hours = int(timedifference.seconds / 60 / 60)
                if hours == 0:
Nicolas Gelot's avatar
Nicolas Gelot committed
448
                    result['publishedDate'] = gettext('{minutes} minute(s) ago').format(minutes=minutes)
449
                else:
Nicolas Gelot's avatar
Nicolas Gelot committed
450
                    result['publishedDate'] = gettext('{hours} hour(s), {minutes} minute(s) ago').format(
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
451
452
453
                        hours=hours, minutes=minutes)  # noqa
            else:
                result['publishedDate'] = format_date(publishedDate)
454

455

456
457
458
459
460
461
462
def index_error(exn, output):
    user_error = gettext("search error")
    if output == "json":
        return jsonify({"error": f"{user_error}: {exn}"})

    request.errors.append(user_error)
    return render('index.html')
463
464


465
@app.route('/search', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
466
@app.route('/', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
467
def index():
468
469
470
    # check the response format
    output = request.form.get("output", "html")

471
    # check if there is query
Adam Tauber's avatar
Adam Tauber committed
472
    if request.form.get('q') is None:
473
474
475
        if output == 'json':
            return jsonify({}), 204
        return render('index.html')
asciimoo's avatar
asciimoo committed
476

Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
477
478
479
480
    selected_category = request.form.get('category') or 'general'
    first_page = request.form.get('pageno')
    is_general_first_page = selected_category == 'general' and (first_page is None or first_page == u'1')

Omer's avatar
Omer committed
481
482
    images = []
    videos = []
dalf's avatar
dalf committed
483
    # search
Omer's avatar
Omer committed
484
    search_data = None
485
    try:
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
486
487
488
489
        if is_general_first_page:
            request.form['categories'] = ['general', 'videos', 'images']
        else:
            request.form['categories'] = [selected_category]
Omer's avatar
Omer committed
490
        search_data = search(request, settings['redis']['host'])
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
491

492
493
    except Exception as e:
        # log exception
494
        logger.exception('search error')
asciimoo's avatar
asciimoo committed
495

496
        # is it an invalid input parameter or something else ?
Omer's avatar
Omer committed
497
        if issubclass(e.__class__, SearxParameterException):
498
            return index_error(e, output), 400
499
        else:
500
            return index_error(e, output), 500
501

Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
502
    if is_general_first_page:
Omer's avatar
Omer committed
503
504
505
506
507
508
509
510
511
512
513
514
        result_copy = copy.copy(search_data.results)
        for res in result_copy:
            if res.get('category') == 'images':
                if len(images) < 5:
                    images.append(res)
                search_data.results.remove(res)
            elif res.get('category') == 'videos':
                if len(videos) < 5:
                    videos.append(res)
                search_data.results.remove(res)
            elif res.get('category') is None:
                search_data.results.remove(res)
515

dalf's avatar
dalf committed
516
    # output
Omer's avatar
Omer committed
517
518
519
    config_results(search_data.results, search_data.query)
    config_results(images, search_data.query)
    config_results(videos, search_data.query)
520

521
    response = dict(
Omer's avatar
Omer committed
522
        results=search_data.results,
Nicolas Gelot's avatar
Nicolas Gelot committed
523
        q=search_data.query,
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
524
        selected_category=selected_category,
Omer's avatar
Omer committed
525
526
527
        pageno=search_data.pageno,
        time_range=search_data.time_range,
        number_of_results=format_decimal(search_data.results_number),
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
528
        advanced_search=request.form.get('advanced_search', None),
529
530
531
        suggestions=list(search_data.suggestions),
        answers=list(search_data.answers),
        corrections=list(search_data.corrections),
Omer's avatar
Omer committed
532
533
        infoboxes=search_data.infoboxes,
        paging=search_data.paging,
534
        unresponsive_engines=list(search_data.unresponsive_engines),
Omer's avatar
Omer committed
535
        current_language=match_language(search_data.language,
536
537
                                        LANGUAGE_CODES,
                                        fallback=settings['search']['language']),
538
        image_results=images,
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
539
        videos_results=videos,
540
        base_url=get_base_url(),
541
        theme=get_current_theme_name(),
542
        favicons=global_favicons[themes.index(get_current_theme_name())]
543
    )
544
545
546
    if output == 'json':
        return jsonify(response)
    return render('results.html', **response)
asciimoo's avatar
asciimoo committed
547

asciimoo's avatar
asciimoo committed
548

asciimoo's avatar
asciimoo committed
549
550
@app.route('/about', methods=['GET'])
def about():
Matej Cotman's avatar
Matej Cotman committed
551
    """Render about page"""
552
553
554
    return render(
        'about.html',
    )
asciimoo's avatar
asciimoo committed
555
556


557
558
559
@app.route('/autocompleter', methods=['GET', 'POST'])
def autocompleter():
    """Return autocompleter results"""
560
    # set blocked engines
561
    disabled_engines = request.preferences.engines.get_disabled()
562
563

    # parse query
Nicolas Gelot's avatar
Nicolas Gelot committed
564
    raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
dalf's avatar
dalf committed
565
    raw_text_query.parse_query()
566

567
    # check if search query is set
dalf's avatar
dalf committed
568
    if not raw_text_query.getSearchQuery():
569
        return '', 400
570

Noemi Vanyi's avatar
Noemi Vanyi committed
571
572
    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
573

574
    # parse searx specific autocompleter results like !bang
dalf's avatar
dalf committed
575
    raw_results = searx_bang(raw_text_query)
576

577
578
    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
a01200356's avatar
a01200356 committed
579
        # get language from cookie
580
        language = request.preferences.get_value('language')
581
582
        if not language:
            language = settings['search']['language']
a01200356's avatar
a01200356 committed
583
        else:
584
            language = language.split('-')[0]
585
        # run autocompletion
dalf's avatar
dalf committed
586
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
587
588
589
590

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
dalf's avatar
dalf committed
591
        raw_text_query.changeSearchQuery(result)
592
593

        # add parsed result
dalf's avatar
dalf committed
594
        results.append(raw_text_query.getFullQuery())
595

596
    # return autocompleter results
597
    if request.form.get('format') == 'x-suggestions':
dalf's avatar
dalf committed
598
        return Response(json.dumps([raw_text_query.query, results]),
599
                        mimetype='application/json')
600
601
602

    return Response(json.dumps(results),
                    mimetype='application/json')
603
604


asciimoo's avatar
asciimoo committed
605
606
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
Noemi Vanyi's avatar
Noemi Vanyi committed
607
    """Render preferences page && save user preferences"""
asciimoo's avatar
asciimoo committed
608

Noemi Vanyi's avatar
Noemi Vanyi committed
609
610
611
612
613
614
    # save preferences
    if request.method == 'POST':
        resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
        try:
            request.preferences.parse_form(request.form)
        except ValidationException:
Adam Tauber's avatar
Adam Tauber committed
615
            request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
616
617
618
619
620
621
            return resp
        return request.preferences.save(resp)

    # render preferences
    image_proxy = request.preferences.get_value('image_proxy')
    lang = request.preferences.get_value('language')
622
    disabled_engines = request.preferences.engines.get_disabled()
Noemi Vanyi's avatar
Noemi Vanyi committed
623
    allowed_plugins = request.preferences.plugins.get_enabled()
624
625
626
627
628
629
630
631
632

    # stats for preferences page
    stats = {}

    for c in categories:
        for e in categories[c]:
            stats[e.name] = {'time': None,
                             'warn_timeout': False,
                             'warn_time': False}
633
            if e.timeout > settings['outgoing']['request_timeout']:
634
                stats[e.name]['warn_timeout'] = True
635
636
637
638
            if match_language(request.preferences.get_value('language'),
                              getattr(e, 'supported_languages', []),
                              getattr(e, 'language_aliases', {}), None):
                stats[e.name]['supports_selected_language'] = True
639

640
641
    # get first element [0], the engine time,
    # and then the second element [1] : the time (the first one is the label)
642
643
    for engine_stat in get_engines_stats()[0][1]:
        stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
644
        if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
645
646
647
            stats[engine_stat.get('name')]['warn_time'] = True
    # end of stats

asciimoo's avatar
asciimoo committed
648
649
    return render('preferences.html',
                  locales=settings['locales'],
asciimoo's avatar
asciimoo committed
650
                  current_locale=get_locale(),
Adam Tauber's avatar
Adam Tauber committed
651
                  image_proxy=image_proxy,
652
                  engines_by_category=categories,
653
                  stats=stats,
654
                  answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
655
                  disabled_engines=disabled_engines,
656
                  autocomplete_backends=autocomplete_backends,
Matej Cotman's avatar
Matej Cotman committed
657
658
                  shortcuts={y: x for x, y in engine_shortcuts.items()},
                  themes=themes,
659
                  plugins=plugins,
660
                  doi_resolvers=settings['doi_resolvers'],
Noémi Ványi's avatar
Noémi Ványi committed
661
                  current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
Noemi Vanyi's avatar
Noemi Vanyi committed
662
                  allowed_plugins=allowed_plugins,
663
                  theme=get_current_theme_name(),
664
665
                  preferences_url_params=request.preferences.get_as_url_params(),
                  base_url=get_base_url(),
666
                  preferences=True)
asciimoo's avatar
asciimoo committed
667
668


Adam Tauber's avatar
Adam Tauber committed
669
670
@app.route('/image_proxy', methods=['GET'])
def image_proxy():
Nicolas Gelot's avatar
Nicolas Gelot committed
671
    url = request.args.get('url')
Adam Tauber's avatar
Adam Tauber committed
672
673
674
675

    if not url:
        return '', 400

Noémi Ványi's avatar
Noémi Ványi committed
676
    h = new_hmac(settings['server']['secret_key'], url)
677
678
679
680
681
682
683

    if h != request.args.get('h'):
        return '', 400

    headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
    headers['User-Agent'] = gen_useragent()

684
685
    resp = requests.get(url,
                        stream=True,
686
                        timeout=settings['outgoing']['request_timeout'],
687
688
                        headers=headers,
                        proxies=outgoing_proxies)
689
690
691

    if resp.status_code == 304:
        return '', resp.status_code
Adam Tauber's avatar
Adam Tauber committed
692
693
694
695
696
697
698
699

    if resp.status_code != 200:
        logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
        if resp.status_code >= 400:
            return '', resp.status_code
        return '', 400

    if not resp.headers.get('content-type', '').startswith('image/'):
Adam Tauber's avatar
Adam Tauber committed
700
        logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
Adam Tauber's avatar
Adam Tauber committed
701
702
        return '', 400

Noémi Ványi's avatar
Noémi Ványi committed
703
    img = b''
Adam Tauber's avatar
Adam Tauber committed
704
705
    chunk_counter = 0

706
    for chunk in resp.iter_content(1024 * 1024):
Adam Tauber's avatar
Adam Tauber committed
707
708
709
710
711
        chunk_counter += 1
        if chunk_counter > 5:
            return '', 502  # Bad gateway - file is too big (>5M)
        img += chunk

712
713
714
    headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

    return Response(img, mimetype=resp.headers['content-type'], headers=headers)
Adam Tauber's avatar
Adam Tauber committed
715
716


asciimoo's avatar
asciimoo committed
717
718
@app.route('/stats', methods=['GET'])
def stats():
Matej Cotman's avatar
Matej Cotman committed
719
    """Render engine statistics page."""
asciimoo's avatar
asciimoo committed
720
    stats = get_engines_stats()
721
722
723
724
    return render(
        'stats.html',
        stats=stats,
    )
asciimoo's avatar
asciimoo committed
725

asciimoo's avatar
asciimoo committed
726

asciimoo's avatar
asciimoo committed
727
728
729
730
731
732
@app.route('/robots.txt', methods=['GET'])
def robots():
    return Response("""User-agent: *
Allow: /
Allow: /about
Disallow: /stats
asciimoo's avatar
asciimoo committed
733
Disallow: /preferences
734
Disallow: /*?*q=*
asciimoo's avatar
asciimoo committed
735
736
""", mimetype='text/plain')

asciimoo's avatar
asciimoo committed
737

asciimoo's avatar
asciimoo committed
738
739
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
asciimoo's avatar
asciimoo committed
740
    method = 'post'
Luc Didry's avatar
Luc Didry committed
741

Noemi Vanyi's avatar
Noemi Vanyi committed
742
    if request.preferences.get_value('method') == 'GET':
Luc Didry's avatar
Luc Didry committed
743
744
        method = 'get'

asciimoo's avatar
asciimoo committed
745
    # chrome/chromium only supports HTTP GET....
asciimoo's avatar
asciimoo committed
746
747
    if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
        method = 'get'
748
749

    ret = render('opensearch.xml',
750
                 opensearch_method=method,
751
                 host=get_base_url(),
752
753
                 urljoin=urljoin,
                 override_theme='__common__')
754

asciimoo's avatar
asciimoo committed
755
    resp = Response(response=ret,
asciimoo's avatar
asciimoo committed
756
                    status=200,
757
                    mimetype="text/xml")
asciimoo's avatar
asciimoo committed
758
759
    return resp

760

asciimoo's avatar
asciimoo committed
761
762
@app.route('/favicon.ico')
def favicon():
Matej Cotman's avatar
Matej Cotman committed
763
    return send_from_directory(os.path.join(app.root_path,
764
765
                                            static_path,
                                            'themes',
Matej Cotman's avatar
Matej Cotman committed
766
767
                                            get_current_theme_name(),
                                            'img'),
asciimoo's avatar
asciimoo committed
768
769
                               'favicon.png',
                               mimetype='image/vnd.microsoft.icon')
asciimoo's avatar
asciimoo committed
770
771


Adam Tauber's avatar
Adam Tauber committed
772
773
@app.route('/clear_cookies')
def clear_cookies():
774
    resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
Adam Tauber's avatar
Adam Tauber committed
775
776
777
778
779
    for cookie_name in request.cookies:
        resp.delete_cookie(cookie_name)
    return resp


780
781
782
783
784
@app.route('/config')
def config():
    return jsonify({'categories': categories.keys(),
                    'engines': [{'name': engine_name,
                                 'categories': engine.categories,
785
                                 'shortcut': engine.shortcut,
786
787
788
789
                                 'enabled': not engine.disabled,
                                 'paging': engine.paging,
                                 'language_support': engine.language_support,
                                 'supported_languages':
790
791
792
                                     engine.supported_languages.keys()
                                     if isinstance(engine.supported_languages, dict)
                                     else engine.supported_languages,
793
794
795
                                 'safesearch': engine.safesearch,
                                 'time_range_support': engine.time_range_support,
                                 'timeout': engine.timeout}
796
797
798
799
800
801
802
803
804
                                for engine_name, engine in engines.items()],
                    'plugins': [{'name': plugin.name,
                                 'enabled': plugin.default_on}
                                for plugin in plugins],
                    'instance_name': settings['general']['instance_name'],
                    'locales': settings['locales'],
                    'default_locale': settings['ui']['default_locale'],
                    'autocomplete': settings['search']['autocomplete'],
                    'safe_search': settings['search']['safe_search'],
805
                    'default_theme': settings['ui']['default_theme'],
806
                    'version': VERSION_STRING,
807
                    'doi_resolvers': [r for r in settings['doi_resolvers']],
808
809
                    'default_doi_resolver': settings['default_doi_resolver'],
                    })
810
811


Noemi Vanyi's avatar
Noemi Vanyi committed
812
813
@app.errorhandler(404)
def page_not_found(e):
814
    return render('404.html'), 404
Noemi Vanyi's avatar
Noemi Vanyi committed
815
816


Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
817
818
819
820
running = threading.Event()


def wait_updating(start_time):
Omer's avatar
Omer committed
821
    wait = settings['redis']['upgrade_history'] - int(time.time() - start_time)
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
822
823
824
825
826
827
828
829
    if wait > 0:
        running.wait(wait)


def update_results():
    start_time = time.time()
    x = 0
    while not running.is_set():
Omer's avatar
Omer committed
830
831
        host = settings['redis']['host']
        queries = get_twenty_queries(x, host)
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
832
833
834
        for query in queries:
            result_container = Search(query).search()
            searchData = search_database.get_search_data(query, result_container)
Omer's avatar
Omer committed
835
            search_database.update(searchData, host)
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
836
837
838
839
840
841
842
843
844
            if running.is_set():
                return
        x += 20
        if len(queries) < 20:
            x = 0
            wait_updating(start_time)
            start_time = time.time()


845
def run():
Adam Tauber's avatar
Adam Tauber committed
846
    logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
847
    threading.Thread(target=update_results, name='results_updater').start()
Nicolas Gelot's avatar
Nicolas Gelot committed
848
    print("engine server starting")
849
    app.run(
850
851
        debug=searx_debug,
        use_debugger=searx_debug,
852
        port=settings['server']['port'],
853
        host=settings['server']['bind_address'],
Omer's avatar
Omer committed
854
        threaded=True
855
    )
Nicolas Gelot's avatar
Nicolas Gelot committed
856
    print("wait for shutdown...")
Johnny Kalajdzic's avatar
Johnny Kalajdzic committed
857
    running.set()
858
859


860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
class ReverseProxyPathFix(object):
    '''Wrap the application in this middleware and configure the
    front-end server to add these headers, to let you quietly bind
    this to a URL other than / and to an HTTP scheme that is
    different than what is used locally.

    http://flask.pocoo.org/snippets/35/

    In nginx:
    location /myprefix {
        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Scheme $scheme;
        proxy_set_header X-Script-Name /myprefix;
        }

    :param app: the WSGI application
    '''
879

880
881
882
883
884
885
886
887
888