webapp.py 30.5 KB
Newer Older
asciimoo's avatar
asciimoo committed
1 2
#!/usr/bin/env python

asciimoo's avatar
asciimoo committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
'''
searx is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

searx is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with searx. If not, see < http://www.gnu.org/licenses/ >.

(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
'''

20
import hashlib
21 22 23
import hmac
import json
import os
24
import time
Adam Tauber's avatar
Adam Tauber committed
25

26
import requests
asciimoo's avatar
asciimoo committed
27

Nicolas Gelot's avatar
Nicolas Gelot committed
28
from searx import logger
29

30

31 32 33 34
from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.util import ClassNotFound
from pygments.formatters import HtmlFormatter
35

Nicolas Gelot's avatar
Nicolas Gelot committed
36
from html import escape
37
from datetime import datetime, timedelta
Nicolas Gelot's avatar
Nicolas Gelot committed
38
from werkzeug.middleware.proxy_fix import ProxyFix
Gabor Nagy's avatar
Gabor Nagy committed
39 40 41 42
from flask import (
    Flask, request, render_template, url_for, Response, make_response,
    redirect, send_from_directory
)
43
from flask_babel import Babel, gettext, format_date, format_decimal
44
from flask.json import jsonify
45
from searx import settings, searx_dir, searx_debug
Adam Tauber's avatar
Adam Tauber committed
46
from searx.exceptions import SearxParameterException
Gabor Nagy's avatar
Gabor Nagy committed
47
from searx.engines import (
Adam Tauber's avatar
Adam Tauber committed
48
    categories, engines, engine_shortcuts, get_engines_stats, initialize_engines
Gabor Nagy's avatar
Gabor Nagy committed
49
)
Matej Cotman's avatar
Matej Cotman committed
50
from searx.utils import (
51
    highlight_content, get_resources_directory,
52
    get_static_files, get_result_templates, get_themes, gen_useragent,
53
    dict_subset, prettify_url, match_language
Matej Cotman's avatar
Matej Cotman committed
54
)
Nicolas Gelot's avatar
Nicolas Gelot committed
55
from searx.version import VERSION_STRING, SEARX_VERSION, METADATA_VERSION
56
from searx.languages import language_codes as languages
Nicolas Gelot's avatar
Nicolas Gelot committed
57 58
from searx.search import Search
from searx.search_database import RedisCache
59
from searx.query import RawTextQuery
60
from searx.autocomplete import searx_bang, backends as autocomplete_backends
61
from searx.plugins import plugins
Noémi Ványi's avatar
Noémi Ványi committed
62
from searx.plugins.oa_doi_rewrite import get_doi_resolver
63
from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES
64
from searx.answerers import answerers
Adam Tauber's avatar
Adam Tauber committed
65
from searx.url_utils import urlencode, urlparse, urljoin
66
from searx.utils import new_hmac
67
import threading
asciimoo's avatar
asciimoo committed
68

Eig8phei's avatar
Eig8phei committed
69 70
# serve pages with HTTP/1.1
from werkzeug.serving import WSGIRequestHandler
71

72 73
logger = logger.getChild('webapp')

74
WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))
asciimoo's avatar
asciimoo committed
75

76 77 78 79
# about static
static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path'])
logger.debug('static directory is %s', static_path)
static_files = get_static_files(static_path)
80

81
# about templates
82
default_theme = settings['ui']['default_theme']
83 84
templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path'])
logger.debug('templates directory is %s', templates_path)
85
themes = get_themes(templates_path)
86 87 88 89 90 91 92
result_templates = get_result_templates(templates_path)
global_favicons = []
for indice, theme in enumerate(themes):
    global_favicons.append([])
    theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons')
    for (dirpath, dirnames, filenames) in os.walk(theme_img_path):
        global_favicons[indice].extend(filenames)
Matej Cotman's avatar
Matej Cotman committed
93

94
# Flask app
95 96
app = Flask(
    __name__,
Matej Cotman's avatar
Matej Cotman committed
97 98
    static_folder=static_path,
    template_folder=templates_path
99 100
)

101 102
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
103
app.secret_key = settings['server']['secret_key']
104

105
if not searx_debug \
106 107
        or os.environ.get("WERKZEUG_RUN_MAIN") == "true" \
        or os.environ.get('UWSGI_ORIGINAL_PROC_NAME') is not None:
108 109
    initialize_engines(settings['engines'])

110 111
babel = Babel(app)

Nicolas Gelot's avatar
Nicolas Gelot committed
112 113
search = Search(RedisCache) if settings["redis"]["enable"] else Search()

114
rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'glk', 'he',
115
               'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi']
116

117
# used when translating category names
118 119 120 121 122 123 124 125
_category_names = (gettext('files'),
                   gettext('general'),
                   gettext('music'),
                   gettext('social media'),
                   gettext('images'),
                   gettext('videos'),
                   gettext('it'),
                   gettext('news'),
126
                   gettext('map'),
Thomas Pointhuber's avatar
Thomas Pointhuber committed
127
                   gettext('science'))
128

129
outgoing_proxies = settings['outgoing'].get('proxies') or None
asciimoo's avatar
asciimoo committed
130 131


132 133
@babel.localeselector
def get_locale():
134 135 136 137 138 139 140 141
    locale = "en-US"

    for lang in request.headers.get("Accept-Language", locale).split(","):
        locale = match_language(lang, settings['locales'].keys(), fallback=None)
        if locale is not None:
            break

    logger.debug("default locale from browser info is `%s`", locale)
asciimoo's avatar
asciimoo committed
142

143 144
    if request.preferences.get_value('locale') != '':
        locale = request.preferences.get_value('locale')
asciimoo's avatar
asciimoo committed
145

146
    if 'locale' in request.form and request.form['locale'] in settings['locales']:
asciimoo's avatar
asciimoo committed
147 148
        locale = request.form['locale']

149 150 151
    if locale == 'zh_TW':
        locale = 'zh_Hant_TW'

152 153
    logger.debug("selected locale is `%s`", locale)

asciimoo's avatar
asciimoo committed
154
    return locale
155 156


157 158 159 160 161 162
# code-highlighter
@app.template_filter('code_highlighter')
def code_highlighter(codelines, language=None):
    if not language:
        language = 'text'

163 164 165
    try:
        # find lexer by programing language
        lexer = get_lexer_by_name(language, stripall=True)
166
    except ClassNotFound:
167
        # if lexer is not found, using default one
Adam Tauber's avatar
Adam Tauber committed
168
        logger.debug('highlighter cannot find lexer for {0}'.format(language))
169 170
        lexer = get_lexer_by_name('text', stripall=True)

171 172 173 174 175 176 177 178 179 180
    html_code = ''
    tmp_code = ''
    last_line = None

    # parse lines
    for line, code in codelines:
        if not last_line:
            line_code_start = line

        # new codeblock is detected
181 182
        if last_line is not None and \
                last_line + 1 != line:
183
            # highlight last codepart
184 185
            formatter = HtmlFormatter(linenos='inline',
                                      linenostart=line_code_start)
186
            html_code = html_code + highlight(tmp_code, lexer, formatter)
187

188 189 190 191 192 193
            # reset conditions for next codepart
            tmp_code = ''
            line_code_start = line

        # add codepart
        tmp_code += code + '\n'
194

195 196 197 198 199 200 201 202 203 204
        # update line
        last_line = line

    # highlight last codepart
    formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start)
    html_code = html_code + highlight(tmp_code, lexer, formatter)

    return html_code


Cqoicebordel's avatar
Cqoicebordel committed
205 206 207 208 209 210
# Extract domain from url
@app.template_filter('extract_domain')
def extract_domain(url):
    return urlparse(url)[1]


211
def get_base_url():
212 213
    if settings['server']['base_url']:
        hostname = settings['server']['base_url']
214 215 216 217 218 219 220 221
    else:
        scheme = 'http'
        if request.is_secure:
            scheme = 'https'
        hostname = url_for('index', _external=True, _scheme=scheme)
    return hostname


Matej Cotman's avatar
Matej Cotman committed
222 223 224 225 226 227 228 229
def get_current_theme_name(override=None):
    """Returns theme name.

    Checks in this order:
    1. override
    2. cookies
    3. settings"""

230
    if override and (override in themes or override == '__common__'):
Matej Cotman's avatar
Matej Cotman committed
231
        return override
Noemi Vanyi's avatar
Noemi Vanyi committed
232
    theme_name = request.args.get('theme', request.preferences.get_value('theme'))
Matej Cotman's avatar
Matej Cotman committed
233 234 235 236 237
    if theme_name not in themes:
        theme_name = default_theme
    return theme_name


238 239 240 241 242 243 244
def get_result_template(theme, template_name):
    themed_path = theme + '/result_templates/' + template_name
    if themed_path in result_templates:
        return themed_path
    return 'result_templates/' + template_name


Matej Cotman's avatar
Matej Cotman committed
245
def url_for_theme(endpoint, override_theme=None, **values):
246
    if endpoint == 'static' and values.get('filename'):
Matej Cotman's avatar
Matej Cotman committed
247
        theme_name = get_current_theme_name(override=override_theme)
248 249 250
        filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
        if filename_with_theme in static_files:
            values['filename'] = filename_with_theme
Matej Cotman's avatar
Matej Cotman committed
251 252 253
    return url_for(endpoint, **values)


254 255 256 257 258 259 260
def proxify(url):
    if url.startswith('//'):
        url = 'https:' + url

    if not settings.get('result_proxy'):
        return url

Nicolas Gelot's avatar
Nicolas Gelot committed
261
    url_params = dict(mortyurl=url)
262 263 264

    if settings['result_proxy'].get('key'):
        url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],
Nicolas Gelot's avatar
Nicolas Gelot committed
265
                                           url,
266
                                           hashlib.sha256).hexdigest()
267 268

    return '{0}?{1}'.format(settings['result_proxy']['url'],
269
                            urlencode(url_params))
270 271


Adam Tauber's avatar
Adam Tauber committed
272 273 274 275
def image_proxify(url):
    if url.startswith('//'):
        url = 'https:' + url

276
    if not request.preferences.get_value('image_proxy'):
Adam Tauber's avatar
Adam Tauber committed
277 278
        return url

Venca24's avatar
Venca24 committed
279 280 281
    if url.startswith('data:image/jpeg;base64,'):
        return url

282 283 284
    if settings.get('result_proxy'):
        return proxify(url)

Nicolas Gelot's avatar
Nicolas Gelot committed
285
    h = new_hmac(settings['server']['secret_key'], url)
dalf's avatar
dalf committed
286

Adam Tauber's avatar
Adam Tauber committed
287
    return '{0}?{1}'.format(url_for('image_proxy'),
Nicolas Gelot's avatar
Nicolas Gelot committed
288
                            urlencode(dict(url=url, h=h)))
Adam Tauber's avatar
Adam Tauber committed
289 290


Matej Cotman's avatar
Matej Cotman committed
291
def render(template_name, override_theme=None, **kwargs):
292
    disabled_engines = request.preferences.engines.get_disabled()
293

294 295 296
    enabled_categories = set(category for engine_name in engines
                             for category in engines[engine_name].categories
                             if (engine_name, category) not in disabled_engines)
297

Adam Tauber's avatar
Adam Tauber committed
298
    if 'categories' not in kwargs:
Adam Tauber's avatar
Adam Tauber committed
299
        kwargs['categories'] = ['general']
300 301 302 303 304
        kwargs["categories"].extend(
            x
            for x in sorted(categories.keys())
            if x != "general" and x in enabled_categories
        )
305

306 307 308 309 310 311
    if 'all_categories' not in kwargs:
        kwargs['all_categories'] = ['general']
        kwargs['all_categories'].extend(x for x in
                                        sorted(categories.keys())
                                        if x != 'general')

Adam Tauber's avatar
Adam Tauber committed
312
    if 'selected_categories' not in kwargs:
313
        kwargs['selected_categories'] = []
Nicolas Gelot's avatar
Nicolas Gelot committed
314
        for arg in request.form:
315 316 317 318
            if arg.startswith('category_'):
                c = arg.split('_', 1)[1]
                if c in categories:
                    kwargs['selected_categories'].append(c)
319

320
    if not kwargs['selected_categories']:
Noemi Vanyi's avatar
Noemi Vanyi committed
321
        cookie_categories = request.preferences.get_value('categories')
322
        for ccateg in cookie_categories:
323
            kwargs['selected_categories'].append(ccateg)
324

325 326
    if not kwargs['selected_categories']:
        kwargs['selected_categories'] = ['general']
327

Adam Tauber's avatar
Adam Tauber committed
328
    if 'autocomplete' not in kwargs:
329
        kwargs['autocomplete'] = request.preferences.get_value('autocomplete')
330

331 332 333
    locale = request.preferences.get_value('locale')

    if locale in rtl_locales and 'rtl' not in kwargs:
334 335
        kwargs['rtl'] = True

Nicolas Gelot's avatar
Nicolas Gelot committed
336 337
    kwargs['searx_version'] = SEARX_VERSION
    kwargs['metadata_version'] = METADATA_VERSION
338

Noemi Vanyi's avatar
Noemi Vanyi committed
339
    kwargs['method'] = request.preferences.get_value('method')
340

Noemi Vanyi's avatar
Noemi Vanyi committed
341
    kwargs['safesearch'] = str(request.preferences.get_value('safesearch'))
342

343
    kwargs['language_codes'] = languages
344
    if 'current_language' not in kwargs:
345 346
        kwargs['current_language'] = match_language(request.preferences.get_value('language'),
                                                    LANGUAGE_CODES,
347
                                                    fallback=locale)
348

Matej Cotman's avatar
Matej Cotman committed
349 350 351
    # override url_for function in templates
    kwargs['url_for'] = url_for_theme

Adam Tauber's avatar
Adam Tauber committed
352 353
    kwargs['image_proxify'] = image_proxify

354
    kwargs['proxify'] = proxify if settings.get('result_proxy', {}).get('url') else None
355

356 357
    kwargs['get_result_template'] = get_result_template

Matej Cotman's avatar
Matej Cotman committed
358
    kwargs['theme'] = get_current_theme_name(override=override_theme)
Adam Tauber's avatar
Adam Tauber committed
359

360
    kwargs['template_name'] = template_name
Matej Cotman's avatar
Matej Cotman committed
361

Cqoicebordel's avatar
Cqoicebordel committed
362 363
    kwargs['cookies'] = request.cookies

Adam Tauber's avatar
Adam Tauber committed
364 365
    kwargs['errors'] = request.errors

366 367
    kwargs['instance_name'] = settings['general']['instance_name']

368 369
    kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

Nicolas Gelot's avatar
Nicolas Gelot committed
370
    kwargs['unicode'] = str
Adam Tauber's avatar
Adam Tauber committed
371

372 373
    kwargs['preferences'] = request.preferences

374 375 376 377 378 379 380 381 382 383
    kwargs['scripts'] = set()
    for plugin in request.user_plugins:
        for script in plugin.js_dependencies:
            kwargs['scripts'].add(script)

    kwargs['styles'] = set()
    for plugin in request.user_plugins:
        for css in plugin.css_dependencies:
            kwargs['styles'].add(css)

Matej Cotman's avatar
Matej Cotman committed
384 385
    return render_template(
        '{}/{}'.format(kwargs['theme'], template_name), **kwargs)
asciimoo's avatar
asciimoo committed
386

387

Adam Tauber's avatar
Adam Tauber committed
388 389
@app.before_request
def pre_request():
Adam Tauber's avatar
Adam Tauber committed
390 391
    request.errors = []

Adam Tauber's avatar
Adam Tauber committed
392
    preferences = Preferences(themes, list(categories.keys()), engines, plugins)
393
    request.preferences = preferences
394
    try:
395
        preferences.parse_dict(request.cookies)
396
    except:
Adam Tauber's avatar
Adam Tauber committed
397
        request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
398

Adam Tauber's avatar
Adam Tauber committed
399
    # merge GET, POST vars
dalf's avatar
dalf committed
400
    # request.form
Adam Tauber's avatar
Adam Tauber committed
401
    request.form = dict(request.form.items())
Adam Tauber's avatar
Adam Tauber committed
402
    for k, v in request.args.items():
Adam Tauber's avatar
Adam Tauber committed
403 404
        if k not in request.form:
            request.form[k] = v
405 406 407 408 409 410

    if request.form.get('preferences'):
        preferences.parse_encoded_data(request.form['preferences'])
    else:
        try:
            preferences.parse_dict(request.form)
411
        except Exception:
412 413
            logger.exception('invalid settings')
            request.errors.append(gettext('Invalid settings'))
Adam Tauber's avatar
Adam Tauber committed
414

415 416 417 418 419 420 421
    # init search language and locale
    locale = get_locale()
    if not preferences.get_value("language"):
        preferences.parse_dict({"language": locale})
    if not preferences.get_value("locale"):
        preferences.parse_dict({"locale": locale})

dalf's avatar
dalf committed
422
    # request.user_plugins
Adam Tauber's avatar
Adam Tauber committed
423
    request.user_plugins = []
Noemi Vanyi's avatar
Noemi Vanyi committed
424 425
    allowed_plugins = preferences.plugins.get_enabled()
    disabled_plugins = preferences.plugins.get_disabled()
Adam Tauber's avatar
Adam Tauber committed
426
    for plugin in plugins:
427 428 429
        if (
            plugin.default_on and plugin.id not in disabled_plugins
        ) or plugin.id in allowed_plugins:
Adam Tauber's avatar
Adam Tauber committed
430 431
            request.user_plugins.append(plugin)

432

433 434 435 436
def config_results(results, query):
    for result in results:
        if 'content' in result and result['content']:
            result['content'] = highlight_content(escape(result['content'][:1024]), query)
Nicolas Gelot's avatar
Nicolas Gelot committed
437
        result['title'] = highlight_content(escape(result['title'] or ''), query)
438
        result['pretty_url'] = prettify_url(result['url'])
Adam Tauber's avatar
Adam Tauber committed
439

440 441 442 443 444 445 446
        if 'pubdate' in result:
            publishedDate = datetime.strptime(result['pubdate'], '%Y-%m-%d %H:%M:%S')
            if publishedDate >= datetime.now() - timedelta(days=1):
                timedifference = datetime.now() - publishedDate
                minutes = int((timedifference.seconds / 60) % 60)
                hours = int(timedifference.seconds / 60 / 60)
                if hours == 0:
Nicolas Gelot's avatar
Nicolas Gelot committed
447
                    result['publishedDate'] = gettext('{minutes} minute(s) ago').format(minutes=minutes)
448
                else:
Nicolas Gelot's avatar
Nicolas Gelot committed
449
                    result['publishedDate'] = gettext('{hours} hour(s), {minutes} minute(s) ago').format(
450 451 452
                        hours=hours, minutes=minutes)  # noqa
            else:
                result['publishedDate'] = format_date(publishedDate)
453

454

455 456 457 458 459 460
def index_error(exn, output):
    user_error = gettext("search error")
    if output == "json":
        return jsonify({"error": f"{user_error}: {exn}"})

    request.errors.append(user_error)
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
461
    return render('index.html', error_details=exn)
462 463


464
@app.route('/search', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
465
@app.route('/', methods=['GET', 'POST'])
asciimoo's avatar
asciimoo committed
466
def index():
467 468 469
    # check the response format
    output = request.form.get("output", "html")

470
    # check if there is query
471
    if not request.form.get('q'):
472 473 474
        if output == 'json':
            return jsonify({}), 204
        return render('index.html')
asciimoo's avatar
asciimoo committed
475

Nicolas Gelot's avatar
Nicolas Gelot committed
476 477 478 479 480 481 482 483 484 485
    if request.form.get('category') is None:
        category = None
        for name, value in request.form.items():
            if name.startswith('category_'):
                category = name[9:]
                if category in categories and value == "on":
                    break
        if category is not None:
            request.form["category"] = category

Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
486 487 488 489
    selected_category = request.form.get('category') or 'general'
    first_page = request.form.get('pageno')
    is_general_first_page = selected_category == 'general' and (first_page is None or first_page == u'1')

Omer's avatar
Omer committed
490 491
    images = []
    videos = []
dalf's avatar
dalf committed
492
    # search
Omer's avatar
Omer committed
493
    search_data = None
494
    try:
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
495 496 497 498
        if is_general_first_page:
            request.form['categories'] = ['general', 'videos', 'images']
        else:
            request.form['categories'] = [selected_category]
499
        search_data = search(request)
500

501 502
    except Exception as e:
        # log exception
503
        logger.exception('search error')
asciimoo's avatar
asciimoo committed
504

505
        # is it an invalid input parameter or something else ?
Omer's avatar
Omer committed
506
        if issubclass(e.__class__, SearxParameterException):
507
            return index_error(e, output), 400
508
        else:
509
            return index_error(e, output), 500
510

Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
511
    if is_general_first_page:
512 513 514
        images = [r for r in search_data.results if r.get('category') == 'images'][:5]
        videos = [r for r in search_data.results if r.get('category') == 'videos'][:2]
        for res in list(search_data.results):
Nicolas Gelot's avatar
Nicolas Gelot committed
515
            if res.get('category') != 'general':
516
                search_data.results.remove(res)
517

dalf's avatar
dalf committed
518
    # output
519
    config_results(search_data.results, search_data.query)
Omer's avatar
Omer committed
520 521
    config_results(images, search_data.query)
    config_results(videos, search_data.query)
522

523
    response = dict(
524
        results=search_data.results,
Nicolas Gelot's avatar
Nicolas Gelot committed
525
        q=search_data.query,
Daniel J. Ramirez's avatar
Daniel J. Ramirez committed
526
        selected_category=selected_category,
Nicolas Gelot's avatar
Nicolas Gelot committed
527
        selected_categories=[selected_category],
Omer's avatar
Omer committed
528 529 530
        pageno=search_data.pageno,
        time_range=search_data.time_range,
        number_of_results=format_decimal(search_data.results_number),
531
        advanced_search=request.form.get('advanced_search', None),
532 533 534
        suggestions=list(search_data.suggestions),
        answers=list(search_data.answers),
        corrections=list(search_data.corrections),
Omer's avatar
Omer committed
535 536
        infoboxes=search_data.infoboxes,
        paging=search_data.paging,
537
        unresponsive_engines=list(search_data.unresponsive_engines),
Omer's avatar
Omer committed
538
        current_language=match_language(search_data.language,
539
                                        LANGUAGE_CODES,
540
                                        fallback=request.preferences.get_value("language")),
541
        image_results=images,
542
        videos_results=videos,
543
        base_url=get_base_url(),
544
        theme=get_current_theme_name(),
545
        favicons=global_favicons[themes.index(get_current_theme_name())]
546
    )
547 548 549
    if output == 'json':
        return jsonify(response)
    return render('results.html', **response)
asciimoo's avatar
asciimoo committed
550

asciimoo's avatar
asciimoo committed
551

asciimoo's avatar
asciimoo committed
552 553
@app.route('/about', methods=['GET'])
def about():
Matej Cotman's avatar
Matej Cotman committed
554
    """Render about page"""
555 556 557
    return render(
        'about.html',
    )
asciimoo's avatar
asciimoo committed
558 559


560 561 562 563 564 565 566 567
@app.route('/privacy', methods=['GET'])
def privacy():
    """Render privacy page"""
    return render(
        'privacy.html',
    )


568 569 570
@app.route('/autocompleter', methods=['GET', 'POST'])
def autocompleter():
    """Return autocompleter results"""
571
    # set blocked engines
572
    disabled_engines = request.preferences.engines.get_disabled()
573 574

    # parse query
Nicolas Gelot's avatar
Nicolas Gelot committed
575
    raw_text_query = RawTextQuery(request.form.get('q', ''), disabled_engines)
dalf's avatar
dalf committed
576
    raw_text_query.parse_query()
577

578
    # check if search query is set
dalf's avatar
dalf committed
579
    if not raw_text_query.getSearchQuery():
580
        return '', 400
581

Noemi Vanyi's avatar
Noemi Vanyi committed
582 583
    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))
584

585
    # parse searx specific autocompleter results like !bang
dalf's avatar
dalf committed
586
    raw_results = searx_bang(raw_text_query)
587

588 589
    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
a01200356's avatar
a01200356 committed
590
        # get language from cookie
591
        language = request.preferences.get_value('language')
592 593
        if not language or language == 'all':
            language = 'en'
a01200356's avatar
a01200356 committed
594
        else:
595
            language = language.split('-')[0]
596
        # run autocompletion
dalf's avatar
dalf committed
597
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))
598 599 600 601

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
dalf's avatar
dalf committed
602
        raw_text_query.changeSearchQuery(result)
603 604

        # add parsed result
dalf's avatar
dalf committed
605
        results.append(raw_text_query.getFullQuery())
606

607
    # return autocompleter results
608
    if request.form.get('format') == 'x-suggestions':
dalf's avatar
dalf committed
609
        return Response(json.dumps([raw_text_query.query, results]),
610
                        mimetype='application/json')
611 612 613

    return Response(json.dumps(results),
                    mimetype='application/json')
614 615


asciimoo's avatar
asciimoo committed
616 617
@app.route('/preferences', methods=['GET', 'POST'])
def preferences():
Noemi Vanyi's avatar
Noemi Vanyi committed
618
    """Render preferences page && save user preferences"""
asciimoo's avatar
asciimoo committed
619

Noemi Vanyi's avatar
Noemi Vanyi committed
620 621 622 623 624 625
    # save preferences
    if request.method == 'POST':
        resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
        try:
            request.preferences.parse_form(request.form)
        except ValidationException:
Adam Tauber's avatar
Adam Tauber committed
626
            request.errors.append(gettext('Invalid settings, please edit your preferences'))
Noemi Vanyi's avatar
Noemi Vanyi committed
627 628 629 630 631
            return resp
        return request.preferences.save(resp)

    # render preferences
    image_proxy = request.preferences.get_value('image_proxy')
632
    disabled_engines = request.preferences.engines.get_disabled()
Noemi Vanyi's avatar
Noemi Vanyi committed
633
    allowed_plugins = request.preferences.plugins.get_enabled()
634 635 636 637 638 639 640 641 642

    # stats for preferences page
    stats = {}

    for c in categories:
        for e in categories[c]:
            stats[e.name] = {'time': None,
                             'warn_timeout': False,
                             'warn_time': False}
643
            if e.timeout > settings['outgoing']['request_timeout']:
644
                stats[e.name]['warn_timeout'] = True
645
            stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)
646

647 648
    # get first element [0], the engine time,
    # and then the second element [1] : the time (the first one is the label)
649 650
    for engine_stat in get_engines_stats()[0][1]:
        stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
651
        if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
652 653 654
            stats[engine_stat.get('name')]['warn_time'] = True
    # end of stats

asciimoo's avatar
asciimoo committed
655 656
    return render('preferences.html',
                  locales=settings['locales'],
657
                  current_locale=request.preferences.get_value("locale"),
Adam Tauber's avatar
Adam Tauber committed
658
                  image_proxy=image_proxy,
659
                  engines_by_category=categories,
660
                  stats=stats,
661
                  answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
662
                  disabled_engines=disabled_engines,
663
                  autocomplete_backends=autocomplete_backends,
Matej Cotman's avatar
Matej Cotman committed
664 665
                  shortcuts={y: x for x, y in engine_shortcuts.items()},
                  themes=themes,
666
                  plugins=plugins,
667
                  doi_resolvers=settings['doi_resolvers'],
Noémi Ványi's avatar
Noémi Ványi committed
668
                  current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
Noemi Vanyi's avatar
Noemi Vanyi committed
669
                  allowed_plugins=allowed_plugins,
670
                  theme=get_current_theme_name(),
671 672
                  preferences_url_params=request.preferences.get_as_url_params(),
                  base_url=get_base_url(),
673
                  preferences=True)
asciimoo's avatar
asciimoo committed
674 675


676
def _is_selected_language_supported(engine, preferences):
677 678 679 680 681 682 683
    language = preferences.get_value("language")
    return language == "all" or match_language(
        language,
        getattr(engine, "supported_languages", []),
        getattr(engine, "language_aliases", {}),
        None,
    )
684 685


Adam Tauber's avatar
Adam Tauber committed
686 687
@app.route('/image_proxy', methods=['GET'])
def image_proxy():
Nicolas Gelot's avatar
Nicolas Gelot committed
688
    url = request.args.get('url')
Adam Tauber's avatar
Adam Tauber committed
689 690 691 692

    if not url:
        return '', 400

693
    h = new_hmac(settings['server']['secret_key'], url)
694 695 696 697 698 699 700

    if h != request.args.get('h'):
        return '', 400

    headers = dict_subset(request.headers, {'If-Modified-Since', 'If-None-Match'})
    headers['User-Agent'] = gen_useragent()

701 702
    resp = requests.get(url,
                        stream=True,
703
                        timeout=settings['outgoing']['request_timeout'],
704 705
                        headers=headers,
                        proxies=outgoing_proxies)
706 707 708

    if resp.status_code == 304:
        return '', resp.status_code
Adam Tauber's avatar
Adam Tauber committed
709 710 711 712 713 714 715 716

    if resp.status_code != 200:
        logger.debug('image-proxy: wrong response code: {0}'.format(resp.status_code))
        if resp.status_code >= 400:
            return '', resp.status_code
        return '', 400

    if not resp.headers.get('content-type', '').startswith('image/'):
Adam Tauber's avatar
Adam Tauber committed
717
        logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
Adam Tauber's avatar
Adam Tauber committed
718 719
        return '', 400

720
    img = b''
Adam Tauber's avatar
Adam Tauber committed
721 722
    chunk_counter = 0

723
    for chunk in resp.iter_content(1024 * 1024):
Adam Tauber's avatar
Adam Tauber committed
724 725 726 727 728
        chunk_counter += 1
        if chunk_counter > 5:
            return '', 502  # Bad gateway - file is too big (>5M)
        img += chunk

729 730 731
    headers = dict_subset(resp.headers, {'Content-Length', 'Length', 'Date', 'Last-Modified', 'Expires', 'Etag'})

    return Response(img, mimetype=resp.headers['content-type'], headers=headers)
Adam Tauber's avatar
Adam Tauber committed
732 733


asciimoo's avatar
asciimoo committed
734 735 736 737 738
@app.route('/robots.txt', methods=['GET'])
def robots():
    return Response("""User-agent: *
Allow: /
Allow: /about
asciimoo's avatar
asciimoo committed
739
Disallow: /preferences
740
Disallow: /*?*q=*
asciimoo's avatar
asciimoo committed
741 742
""", mimetype='text/plain')

asciimoo's avatar
asciimoo committed
743

asciimoo's avatar
asciimoo committed
744 745
@app.route('/opensearch.xml', methods=['GET'])
def opensearch():
asciimoo's avatar
asciimoo committed
746
    method = 'post'
Luc Didry's avatar
Luc Didry committed
747

Noemi Vanyi's avatar
Noemi Vanyi committed
748
    if request.preferences.get_value('method') == 'GET':
Luc Didry's avatar
Luc Didry committed
749 750
        method = 'get'

asciimoo's avatar
asciimoo committed
751
    # chrome/chromium only supports HTTP GET....
asciimoo's avatar
asciimoo committed
752 753
    if request.headers.get('User-Agent', '').lower().find('webkit') >= 0:
        method = 'get'
754 755

    ret = render('opensearch.xml',
756
                 opensearch_method=method,
757
                 host=get_base_url(),
758 759
                 urljoin=urljoin,
                 override_theme='__common__')
760

asciimoo's avatar
asciimoo committed
761
    resp = Response(response=ret,
asciimoo's avatar
asciimoo committed
762
                    status=200,
763
                    mimetype="text/xml")
asciimoo's avatar
asciimoo committed
764 765
    return resp

766

asciimoo's avatar
asciimoo committed
767 768
@app.route('/favicon.ico')
def favicon():
Matej Cotman's avatar
Matej Cotman committed
769
    return send_from_directory(os.path.join(app.root_path,
770 771
                                            static_path,
                                            'themes',
Matej Cotman's avatar
Matej Cotman committed
772 773
                                            get_current_theme_name(),
                                            'img'),
asciimoo's avatar
asciimoo committed
774 775
                               'favicon.png',
                               mimetype='image/vnd.microsoft.icon')
asciimoo's avatar
asciimoo committed
776 777


778 779
@app.route('/clear_cookies')
def clear_cookies():
780
    resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
781 782 783 784 785
    for cookie_name in request.cookies:
        resp.delete_cookie(cookie_name)
    return resp


786 787 788 789 790
@app.route('/config')
def config():
    return jsonify({'categories': categories.keys(),
                    'engines': [{'name': engine_name,
                                 'categories': engine.categories,
791
                                 'shortcut': engine.shortcut,
792 793 794 795
                                 'enabled': not engine.disabled,
                                 'paging': engine.paging,
                                 'language_support': engine.language_support,
                                 'supported_languages':
796 797 798
                                     engine.supported_languages.keys()
                                     if isinstance(engine.supported_languages, dict)
                                     else engine.supported_languages,
799 800 801
                                 'safesearch': engine.safesearch,
                                 'time_range_support': engine.time_range_support,
                                 'timeout': engine.timeout}
802 803 804 805 806 807 808 809 810
                                for engine_name, engine in engines.items()],
                    'plugins': [{'name': plugin.name,
                                 'enabled': plugin.default_on}
                                for plugin in plugins],
                    'instance_name': settings['general']['instance_name'],
                    'locales': settings['locales'],
                    'default_locale': settings['ui']['default_locale'],
                    'autocomplete': settings['search']['autocomplete'],
                    'safe_search': settings['search']['safe_search'],
811
                    'default_theme': settings['ui']['default_theme'],
812
                    'version': VERSION_STRING,
813
                    'doi_resolvers': [r for r in settings['doi_resolvers']],
814 815
                    'default_doi_resolver': settings['default_doi_resolver'],
                    })
816 817


Noemi Vanyi's avatar
Noemi Vanyi committed
818 819
@app.errorhandler(404)
def page_not_found(e):
820
    return render('404.html'), 404
Noemi Vanyi's avatar
Noemi Vanyi committed
821 822


823 824 825 826
running = threading.Event()


def wait_updating(start_time):
Omer's avatar
Omer committed
827
    wait = settings['redis']['upgrade_history'] - int(time.time() - start_time)
828 829 830 831 832 833 834 835
    if wait > 0:
        running.wait(wait)


def update_results():
    start_time = time.time()
    x = 0
    while not running.is_set():
Nicolas Gelot's avatar
Nicolas Gelot committed
836
        queries = search.cache.get_twenty_queries(x)
837
        for query in queries:
838
            result_container = search.search(query)
839
            searchData = search.create_search_data(query, result_container)
Nicolas Gelot's avatar
Nicolas Gelot committed
840
            search.cache.update(searchData)
841 842
            if running.is_set():
                return
Nicolas Gelot's avatar
Nicolas Gelot committed
843
        x += len(queries)
844 845 846 847 848 849
        if len(queries) < 20:
            x = 0
            wait_updating(start_time)
            start_time = time.time()


850
def run():
Adam Tauber's avatar
Adam Tauber committed
851
    logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])
852
    threading.Thread(target=update_results, name='results_updater').start()
Nicolas Gelot's avatar
Nicolas Gelot committed
853
    print("engine server starting")
854
    app.run(
855 856
        debug=searx_debug,
        use_debugger=searx_debug,
857
        port=settings['server']['port'],
858
        host=settings['server']['bind_address'],
859
        threaded=True
860
    )
Nicolas Gelot's avatar
Nicolas Gelot committed
861
    print("wait for shutdown...")
862
    running.set()
863 864


865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
class ReverseProxyPathFix(object):
    '''Wrap the application in this middleware and configure the
    front-end server to add these headers, to let you quietly bind
    this to a URL other than / and to an HTTP scheme that is
    different than what is used locally.

    http://flask.pocoo.org/snippets/35/

    In nginx:
    location /myprefix {
        proxy_pass http://127.0.0.1:8000;
        proxy_set_header Host $host;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Scheme $scheme;
        proxy_set_header X-Script-Name /myprefix;
        }

    :param app: the WSGI application
    '''
884

885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
    def __init__(self, app):
        self.app = app

    def __call__(self, environ, start_response):
        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
        if script_name:
            environ['SCRIPT_NAME'] = script_name
            path_info = environ['PATH_INFO']
            if path_info.startswith(script_name):
                environ['PATH_INFO'] = path_info[len(script_name):]

        scheme = environ.get('HTTP_X_SCHEME', '')
        if scheme:
            environ['wsgi.url_scheme'] = scheme
        return self.app(environ, start_response)
Martin Zimmermann's avatar
Martin Zimmermann committed
900

901

902 903 904
application = app
# patch app to handle non root url-s behind proxy & wsgi
app.wsgi_app = ReverseProxyPathFix(ProxyFix(application.wsgi_app))
Martin Zimmermann's avatar
Martin Zimmermann committed
905

906 907
if __name__ == "__main__":
    run()