Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Unverified Commit 9302d1fc authored by Markus Heiser's avatar Markus Heiser Committed by GitHub
Browse files

Merge branch 'master' into master

parents 3a26093c 0b7e7acf

requirements.txt

+8 −8
Original line number Diff line number Diff line 
certifi==2019.3.9

certifi==2020.4.5.1

babel==2.7.0

flask-babel==1.0.0

flask==1.0.2

idna==2.8

jinja2==2.10.1

lxml==4.3.3

flask==1.1.2

idna==2.9

jinja2==2.11.1

lxml==4.5.0

pygments==2.1.3

pyopenssl==19.0.0

pyopenssl==19.1.0

python-dateutil==2.8.0

pyyaml==5.1

requests[socks]==2.22.0
 
pyyaml==5.3.1

requests[socks]==2.23.0

searx/engines/startpage.py

+5 −2
Original line number Diff line number Diff line
@@ -99,11 +99,14 @@ def response(resp): 
        if re.match(r"^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content):

            date_pos = content.find('...') + 4

            date_string = content[0:date_pos - 5]

            published_date = parser.parse(date_string, dayfirst=True)



            # fix content string

            content = content[date_pos:]



            try:

                published_date = parser.parse(date_string, dayfirst=True)

            except ValueError:

                pass



        # check if search result starts with something like: "5 days ago ... "

        elif re.match(r"^[0-9]+ days? ago \.\.\. ", content):

            date_pos = content.find('...') + 4

searx/results.py

+2 −2
Original line number Diff line number Diff line
@@ -345,8 +345,8 @@ class ResultContainer(object): 
            return 0

        return resultnum_sum / len(self._number_of_results)



    def add_unresponsive_engine(self, engine_error):

        self.unresponsive_engines.add(engine_error)

    def add_unresponsive_engine(self, engine_name, error_type, error_message=None):

        self.unresponsive_engines.add((engine_name, error_type, error_message))



    def add_timing(self, engine_name, engine_time, page_load_time):

        self.timings.append({

searx/search.py

+5 −12
Original line number Diff line number Diff line
@@ -127,11 +127,7 @@ def search_one_offline_request_safe(engine_name, query, request_params, result_c 
        logger.exception('engine {0} : invalid input : {1}'.format(engine_name, e))

    except Exception as e:

        record_offline_engine_stats_on_error(engine, result_container, start_time)



        result_container.add_unresponsive_engine((

            engine_name,

            u'{0}: {1}'.format(gettext('unexpected crash'), e),

        ))

        result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))

        logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
@@ -186,24 +182,21 @@ def search_one_http_request_safe(engine_name, query, request_params, result_cont 
            engine.stats['errors'] += 1



        if (issubclass(e.__class__, requests.exceptions.Timeout)):

            result_container.add_unresponsive_engine((engine_name, gettext('timeout')))

            result_container.add_unresponsive_engine(engine_name, 'timeout')

            # requests timeout (connect or read)

            logger.error("engine {0} : HTTP requests timeout"

                         "(search duration : {1} s, timeout: {2} s) : {3}"

                         .format(engine_name, engine_time, timeout_limit, e.__class__.__name__))

            requests_exception = True

        elif (issubclass(e.__class__, requests.exceptions.RequestException)):

            result_container.add_unresponsive_engine((engine_name, gettext('request exception')))

            result_container.add_unresponsive_engine(engine_name, 'request exception')

            # other requests exception

            logger.exception("engine {0} : requests exception"

                             "(search duration : {1} s, timeout: {2} s) : {3}"

                             .format(engine_name, engine_time, timeout_limit, e))

            requests_exception = True

        else:

            result_container.add_unresponsive_engine((

                engine_name,

                u'{0}: {1}'.format(gettext('unexpected crash'), e),

            ))

            result_container.add_unresponsive_engine(engine_name, 'unexpected crash', str(e))

            # others errors

            logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
@@ -238,7 +231,7 @@ def search_multiple_requests(requests, result_container, start_time, timeout_lim 
            remaining_time = max(0.0, timeout_limit - (time() - start_time))

            th.join(remaining_time)

            if th.isAlive():

                result_container.add_unresponsive_engine((th._engine_name, gettext('timeout')))

                result_container.add_unresponsive_engine(th._engine_name, 'timeout')

                logger.warning('engine timeout: {0}'.format(th._engine_name))

searx/webapp.py

+17 −8
Original line number Diff line number Diff line
@@ -56,6 +56,7 @@ from flask import ( 
from babel.support import Translations

import flask_babel

from flask_babel import Babel, gettext, format_date, format_decimal

from flask.ctx import has_request_context

from flask.json import jsonify

from searx import brand

from searx import settings, searx_dir, searx_debug
@@ -165,13 +166,11 @@ _flask_babel_get_translations = flask_babel.get_translations 


# monkey patch for flask_babel.get_translations

def _get_translations():

    translation_locale = request.form.get('use-translation')

    if translation_locale:

    if has_request_context() and request.form.get('use-translation') == 'oc':

        babel_ext = flask_babel.current_app.extensions['babel']

        translation = Translations.load(next(babel_ext.translation_directories), 'oc')

    else:

        translation = _flask_babel_get_translations()

    return translation

        return Translations.load(next(babel_ext.translation_directories), 'oc')



    return _flask_babel_get_translations()





flask_babel.get_translations = _get_translations
@@ -627,7 +626,7 @@ def index(): 
                                    'corrections': list(result_container.corrections),

                                    'infoboxes': result_container.infoboxes,

                                    'suggestions': list(result_container.suggestions),

                                    'unresponsive_engines': list(result_container.unresponsive_engines)},

                                    'unresponsive_engines': __get_translated_errors(result_container.unresponsive_engines)},  # noqa

                                   default=lambda item: list(item) if isinstance(item, set) else item),

                        mimetype='application/json')

    elif output_format == 'csv':
@@ -695,7 +694,7 @@ def index(): 
        corrections=correction_urls,

        infoboxes=result_container.infoboxes,

        paging=result_container.paging,

        unresponsive_engines=result_container.unresponsive_engines,

        unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),

        current_language=match_language(search_query.lang,

                                        LANGUAGE_CODES,

                                        fallback=request.preferences.get_value("language")),
@@ -706,6 +705,16 @@ def index(): 
    )





def __get_translated_errors(unresponsive_engines):

    translated_errors = []

    for unresponsive_engine in unresponsive_engines:

        error_msg = gettext(unresponsive_engine[1])

        if unresponsive_engine[2]:

            error_msg = "{} {}".format(error_msg, unresponsive_engine[2])

        translated_errors.append((unresponsive_engine[0], error_msg))

    return translated_errors





@app.route('/about', methods=['GET'])

def about():

    """Render about page"""