Merge pull request #1856 from dalf/checker_requires_redis

.. admonition:: Tip for developers

   To set up a local redis instance using sockets simply use::

     $ ./manage redis.build

     $ sudo -H ./manage redis.install

     $ sudo -H ./manage redis.addgrp "${USER}"

     # don't forget to logout & login to get member of group

   The YAML setting for such a redis instance is:

   To set up a local redis instance, first set the socket path of the Redis DB

   in your YAML setting:

   .. code:: yaml

      redis:

        url: unix:///usr/local/searxng-redis/run/redis.sock?db=0

   Then use the following commands to install the redis instance ::

     $ ./manage redis.build

     $ sudo -H ./manage redis.install

     $ sudo -H ./manage redis.addgrp "${USER}"

     # don't forget to logout & login to get member of group

.. _settings outgoing:

    if not settings['server']['limiter']:

        return False

    logger.debug("init limiter DB")  # pylint: disable=undefined-variable

    if not redisdb.init():

        logger.error("init limiter DB failed!!!")  # pylint: disable=undefined-variable

    if not redisdb.client():

        logger.error("The limiter requires Redis")  # pylint: disable=undefined-variable

        return False

    app.before_request(pre_request)

from .impl import Checker

from .background import initialize, get_result

__all__ = ('Checker', 'initialize', 'get_result')

# SPDX-License-Identifier: AGPL-3.0-or-later

# lint: pylint

# pylint: disable=missing-module-docstring

# pyright: strict

# pyright: basic

import json

import random

import time

import threading

import os

import signal

from typing import Dict, Union, List, Any, Tuple

from typing import Dict, Union, List, Any, Tuple, Optional

from typing_extensions import TypedDict, Literal

import redis.exceptions

from searx import logger, settings, searx_debug

from searx.shared.redisdb import client as get_redis_client

from searx.exceptions import SearxSettingsException

from searx.search.processors import PROCESSORS

from searx.search.checker import Checker

from searx.shared import schedule, storage  # pyright: ignore

from searx.search.checker.scheduler import scheduler_function

CHECKER_RESULT = 'CHECKER_RESULT'

running = threading.Lock()

REDIS_RESULT_KEY = 'SearXNG_checker_result'

REDIS_LOCK_KEY = 'SearXNG_checker_lock'

CheckerResult = Union['CheckerOk', 'CheckerErr', 'CheckerOther']

    return (every[0], every[1])

def _get_every():

    every = settings.get('checker', {}).get('scheduling', {}).get('every', (300, 1800))

    return _get_interval(every, 'checker.scheduling.every is not a int or list')

def get_result() -> CheckerResult:

    serialized_result = storage.get_str(CHECKER_RESULT)

    if serialized_result is not None:

        return json.loads(serialized_result)

    client = get_redis_client()

    if client is None:

        # without Redis, the checker is disabled

        return {'status': 'disabled'}

    serialized_result: Optional[bytes] = client.get(REDIS_RESULT_KEY)

    if serialized_result is None:

        # the Redis key does not exist

        return {'status': 'unknown'}

    return json.loads(serialized_result)

def _set_result(result: CheckerResult):

    storage.set_str(CHECKER_RESULT, json.dumps(result))

    client = get_redis_client()

    if client is None:

        # without Redis, the function does nothing

        return

    client.set(REDIS_RESULT_KEY, json.dumps(result))

def _timestamp():

def run():

    if not running.acquire(blocking=False):  # pylint: disable=consider-using-with

        return

    try:

        # use a Redis lock to make sure there is no checker running at the same time

        # (this should not happen, this is a safety measure)

        with get_redis_client().lock(REDIS_LOCK_KEY, blocking_timeout=60, timeout=3600):

            logger.info('Starting checker')

            result: CheckerOk = {'status': 'ok', 'engines': {}, 'timestamp': _timestamp()}

            for name, processor in PROCESSORS.items():

            _set_result(result)

            logger.info('Check done')

    except redis.exceptions.LockError:

        _set_result({'status': 'error', 'timestamp': _timestamp()})

        logger.exception('Error while running the checker')

    except Exception:  # pylint: disable=broad-except

        _set_result({'status': 'error', 'timestamp': _timestamp()})

        logger.exception('Error while running the checker')

    finally:

        running.release()

def _run_with_delay():

    every = _get_every()

    delay = random.randint(0, every[1] - every[0])

    logger.debug('Start checker in %i seconds', delay)

    time.sleep(delay)

    run()

def _start_scheduling():

    every = _get_every()

    if schedule(every[0], _run_with_delay):

        run()

def _signal_handler(_signum: int, _frame: Any):

        logger.info('Send SIGUSR1 signal to pid %i to start the checker', os.getpid())

        signal.signal(signal.SIGUSR1, _signal_handler)

    # disabled by default

    _set_result({'status': 'disabled'})

    # special case when debug is activate

    if searx_debug and settings.get('checker', {}).get('off_when_debug', True):

    if searx_debug and settings['checker']['off_when_debug']:

        logger.info('debug mode: checker is disabled')

        return

    # check value of checker.scheduling.every now

    scheduling = settings.get('checker', {}).get('scheduling', None)

    scheduling = settings['checker']['scheduling']

    if scheduling is None or not scheduling:

        logger.info('Checker scheduler is disabled')

        return

    #

    _set_result({'status': 'unknown'})

    # make sure there is a Redis connection

    if get_redis_client() is None:

        logger.error('The checker requires Redis')

        return

    start_after = scheduling.get('start_after', (300, 1800))

    start_after = _get_interval(start_after, 'checker.scheduling.start_after is not a int or list')

    delay = random.randint(start_after[0], start_after[1])

    logger.info('Start checker in %i seconds', delay)

    t = threading.Timer(delay, _start_scheduling)

    # start the background scheduler

    every_range = _get_interval(scheduling.get('every', (300, 1800)), 'checker.scheduling.every is not a int or list')

    start_after_range = _get_interval(

        scheduling.get('start_after', (300, 1800)), 'checker.scheduling.start_after is not a int or list'

    )

    t = threading.Thread(

        target=scheduler_function,

        args=(start_after_range[0], start_after_range[1], every_range[0], every_range[1], run),

        name='checker_scheduler',

    )

    t.daemon = True

    t.start()

-- SPDX-License-Identifier: AGPL-3.0-or-later

--

-- This script is not a string in scheduler.py, so editors can provide syntax highlighting.

-- The Redis KEY is defined here and not in Python on purpose:

-- only this LUA script can read and update this key to avoid lock and concurrency issues.

local redis_key = 'SearXNG_checker_next_call_ts'

local now = redis.call('TIME')[1]

local start_after_from = ARGV[1]

local start_after_to = ARGV[2]

local every_from = ARGV[3]

local every_to = ARGV[4]

local next_call_ts = redis.call('GET', redis_key)

if (next_call_ts == false or next_call_ts == nil) then

    -- the scheduler has never run on this Redis instance, so:

    -- 1/ the scheduler does not run now

    -- 2/ the next call is a random time between start_after_from and start_after_to

    local initial_delay = math.random(start_after_from, start_after_to)

    redis.call('SET', redis_key, now + initial_delay)

    return { false, delay }

end

-- next_call_ts is defined

-- --> if now is lower than next_call_ts then we don't run the embedded checker

-- --> if now is higher then we update next_call_ts and ask to run the embedded checker now.

local call_now = next_call_ts <= now

if call_now then

    -- the checker runs now, define the timestamp of the next call:

    -- this is a random delay between every_from and every_to

    local periodic_delay = math.random(every_from, every_to)

    next_call_ts = redis.call('INCRBY', redis_key, periodic_delay)

end

return { call_now, next_call_ts - now }