Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f62ce21f authored by marc's avatar marc
Browse files

[mod] fetch supported languages for several engines 

utils/fetch_languages.py gets languages supported by each engine and
generates engines_languages.json with each engine's supported language.
parent 92c6e88a

searx/data/engines_languages.json

0 → 100644
+3256 −0

File added.

Preview size limit exceeded, changes collapsed.

searx/engines/__init__.py

+6 −0
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ from os.path import realpath, dirname 
import sys

from flask_babel import gettext

from operator import itemgetter

from json import loads

from searx import settings

from searx import logger

from searx.utils import load_module
@@ -78,6 +79,9 @@ def load_engine(engine_data): 
        if not hasattr(engine, arg_name):

            setattr(engine, arg_name, arg_value)



    if engine_data['name'] in languages:

        setattr(engine, 'supported_languages', languages[engine_data['name']])



    # checking required variables

    for engine_attr in dir(engine):

        if engine_attr.startswith('_'):
@@ -207,6 +211,8 @@ if 'engines' not in settings or not settings['engines']: 
    logger.error('No engines found. Edit your settings.yml')

    exit(2)



languages = loads(open(engine_dir + '/../data/engines_languages.json').read())



for engine_data in settings['engines']:

    engine = load_engine(engine_data)

    if engine is not None:

searx/engines/bing.py

+15 −0
Original line number Diff line number Diff line
@@ -15,12 +15,14 @@ 


from urllib import urlencode

from lxml import html

from requests import get

from searx.engines.xpath import extract_text



# engine dependent config

categories = ['general']

paging = True

language_support = True

supported_languages_url = 'https://www.bing.com/account/general'



# search-url

base_url = 'https://www.bing.com/'
@@ -81,3 +83,16 @@ def response(resp): 


    # return results

    return results





# get supported languages from their site

def fetch_supported_languages():

    supported_languages = []

    response = get(supported_languages_url)

    dom = html.fromstring(response.text)

    options = dom.xpath('//div[@id="limit-languages"]//input')

    for option in options:

        code = option.xpath('./@id')[0].replace('_', '-')

        supported_languages.append(code)



    return supported_languages

searx/engines/bing_images.py

+1 −1
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ from urllib import urlencode 
from lxml import html

from json import loads

import re

from searx.engines.bing import supported_languages

from searx.engines.bing import fetch_supported_languages



# engine dependent config

categories = ['images']

searx/engines/bing_news.py

+1 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ from datetime import datetime 
from dateutil import parser

from lxml import etree

from searx.utils import list_get

from searx.engines.bing import supported_languages

from searx.engines.bing import fetch_supported_languages



# engine dependent config

categories = ['news']