Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 46ca32c3 authored by Alexandre Flament's avatar Alexandre Flament
Browse files

[mod] update currencies.json and fetch_currencies.py 

use a sparql request on wikidata to get the list of currencies.

currencies.json contains the translation for all supported searx languages.

Supersede #993
parent 1be6ab2a

.github/workflows/data-update.yml

+1 −1
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ jobs: 
          python utils/fetch_languages.py

          python utils/fetch_ahmia_blacklist.py

          python utils/fetch_wikidata_units.py

          # python utils/fetch_currencies.py

          python utils/fetch_currencies.py



      - name: Create Pull Request

        id: cpr

searx/data/currencies.json

+14898 −7618

File changed.

Preview size limit exceeded, changes collapsed.

searx/search/processors/online_currency.py

+2 −0
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ def name_to_iso4217(name): 
    global CURRENCIES

    name = normalize_name(name)

    currency = CURRENCIES['names'].get(name, [name])

    if isinstance(currency, str):

        return currency

    return currency[0]

utils/fetch_currencies.py

+115 −127
Original line number Diff line number Diff line 
# -*- coding: utf-8 -*-

#!/usr/bin/env python





import json

import re

import unicodedata

import string

from urllib.parse import urlencode

from requests import get



languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}



url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)

url_wmflabs_template = 'http://wdq.wmflabs.org/api?q='

url_wikidata_search_template = 'http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'



wmflabs_queries = [

    'CLAIM[31:8142]',  # all devise

]

import json



db = {

    'iso4217': {

    },

    'names': {

# set path

from sys import path

from os.path import realpath, dirname, join

path.append(realpath(dirname(realpath(__file__)) + '/../'))



from searx import searx_dir, settings

from searx.engines.wikidata import send_wikidata_query





# ORDER BY (with all the query fields) is important to keep a deterministic result order

# so multiple invokation of this script doesn't change currencies.json

SARQL_REQUEST = """

SELECT DISTINCT ?iso4217 ?unit ?unicode ?label ?alias WHERE {

  ?item wdt:P498 ?iso4217; rdfs:label ?label.

  OPTIONAL { ?item skos:altLabel ?alias FILTER (LANG (?alias) = LANG(?label)). }

  OPTIONAL { ?item wdt:P5061 ?unit. }

  OPTIONAL { ?item wdt:P489 ?symbol.

             ?symbol wdt:P487 ?unicode. }

  MINUS { ?item wdt:P582 ?end_data . }                  # Ignore monney with an end date

  MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . }      # Ignore "former entity" (obsolete currency)

  FILTER(LANG(?label) IN (%LANGUAGES_SPARQL%)).

}

ORDER BY ?iso4217 ?unit ?unicode ?label ?alias

"""



# ORDER BY (with all the query fields) is important to keep a deterministic result order

# so multiple invokation of this script doesn't change currencies.json

SPARQL_WIKIPEDIA_NAMES_REQUEST = """

SELECT DISTINCT ?iso4217 ?article_name WHERE {

  ?item wdt:P498 ?iso4217 .

  ?article schema:about ?item ;

           schema:name ?article_name ;

           schema:isPartOf [ wikibase:wikiGroup "wikipedia" ]

  MINUS { ?item wdt:P582 ?end_data . }                  # Ignore monney with an end date

  MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . }      # Ignore "former entity" (obsolete currency)

  FILTER(LANG(?article_name) IN (%LANGUAGES_SPARQL%)).

}

ORDER BY ?iso4217 ?article_name

"""





def remove_accents(data):

    return unicodedata.normalize('NFKD', data).lower()





def normalize_name(name):

    return re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))

LANGUAGES = settings['locales'].keys()

LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))





def add_currency_name(name, iso4217):

    global db



    db_names = db['names']

def remove_accents(name):

    return unicodedata.normalize('NFKD', name).lower()



    if not isinstance(iso4217, str):

        print("problem", name, iso4217)

        return



    name = normalize_name(name)

def remove_extra(name):

    for c in ('(', ':'):

        if c in name:

            name = name.split(c)[0].strip()

    return name



    if name == '':

        print("name empty", iso4217)

        return



    iso4217_set = db_names.get(name, None)

    if iso4217_set is not None and iso4217 not in iso4217_set:

        db_names[name].append(iso4217)

    else:

        db_names[name] = [iso4217]

def _normalize_name(name):

    name = re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))

    name = remove_extra(name)

    return name





def add_currency_label(label, iso4217, language):

    global db



    db['iso4217'][iso4217] = db['iso4217'].get(iso4217, {})

    db['iso4217'][iso4217][language] = label





def get_property_value(data, name):

    prop = data.get('claims', {}).get(name, {})

    if len(prop) == 0:

        return None



    value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '')

    if value == '':

        return None



    return value





def parse_currency(data):

    iso4217 = get_property_value(data, 'P498')

def add_currency_name(db, name, iso4217, normalize_name=True):

    db_names = db['names']



    if iso4217 is not None:

        unit = get_property_value(data, 'P558')

        if unit is not None:

            add_currency_name(unit, iso4217)

    if normalize_name:

        name = _normalize_name(name)



        labels = data.get('labels', {})

        for language in languages:

            name = labels.get(language, {}).get('value', None)

            if name is not None:

                add_currency_name(name, iso4217)

                add_currency_label(name, iso4217, language)

    iso4217_set = db_names.setdefault(name, [])

    if iso4217 not in iso4217_set:

        iso4217_set.insert(0, iso4217)



        aliases = data.get('aliases', {})

        for language in aliases:

            for i in range(0, len(aliases[language])):

                alias = aliases[language][i].get('value', None)

                add_currency_name(alias, iso4217)



def add_currency_label(db, label, iso4217, language):

    labels = db['iso4217'].setdefault(iso4217, {})

    labels[language] = label



def fetch_data(wikidata_ids):

    url = url_template.format(query=urlencode({'ids': '|'.join(wikidata_ids)}))

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    entities = jsonresponse.get('entities', {})



    for pname in entities:

        pvalue = entities.get(pname)

        parse_currency(pvalue)

def wikidata_request_result_iterator(request):

    result = send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))

    if result is not None:

        for r in result['results']['bindings']:

            yield r





def add_q(i):

    return "Q" + str(i)

def fetch_db():

    db = {

        'names': {},

        'iso4217': {},

    }



    for r in wikidata_request_result_iterator(SPARQL_WIKIPEDIA_NAMES_REQUEST):

        iso4217 = r['iso4217']['value']

        article_name = r['article_name']['value']

        article_lang = r['article_name']['xml:lang']

        add_currency_name(db, article_name, iso4217)

        add_currency_label(db, article_name, iso4217, article_lang)



def fetch_data_batch(wikidata_ids):

    while len(wikidata_ids) > 0:

        if len(wikidata_ids) > 50:

            fetch_data(wikidata_ids[0:49])

            wikidata_ids = wikidata_ids[50:]

        else:

            fetch_data(wikidata_ids)

            wikidata_ids = []

    for r in wikidata_request_result_iterator(SARQL_REQUEST):

        iso4217 = r['iso4217']['value']

        if 'label' in r:

            label = r['label']['value']

            label_lang = r['label']['xml:lang']

            add_currency_name(db, label, iso4217)

            add_currency_label(db, label, iso4217, label_lang)



        if 'alias' in r:

            add_currency_name(db, r['alias']['value'], iso4217)



def wdq_query(query):

    url = url_wmflabs_template + query

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    qlist = list(map(add_q, jsonresponse.get('items', {})))

    error = jsonresponse.get('status', {}).get('error', None)

    if error is not None and error != 'OK':

        print("error for query '" + query + "' :" + error)

        if 'unicode' in r:

            add_currency_name(db, r['unicode']['value'], iso4217, normalize_name=False)



    fetch_data_batch(qlist)

        if 'unit' in r:

            add_currency_name(db, r['unit']['value'], iso4217, normalize_name=False)



    # reduce memory usage:

    # replace lists with one item by the item.

    # see searx.search.processors.online_currency.name_to_iso4217

    for name in db['names']:

        if len(db['names'][name]) == 1:

            db['names'][name] = db['names'][name][0]



def wd_query(query, offset=0):

    qlist = []

    return db



    url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    for r in jsonresponse.get('query', {}).get('search', {}):

        qlist.append(r.get('title', ''))

    fetch_data_batch(qlist)



def get_filename():

    return join(join(searx_dir, "data"), "currencies.json")



# fetch #

for q in wmflabs_queries:

    wdq_query(q)



def main():

    #

    db = fetch_db()

    # static

add_currency_name("euro", 'EUR')

add_currency_name("euros", 'EUR')

add_currency_name("dollar", 'USD')

add_currency_name("dollars", 'USD')

add_currency_name("peso", 'MXN')

add_currency_name("pesos", 'MXN')



# write

f = open("currencies.json", "wb")

json.dump(db, f, indent=4, encoding="utf-8")

f.close()
 
    add_currency_name(db, "euro", 'EUR')

    add_currency_name(db, "euros", 'EUR')

    add_currency_name(db, "dollar", 'USD')

    add_currency_name(db, "dollars", 'USD')

    add_currency_name(db, "peso", 'MXN')

    add_currency_name(db, "pesos", 'MXN')



    with open(get_filename(), 'w', encoding='utf8') as f:

        json.dump(db, f, ensure_ascii=False, indent=4)



if __name__ == '__main__':

    main()