Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3035e140 authored by Adam Tauber's avatar Adam Tauber
Browse files

Merge pull request #320 from dalf/currency 

[enh] currency_convert engine : "1 dollars in euros"
parents c5135f69 5525625d

searx/data/currencies.json

0 → 100644
+7655 −0

File added.

Preview size limit exceeded, changes collapsed.

searx/engines/currency_convert.py

+47 −3
Original line number Diff line number Diff line 
from datetime import datetime

import re

import os

import json

import unicodedata





categories = []

url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'

weight = 100



parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)  # noqa

parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W*in?\W*([^\.]+)\W*$', re.I)  # noqa



db = 1





def normalize_name(name):

    name = name.lower().replace('-', ' ')

    name = re.sub(' +', ' ', name)

    return unicodedata.normalize('NFKD', u"" + name).lower()





def name_to_iso4217(name):

    global db



    name = normalize_name(name)

    currencies = db['names'].get(name, [name])

    return currencies[0]





def iso4217_to_name(iso4217, language):

    global db



    return db['iso4217'].get(iso4217, {}).get(language, iso4217)





def request(query, params):
@@ -16,6 +42,8 @@ def request(query, params): 


    ammount, from_currency, to_currency = m.groups()

    ammount = float(ammount)

    from_currency = name_to_iso4217(from_currency.strip())

    to_currency = name_to_iso4217(to_currency.strip())



    q = (from_currency + to_currency).upper()
@@ -23,6 +51,8 @@ def request(query, params): 
    params['ammount'] = ammount

    params['from'] = from_currency

    params['to'] = to_currency

    params['from_name'] = iso4217_to_name(from_currency, 'en')

    params['to_name'] = iso4217_to_name(to_currency, 'en')



    return params
@@ -35,12 +65,14 @@ def response(resp): 
    except:

        return results



    answer = '{0} {1} = {2} {3} (1 {1} = {4} {3})'.format(

    answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(

        resp.search_params['ammount'],

        resp.search_params['from'],

        resp.search_params['ammount'] * conversion_rate,

        resp.search_params['to'],

        conversion_rate

        conversion_rate,

        resp.search_params['from_name'],

        resp.search_params['to_name'],

    )



    now_date = datetime.now().strftime('%Y%m%d')
@@ -55,3 +87,15 @@ def response(resp): 
    results.append({'answer': answer, 'url': url})



    return results





def load():

    global db



    current_dir = os.path.dirname(os.path.realpath(__file__))

    json_data = open(current_dir + "/../data/currencies.json").read()



    db = json.loads(json_data)





load()

searx/tests/engines/test_currency_convert.py

+5 −3
Original line number Diff line number Diff line
@@ -27,9 +27,11 @@ class TestCurrencyConvertEngine(SearxTestCase): 


    def test_response(self):

        dicto = defaultdict(dict)

        dicto['ammount'] = 10

        dicto['ammount'] = float(10)

        dicto['from'] = "EUR"

        dicto['to'] = "USD"

        dicto['from_name'] = "euro"

        dicto['to_name'] = "United States dollar"

        response = mock.Mock(text='a,b,c,d', search_params=dicto)

        self.assertEqual(currency_convert.response(response), [])
@@ -38,7 +40,7 @@ class TestCurrencyConvertEngine(SearxTestCase): 
        results = currency_convert.response(response)

        self.assertEqual(type(results), list)

        self.assertEqual(len(results), 1)

        self.assertEqual(results[0]['answer'], '10 EUR = 5.0 USD (1 EUR = 0.5 USD)')

        self.assertEqual(results[0]['answer'], '10.0 EUR = 5.0 USD, 1 EUR (euro) = 0.5 USD (United States dollar)')

        now_date = datetime.now().strftime('%Y%m%d')

        self.assertEqual(results[0]['url'], 'https://finance.yahoo.com/currency/converter-results/' +

                                            now_date + '/10-eur-to-usd.html')
 
                                            now_date + '/10.0-eur-to-usd.html')

utils/fetch_currencies.py

0 → 100644
+161 −0
Original line number Diff line number Diff line 
# -*- coding: utf-8 -*-

import json

import re

import unicodedata

import string

from urllib import urlencode

from requests import get

 

languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}

 

url_template = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&{query}&props=labels%7Cdatatype%7Cclaims%7Caliases&languages=' + '|'.join(languages)

url_wmflabs_template = 'http://wdq.wmflabs.org/api?q=' 

url_wikidata_search_template='http://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'

 

wmflabs_queries = [ 

    'CLAIM[31:8142]', # all devise

]

 

db = {

    'iso4217' : {

        },

    'names' : {

        }

}





def remove_accents(data):

    return unicodedata.normalize('NFKD', data).lower()

        



def normalize_name(name):

    return re.sub(' +',' ', remove_accents(name.lower()).replace('-', ' '))





def add_currency_name(name, iso4217):

    global db



    db_names = db['names']





    if not isinstance(iso4217, basestring):

        print "problem", name, iso4217

        return



    name = normalize_name(name)



    if name == '':

        print "name empty", iso4217

        return



    iso4217_set = db_names.get(name, None)

    if iso4217_set is not None and iso4217 not in iso4217_set:

        db_names[name].append(iso4217)

    else:

        db_names[name] = [ iso4217 ]





def add_currency_label(label, iso4217, language):

    global db



    db['iso4217'][iso4217] = db['iso4217'].get(iso4217, {})

    db['iso4217'][iso4217][language] = label





def get_property_value(data, name):

    prop = data.get('claims', {}).get(name, {})

    if len(prop) == 0:

        return None

    

    value = prop[0].get('mainsnak', {}).get('datavalue', {}).get('value', '') 

    if value == '':

        return None



    return value

    



def parse_currency(data):

    iso4217 = get_property_value(data, 'P498')

        

    if iso4217 is not None:

        unit = get_property_value(data, 'P558')

        if unit is not None:

            add_currency_name(unit, iso4217)

                

        labels = data.get('labels', {})

        for language in languages:

            name = labels.get(language, {}).get('value', None)

            if name != None:

                add_currency_name(name, iso4217)

                add_currency_label(name, iso4217, language)



        aliases = data.get('aliases', {})

        for language in aliases:

            for i in range(0, len(aliases[language])):

                alias = aliases[language][i].get('value', None)

                add_currency_name(alias, iso4217)



 

def fetch_data(wikidata_ids):

    url = url_template.format(query=urlencode({'ids' : '|'.join(wikidata_ids)}))

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    entities = jsonresponse.get('entities', {})

 

    for pname in entities:

        pvalue = entities.get(pname)

        parse_currency(pvalue)

 

 

def add_q(i):

    return "Q" + str(i)

 

 

def fetch_data_batch(wikidata_ids):

    while len(wikidata_ids) > 0:

        if len(wikidata_ids) > 50:

            fetch_data(wikidata_ids[0:49])

            wikidata_ids = wikidata_ids[50:]

        else:

            fetch_data(wikidata_ids)

            wikidata_ids = []

    

 

def wdq_query(query):

    url = url_wmflabs_template + query

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    qlist = map(add_q, jsonresponse.get('items', {}))

    error = jsonresponse.get('status', {}).get('error', None)

    if error != None and error != 'OK':

        print "error for query '" + query + "' :" + error



    fetch_data_batch(qlist)

 

 

def wd_query(query, offset=0):

    qlist = []

 

    url = url_wikidata_search_template.format(query=urlencode({'srsearch': query, 'srlimit': 50, 'sroffset': offset}))

    htmlresponse = get(url)

    jsonresponse = json.loads(htmlresponse.content)

    for r in jsonresponse.get('query', {}).get('search', {}):

        qlist.append(r.get('title', ''))

    fetch_data_batch(qlist)

 

## fetch ##

for q in wmflabs_queries:

    wdq_query(q)



# static 

add_currency_name(u"euro", 'EUR')

add_currency_name(u"euros", 'EUR')

add_currency_name(u"dollar", 'USD')

add_currency_name(u"dollars", 'USD')

add_currency_name(u"peso", 'MXN')

add_currency_name(u"pesos", 'MXN')



# write

f = open("currencies.json", "wb")

json.dump(db, f, indent=4, encoding="utf-8")

f.close()