Merge pull request #91 from pointhi/engines

## Yacy (Web, Images, Videos, Music, Files)

# 

# @website     http://yacy.net

# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)

# 

# @using-api   yes

# @results     JSON

# @stable      yes

# @parse       (general)    url, title, content, publishedDate

# @parse       (images)     url, title, img_src

#

# @todo        parse video, audio and file results

from json import loads

from urllib import urlencode

from dateutil import parser

# engine dependent config

categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'

paging = True

language_support = True

number_of_results = 5

# search-url

base_url = 'http://localhost:8090'

search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global'

url = 'http://localhost:8090'

search_url = '/yacysearch.json?{query}&maximumRecords=10'

# yacy specific type-definitions

search_types = {'general': 'text',

                'images': 'image',

                'files': 'app',               

                'music': 'audio',

                'videos': 'video'}

# do search-request

def request(query, params):

    params['url'] = url + search_url.format(query=urlencode({'query': query}))

    offset = (params['pageno'] - 1) * number_of_results

    search_type = search_types.get(params['category'], '0')

    params['url'] = base_url + search_url.format(query=urlencode({'query': query}),

                                                 offset=offset,

                                                 limit=number_of_results,

                                                 search_type=search_type)

    # add language tag if specified

    if params['language'] != 'all':

        params['url'] += '&lr=lang_' + params['language'].split('_')[0]

    return params

# get response from search-request

def response(resp):

    results = []

    raw_search_results = loads(resp.text)

    # return empty array if there are no results

    if not raw_search_results:

        return []

    search_results = raw_search_results.get('channels', {})[0].get('items', [])

    results = []

    if resp.search_params['category'] == 'general':

        # parse general results

        for result in search_results:

        tmp_result = {}

        tmp_result['title'] = result['title']

        tmp_result['url'] = result['link']

        tmp_result['content'] = ''

        if result['description']:

            tmp_result['content'] += result['description'] + "<br/>"

            publishedDate = parser.parse(result['pubDate'])

        if result['pubDate']:

            tmp_result['content'] += result['pubDate'] + "<br/>"

            # append result

            results.append({'url': result['link'],

                        'title': result['title'],

                        'content': result['description'],

                        'publishedDate': publishedDate})

        if result['size'] != '-1':

            tmp_result['content'] += result['sizename']

    elif resp.search_params['category'] == 'images':

        # parse image results

        for result in search_results:

            # append result

            results.append({'url': result['url'],

                        'title': result['title'],

                        'content': '',

                        'img_src': result['image'],

                        'template': 'images.html'})

        results.append(tmp_result)

    #TODO parse video, audio and file results

    # return results

    return results

    locale : en-US

    shortcut : vm

#  - name : yacy

#    engine : yacy

#    shortcut : ya

#    base_url : 'http://localhost:8090'

#    number_of_results : 5

#    timeout: 3.0

locales:

    en : English

    de : Deutsch