yacy.py 3.06 KB
Newer Older
1
# Yacy (Web, Images, Videos, Music, Files)
2
#
3
# @website     http://yacy.net
4
5
6
# @provide-api yes
#              (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
#
7
8
9
10
11
12
13
14
# @using-api   yes
# @results     JSON
# @stable      yes
# @parse       (general)    url, title, content, publishedDate
# @parse       (images)     url, title, img_src
#
# @todo        parse video, audio and file results

15
from json import loads
16
from dateutil import parser
Adam Tauber's avatar
Adam Tauber committed
17
from searx.url_utils import urlencode
18

Thomas Renard's avatar
Thomas Renard committed
19
20
from searx.utils import html_to_text

21
# engine dependent config
22
categories = ['general', 'images']  # TODO , 'music', 'videos', 'files'
23
24
25
26
27
28
paging = True
language_support = True
number_of_results = 5

# search-url
base_url = 'http://localhost:8090'
Thomas Pointhuber's avatar
Thomas Pointhuber committed
29
search_url = '/yacysearch.json?{query}'\
Cqoicebordel's avatar
Cqoicebordel committed
30
31
32
33
             '&startRecord={offset}'\
             '&maximumRecords={limit}'\
             '&contentdom={search_type}'\
             '&resource=global'
34

35
36
37
# yacy specific type-definitions
search_types = {'general': 'text',
                'images': 'image',
38
                'files': 'app',
39
40
                'music': 'audio',
                'videos': 'video'}
41

asciimoo's avatar
asciimoo committed
42

43
# do search-request
44
def request(query, params):
45
    offset = (params['pageno'] - 1) * number_of_results
Cqoicebordel's avatar
Cqoicebordel committed
46
    search_type = search_types.get(params.get('category'), '0')
47

Thomas Pointhuber's avatar
Thomas Pointhuber committed
48
49
50
51
52
    params['url'] = base_url +\
        search_url.format(query=urlencode({'query': query}),
                          offset=offset,
                          limit=number_of_results,
                          search_type=search_type)
53

54
55
56
    # add language tag if specified
    if params['language'] != 'all':
        params['url'] += '&lr=lang_' + params['language'].split('-')[0]
57

58
59
    return params

asciimoo's avatar
asciimoo committed
60

61
# get response from search-request
62
def response(resp):
63
64
    results = []

65
    raw_search_results = loads(resp.text)
66

67
    # return empty array if there are no results
asciimoo's avatar
asciimoo committed
68
    if not raw_search_results:
69
70
        return []

Cqoicebordel's avatar
Cqoicebordel committed
71
    search_results = raw_search_results.get('channels', [])
72

Cqoicebordel's avatar
Cqoicebordel committed
73
74
75
76
    if len(search_results) == 0:
        return []

    for result in search_results[0].get('items', []):
Thomas Pointhuber's avatar
Thomas Pointhuber committed
77
78
        # parse image results
        if result.get('image'):
Noémi Ványi's avatar
Noémi Ványi committed
79
80
81
82
83
84
85
86
87

            result_url = ''
            if 'url' in result:
                result_url = result['url']
            elif 'link' in result:
                result_url = result['link']
            else:
                continue

Thomas Pointhuber's avatar
Thomas Pointhuber committed
88
            # append result
Noémi Ványi's avatar
Noémi Ványi committed
89
            results.append({'url': result_url,
Thomas Pointhuber's avatar
Thomas Pointhuber committed
90
91
92
93
94
                            'title': result['title'],
                            'content': '',
                            'img_src': result['image'],
                            'template': 'images.html'})

95
        # parse general results
Thomas Pointhuber's avatar
Thomas Pointhuber committed
96
        else:
97
            publishedDate = parser.parse(result['pubDate'])
98

99
100
            # append result
            results.append({'url': result['link'],
Thomas Pointhuber's avatar
Thomas Pointhuber committed
101
                            'title': result['title'],
Thomas Renard's avatar
Thomas Renard committed
102
                            'content': html_to_text(result['description']),
Thomas Pointhuber's avatar
Thomas Pointhuber committed
103
                            'publishedDate': publishedDate})
104

Cqoicebordel's avatar
Cqoicebordel committed
105
        # TODO parse video, audio and file results
106

107
    # return results
108
    return results