swisscows.py 3.7 KB
Newer Older
1
"""
2
 Swisscows (Web, Images)
3
4
5
6
7
8
9
10
11
12
13
14

 @website     https://swisscows.ch
 @provide-api no

 @using-api   no
 @results     HTML (using search portal)
 @stable      no (HTML can change)
 @parse       url, title, content
"""

from json import loads
import re
15
from lxml.html import fromstring
Adam Tauber's avatar
Adam Tauber committed
16
from searx.url_utils import unquote, urlencode
17
18

# engine dependent config
19
categories = ['general', 'images']
20
21
22
23
24
25
26
paging = True
language_support = True

# search-url
base_url = 'https://swisscows.ch/'
search_string = '?{query}&page={page}'

27
28
supported_languages_url = base_url

29
# regex
Adam Tauber's avatar
Adam Tauber committed
30
31
32
33
regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')
regex_json_remove_start = re.compile(b'^initialData:\s*')
regex_json_remove_end = re.compile(b',\s*environment$')
regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')
34
35
36
37


# do search-request
def request(query, params):
38
    if params['language'].split('-')[0] == 'no':
marc's avatar
marc committed
39
        region = 'nb-NO'
40
    else:
41
42
        region = params['language']
        ui_language = params['language'].split('-')[0]
43
44

    search_path = search_string.format(
Adam Tauber's avatar
Adam Tauber committed
45
46
47
        query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),
        page=params['pageno']
    )
48

49
50
51
52
    # image search query is something like 'image?{query}&page={page}'
    if params['category'] == 'images':
        search_path = 'image' + search_path

53
54
55
56
57
58
59
60
61
    params['url'] = base_url + search_path

    return params


# get response from search-request
def response(resp):
    results = []

Adam Tauber's avatar
Adam Tauber committed
62
    json_regex = regex_json.search(resp.text)
63
64
65
66
67

    # check if results are returned
    if not json_regex:
        return []

Adam Tauber's avatar
Adam Tauber committed
68
69
    json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))
    json = loads(json_raw.decode('utf-8'))
70

71
    # parse results
72
    for result in json['Results'].get('items', []):
73
74
75
76
        result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')

        # parse image results
        if result.get('ContentType', '').startswith('image'):
Adam Tauber's avatar
Adam Tauber committed
77
            img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
78
79
80

            # append result
            results.append({'url': result['SourceUrl'],
81
                            'title': result['Title'],
82
83
84
85
86
87
88
89
90
91
92
                            'content': '',
                            'img_src': img_url,
                            'template': 'images.html'})

        # parse general results
        else:
            result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
            result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')

            # append result
            results.append({'url': result_url,
93
94
                            'title': result_title,
                            'content': result_content})
95
96
97
98

    # parse images
    for result in json.get('Images', []):
        # decode image url
Adam Tauber's avatar
Adam Tauber committed
99
        img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))
100
101
102

        # append result
        results.append({'url': result['SourceUrl'],
103
                        'title': result['Title'],
104
105
106
107
108
109
                        'content': '',
                        'img_src': img_url,
                        'template': 'images.html'})

    # return results
    return results
110
111
112


# get supported languages from their site
113
def _fetch_supported_languages(resp):
114
    supported_languages = []
115
    dom = fromstring(resp.text)
116
117
    options = dom.xpath('//div[@id="regions-popup"]//ul/li/a')
    for option in options:
118
        code = option.xpath('./@data-search-language')[0]
119
120
        if code.startswith('nb-'):
            code = code.replace('nb', 'no', 1)
121
122
123
        supported_languages.append(code)

    return supported_languages