google_news.py 2.19 KB
Newer Older
1
2
3
"""
 Google (News)

4
5
 @website     https://news.google.com
 @provide-api no
6

7
8
9
 @using-api   no
 @results     HTML
 @stable      no
10
11
 @parse       url, title, content, publishedDate
"""
Thomas Pointhuber's avatar
Thomas Pointhuber committed
12

13
from lxml import html
14
from searx.engines.google import _fetch_supported_languages, supported_languages_url
Adam Tauber's avatar
Adam Tauber committed
15
from searx.url_utils import urlencode
16
from searx.utils import match_language
Thomas Pointhuber's avatar
Thomas Pointhuber committed
17

18
# search-url
Thomas Pointhuber's avatar
Thomas Pointhuber committed
19
categories = ['news']
20
21
paging = True
language_support = True
22
23
24
safesearch = True
time_range_support = True
number_of_results = 10
Thomas Pointhuber's avatar
Thomas Pointhuber committed
25

26
27
28
29
30
31
32
33
search_url = 'https://www.google.com/search'\
    '?{query}'\
    '&tbm=nws'\
    '&gws_rd=cr'\
    '&{search_options}'
time_range_attr = "qdr:{range}"
time_range_dict = {'day': 'd',
                   'week': 'w',
34
35
                   'month': 'm',
                   'year': 'y'}
Thomas Pointhuber's avatar
Thomas Pointhuber committed
36
37


38
# do search-request
Thomas Pointhuber's avatar
Thomas Pointhuber committed
39
def request(query, params):
40

41
42
43
44
45
46
47
48
49
    search_options = {
        'start': (params['pageno'] - 1) * number_of_results
    }

    if params['time_range'] in time_range_dict:
        search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])

    if safesearch and params['safesearch']:
        search_options['safe'] = 'on'
50

51
52
53
    params['url'] = search_url.format(query=urlencode({'q': query}),
                                      search_options=urlencode(search_options))

54
    language = match_language(params['language'], supported_languages, language_aliases).split('-')[0]
55
56
    if language:
        params['url'] += '&lr=lang_' + language
57

Thomas Pointhuber's avatar
Thomas Pointhuber committed
58
59
60
    return params


61
# get response from search-request
Thomas Pointhuber's avatar
Thomas Pointhuber committed
62
63
def response(resp):
    results = []
64

65
    dom = html.fromstring(resp.text)
Thomas Pointhuber's avatar
Thomas Pointhuber committed
66

67
    # parse results
68
    for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):
69
70
        try:
            r = {
Marc Abonce Seguin's avatar
Marc Abonce Seguin committed
71
72
                'url': result.xpath('.//a[@class="l lLrAF"]')[0].attrib.get("href"),
                'title': ''.join(result.xpath('.//a[@class="l lLrAF"]//text()')),
73
74
75
76
                'content': ''.join(result.xpath('.//div[@class="st"]//text()')),
            }
        except:
            continue
77

78
79
80
        imgs = result.xpath('.//img/@src')
        if len(imgs) and not imgs[0].startswith('data'):
            r['img_src'] = imgs[0]
81
82

        results.append(r)
83
84

    # return results
Thomas Pointhuber's avatar
Thomas Pointhuber committed
85
    return results