google.py 13.2 KB
Newer Older
Adam Tauber's avatar
Adam Tauber committed
1
2
#  Google (Web)
#
3
# @website     https://www.google.com
4
# @provide-api yes (https://developers.google.com/custom-search/)
Adam Tauber's avatar
Adam Tauber committed
5
#
6
7
8
9
# @using-api   no
# @results     HTML
# @stable      no (HTML can change)
# @parse       url, title, content, suggestion
asciimoo's avatar
asciimoo committed
10

11
import re
Adam Tauber's avatar
Adam Tauber committed
12
from flask_babel import gettext
13
from lxml import html, etree
14
from searx.engines.xpath import extract_text, extract_url
Adam Tauber's avatar
Adam Tauber committed
15
16
from searx import logger
from searx.url_utils import urlencode, urlparse, parse_qsl
17
from searx.utils import match_language
18
19

logger = logger.getChild('google engine')
asciimoo's avatar
asciimoo committed
20

21

22
# engine dependent config
asciimoo's avatar
asciimoo committed
23
categories = ['general']
24
25
paging = True
language_support = True
26
use_locale_domain = True
Noemi Vanyi's avatar
Noemi Vanyi committed
27
time_range_support = True
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests
default_hostname = 'www.google.com'

country_to_hostname = {
    'BG': 'www.google.bg',  # Bulgaria
    'CZ': 'www.google.cz',  # Czech Republic
    'DE': 'www.google.de',  # Germany
    'DK': 'www.google.dk',  # Denmark
    'AT': 'www.google.at',  # Austria
    'CH': 'www.google.ch',  # Switzerland
    'GR': 'www.google.gr',  # Greece
    'AU': 'www.google.com.au',  # Australia
    'CA': 'www.google.ca',  # Canada
    'GB': 'www.google.co.uk',  # United Kingdom
    'ID': 'www.google.co.id',  # Indonesia
    'IE': 'www.google.ie',  # Ireland
    'IN': 'www.google.co.in',  # India
    'MY': 'www.google.com.my',  # Malaysia
    'NZ': 'www.google.co.nz',  # New Zealand
    'PH': 'www.google.com.ph',  # Philippines
    'SG': 'www.google.com.sg',  # Singapore
50
    # 'US': 'www.google.us',  # United States, redirect to .com
51
52
53
    'ZA': 'www.google.co.za',  # South Africa
    'AR': 'www.google.com.ar',  # Argentina
    'CL': 'www.google.cl',  # Chile
54
    'ES': 'www.google.es',  # Spain
55
56
57
58
59
60
61
62
63
64
    'MX': 'www.google.com.mx',  # Mexico
    'EE': 'www.google.ee',  # Estonia
    'FI': 'www.google.fi',  # Finland
    'BE': 'www.google.be',  # Belgium
    'FR': 'www.google.fr',  # France
    'IL': 'www.google.co.il',  # Israel
    'HR': 'www.google.hr',  # Croatia
    'HU': 'www.google.hu',  # Hungary
    'IT': 'www.google.it',  # Italy
    'JP': 'www.google.co.jp',  # Japan
65
    'KR': 'www.google.co.kr',  # South Korea
66
67
68
69
70
71
72
73
74
75
    'LT': 'www.google.lt',  # Lithuania
    'LV': 'www.google.lv',  # Latvia
    'NO': 'www.google.no',  # Norway
    'NL': 'www.google.nl',  # Netherlands
    'PL': 'www.google.pl',  # Poland
    'BR': 'www.google.com.br',  # Brazil
    'PT': 'www.google.pt',  # Portugal
    'RO': 'www.google.ro',  # Romania
    'RU': 'www.google.ru',  # Russia
    'SK': 'www.google.sk',  # Slovakia
76
    'SI': 'www.google.si',  # Slovenia
77
78
79
    'SE': 'www.google.se',  # Sweden
    'TH': 'www.google.co.th',  # Thailand
    'TR': 'www.google.com.tr',  # Turkey
80
81
82
    'UA': 'www.google.com.ua',  # Ukraine
    # 'CN': 'www.google.cn',  # China, only from China ?
    'HK': 'www.google.com.hk',  # Hong Kong
83
84
    'TW': 'www.google.com.tw'  # Taiwan
}
asciimoo's avatar
asciimoo committed
85

86
87
88
89
# osm
url_map = 'https://www.openstreetmap.org/'\
    + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

90
# search-url
91
search_path = '/search'
92
93
search_url = ('https://{hostname}' + search_path + '?{query}'
              '&start={offset}&gws_rd=cr&gbv=1&lr={lang}&hl={lang_short}&ei=x')
asciimoo's avatar
asciimoo committed
94

Noemi Vanyi's avatar
Noemi Vanyi committed
95
96
97
time_range_search = "&tbs=qdr:{range}"
time_range_dict = {'day': 'd',
                   'week': 'w',
98
99
                   'month': 'm',
                   'year': 'y'}
Noemi Vanyi's avatar
Noemi Vanyi committed
100

101
102
103
104
105
# other URLs
map_hostname_start = 'maps.google.'
maps_path = '/maps'
redirect_path = '/url'
images_path = '/images'
106
supported_languages_url = 'https://www.google.com/preferences?#languages'
107

108
# specific xpath variables
109
results_xpath = '//div[@class="g"]'
110
111
112
url_xpath = './/h3/a/@href'
title_xpath = './/h3'
content_xpath = './/span[@class="st"]'
113
content_misc_xpath = './/div[@class="f slp"]'
114
suggestion_xpath = '//p[@class="_Bmc"]'
115
spelling_suggestion_xpath = '//a[@class="spell"]'
116

117
118
119
120
121
122
123
124
125
126
127
128
129
# map : detail location
map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span'
map_website_url_xpath = 'h3[2]/a/@href'
map_website_title_xpath = 'h3[2]'

# map : near the location
map_near = 'table[@class="ts"]//tr'
map_near_title = './/h4'
map_near_url = './/h4/a/@href'
map_near_phone = './/span[@class="nobr"]'

# images
130
131
132
133
images_xpath = './/div/a'
image_url_xpath = './@href'
image_img_src_xpath = './img/@src'

134
135
136
137
138
# property names
# FIXME : no translation
property_address = "Address"
property_phone = "Phone number"

139

140
# remove google-specific tracking-url
141
def parse_url(url_string, google_hostname):
142
143
144
145
146
    # sanity check
    if url_string is None:
        return url_string

    # normal case
147
    parsed_url = urlparse(url_string)
Adam Tauber's avatar
Adam Tauber committed
148
149
    if (parsed_url.netloc in [google_hostname, '']
            and parsed_url.path == redirect_path):
150
151
152
153
        query = dict(parse_qsl(parsed_url.query))
        return query['q']
    else:
        return url_string
asciimoo's avatar
asciimoo committed
154

Adam Tauber's avatar
Adam Tauber committed
155

156
157
158
159
# returns extract_text on the first result selected by the xpath or None
def extract_text_from_dom(result, xpath):
    r = result.xpath(xpath)
    if len(r) > 0:
160
        return extract_text(r[0])
161
162
163
    return None


164
# do search-request
asciimoo's avatar
asciimoo committed
165
def request(query, params):
166
    offset = (params['pageno'] - 1) * 10
167

168
169
170
171
172
173
174
175
176
177
    language = match_language(params['language'], supported_languages)
    language_array = language.split('-')
    if params['language'].find('-') > 0:
        country = params['language'].split('-')[1]
    elif len(language_array) == 2:
        country = language_array[1]
    else:
        country = 'US'

    url_lang = 'lang_' + language
178
179
180
181
182

    if use_locale_domain:
        google_hostname = country_to_hostname.get(country.upper(), default_hostname)
    else:
        google_hostname = default_hostname
183

184
185
    # original format: ID=3e2b6616cee08557:TM=5556667580:C=r:IP=4.1.12.5-:S=23ASdf0soFgF2d34dfgf-_22JJOmHdfgg
    params['cookies']['GOOGLE_ABUSE_EXEMPTION'] = 'x'
asciimoo's avatar
asciimoo committed
186
    params['url'] = search_url.format(offset=offset,
187
                                      query=urlencode({'q': query}),
188
                                      hostname=google_hostname,
189
190
                                      lang=url_lang,
                                      lang_short=language)
Adam Tauber's avatar
Adam Tauber committed
191
    if params['time_range'] in time_range_dict:
Noemi Vanyi's avatar
Noemi Vanyi committed
192
        params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])
193

194
    params['headers']['Accept-Language'] = language + ',' + language + '-' + country
195
196
197
    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

    params['google_hostname'] = google_hostname
198

asciimoo's avatar
asciimoo committed
199
200
201
    return params


202
# get response from search-request
asciimoo's avatar
asciimoo committed
203
204
def response(resp):
    results = []
205

206
207
208
209
210
    # detect google sorry
    resp_url = urlparse(resp.url)
    if resp_url.netloc == 'sorry.google.com' or resp_url.path == '/sorry/IndexRedirect':
        raise RuntimeWarning('sorry.google.com')

Adam Tauber's avatar
Adam Tauber committed
211
212
213
    if resp_url.path.startswith('/sorry'):
        raise RuntimeWarning(gettext('CAPTCHA required'))

214
215
216
217
218
    # which hostname ?
    google_hostname = resp.search_params.get('google_hostname')
    google_url = "https://" + google_hostname

    # convert the text to dom
219
    dom = html.fromstring(resp.text)
asciimoo's avatar
asciimoo committed
220

221
222
    instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
    if instant_answer:
Nicolas Gelot's avatar
Nicolas Gelot committed
223
        results.append({'answer': ' '.join(instant_answer)})
224
225
226
227
228
229
    try:
        results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
                          .split()[1].replace(',', ''))
        results.append({'number_of_results': results_num})
    except:
        pass
230

231
    # parse results
232
233
    for result in dom.xpath(results_xpath):
        try:
234
            title = extract_text(result.xpath(title_xpath)[0])
235
236
            url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
            parsed_url = urlparse(url, google_hostname)
237
238

            # map result
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
            if parsed_url.netloc == google_hostname:
                # TODO fix inside links
                continue
                # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
                #     print "yooooo"*30
                #     x = result.xpath(map_near)
                #     if len(x) > 0:
                #         # map : near the location
                #         results = results + parse_map_near(parsed_url, x, google_hostname)
                #     else:
                #         # map : detail about a location
                #         results = results + parse_map_detail(parsed_url, result, google_hostname)
                # # google news
                # elif parsed_url.path == search_path:
                #     # skipping news results
                #     pass

                # # images result
                # elif parsed_url.path == images_path:
                #     # only thumbnail image provided,
                #     # so skipping image results
                #     # results = results + parse_images(result, google_hostname)
                #     pass
262

263
264
            else:
                # normal result
265
266
267
268
269
270
                content = extract_text_from_dom(result, content_xpath)
                if content is None:
                    continue
                content_misc = extract_text_from_dom(result, content_misc_xpath)
                if content_misc is not None:
                    content = content_misc + "<br />" + content
271
                # append result
Adam Tauber's avatar
Adam Tauber committed
272
273
                results.append({'url': url,
                                'title': title,
Dalf's avatar
Dalf committed
274
275
276
                                'content': content
                                })
        except:
277
            logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))
278
279
280
281
282
            continue

    # parse suggestion
    for suggestion in dom.xpath(suggestion_xpath):
        # append suggestion
283
        results.append({'suggestion': extract_text(suggestion)})
284

285
286
287
    for correction in dom.xpath(spelling_suggestion_xpath):
        results.append({'correction': extract_text(correction)})

288
    # return results
asciimoo's avatar
asciimoo committed
289
    return results
290

Adam Tauber's avatar
Adam Tauber committed
291

292
def parse_images(result, google_hostname):
293
294
    results = []
    for image in result.xpath(images_xpath):
295
        url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
296
        img_src = extract_text(image.xpath(image_img_src_xpath)[0])
Adam Tauber's avatar
Adam Tauber committed
297

298
299
300
301
302
        # append result
        results.append({'url': url,
                        'title': '',
                        'content': '',
                        'img_src': img_src,
Dalf's avatar
Dalf committed
303
304
                        'template': 'images.html'
                        })
305
306

    return results
307
308
309
310
311
312
313
314


def parse_map_near(parsed_url, x, google_hostname):
    results = []

    for result in x:
        title = extract_text_from_dom(result, map_near_title)
        url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname)
Dalf's avatar
Dalf committed
315
        attributes = []
316
        phone = extract_text_from_dom(result, map_near_phone)
Dalf's avatar
Dalf committed
317
318
319
320
321
        add_attributes(attributes, property_phone, phone, 'tel:' + phone)
        results.append({'title': title,
                        'url': url,
                        'content': attributes_to_html(attributes)
                        })
322
323
324
325
326
327
328
329

    return results


def parse_map_detail(parsed_url, result, google_hostname):
    results = []

    # try to parse the geoloc
stepshal's avatar
stepshal committed
330
    m = re.search(r'@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path)
331
    if m is None:
stepshal's avatar
stepshal committed
332
        m = re.search(r'll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query)
333
334

    if m is not None:
Dalf's avatar
Dalf committed
335
336
337
338
        # geoloc found (ignored)
        lon = float(m.group(2))  # noqa
        lat = float(m.group(1))  # noqa
        zoom = int(m.group(3))  # noqa
339
340
341

        # attributes
        attributes = []
Dalf's avatar
Dalf committed
342
343
344
345
        address = extract_text_from_dom(result, map_address_xpath)
        phone = extract_text_from_dom(result, map_phone_xpath)
        add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon))
        add_attributes(attributes, property_phone, phone, 'tel:' + phone)
346
347
348
349
350
351

        # title / content / url
        website_title = extract_text_from_dom(result, map_website_title_xpath)
        content = extract_text_from_dom(result, content_xpath)
        website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname)

Dalf's avatar
Dalf committed
352
        # add a result if there is a website
353
        if website_url is not None:
Dalf's avatar
Dalf committed
354
355
356
357
            results.append({'title': website_title,
                            'content': (content + '<br />' if content is not None else '')
                            + attributes_to_html(attributes),
                            'url': website_url
358
359
360
361
362
                            })

    return results


Dalf's avatar
Dalf committed
363
def add_attributes(attributes, name, value, url):
364
    if value is not None and len(value) > 0:
Dalf's avatar
Dalf committed
365
        attributes.append({'label': name, 'value': value, 'url': url})
366
367
368
369
370


def attributes_to_html(attributes):
    retval = '<table class="table table-striped">'
    for a in attributes:
Dalf's avatar
Dalf committed
371
372
373
374
        value = a.get('value')
        if 'url' in a:
            value = '<a href="' + a.get('url') + '">' + value + '</a>'
        retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>'
375
376
    retval = retval + '</table>'
    return retval
377
378
379


# get supported languages from their site
380
def _fetch_supported_languages(resp):
381
    supported_languages = {}
382
383
    dom = html.fromstring(resp.text)
    options = dom.xpath('//table//td/font/label/span')
384
    for option in options:
385
386
        code = option.xpath('./@id')[0][1:]
        name = option.text.title()
387
388
389
        supported_languages[code] = {"name": name}

    return supported_languages