Merge branch 'google-images-fix'

#  Google (Web)

# SPDX-License-Identifier: AGPL-3.0-or-later

#

"""Google (Web)

# @website     https://www.google.com

# @provide-api yes (https://developers.google.com/custom-search/)

:website:     https://www.google.com

#

:provide-api: yes (https://developers.google.com/custom-search/)

# @using-api   no

:using-api:   not the offical, since it needs registration to another service

# @results     HTML

:results:     HTML

# @stable      no (HTML can change)

:stable:      no

# @parse       url, title, content, suggestion

:parse:       url, title, content, number_of_results, answer, suggestion, correction

import re

For detailed description of the *REST-full* API see: `Query Parameter

Definitions`_.

.. _Query Parameter Definitions:

   https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

"""

# pylint: disable=invalid-name, missing-function-docstring

from lxml import html

from flask_babel import gettext

from flask_babel import gettext

from lxml import html, etree

from searx.engines.xpath import extract_text

from searx.engines.xpath import extract_text, extract_url

from searx import logger

from searx import logger

from searx.url_utils import urlencode, urlparse, parse_qsl

from searx.url_utils import urlencode, urlparse

from searx.utils import match_language, eval_xpath

from searx.utils import match_language, eval_xpath

logger = logger.getChild('google engine')

logger = logger.getChild('google engine')

# engine dependent config

# engine dependent config

categories = ['general']

categories = ['general']

paging = True

paging = True

language_support = True

language_support = True

use_locale_domain = True

time_range_support = True

time_range_support = True

safesearch = True

supported_languages_url = 'https://www.google.com/preferences?#languages'

# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests

# based on https://en.wikipedia.org/wiki/List_of_Google_domains and tests

default_hostname = 'www.google.com'

google_domains = {

    'BG': 'google.bg',      # Bulgaria

country_to_hostname = {

    'CZ': 'google.cz',      # Czech Republic

    'BG': 'www.google.bg',  # Bulgaria

    'DE': 'google.de',      # Germany

    'CZ': 'www.google.cz',  # Czech Republic

    'DK': 'google.dk',      # Denmark

    'DE': 'www.google.de',  # Germany

    'AT': 'google.at',      # Austria

    'DK': 'www.google.dk',  # Denmark

    'CH': 'google.ch',      # Switzerland

    'AT': 'www.google.at',  # Austria

    'GR': 'google.gr',      # Greece

    'CH': 'www.google.ch',  # Switzerland

    'AU': 'google.com.au',  # Australia

    'GR': 'www.google.gr',  # Greece

    'CA': 'google.ca',      # Canada

    'AU': 'www.google.com.au',  # Australia

    'GB': 'google.co.uk',   # United Kingdom

    'CA': 'www.google.ca',  # Canada

    'ID': 'google.co.id',   # Indonesia

    'GB': 'www.google.co.uk',  # United Kingdom

    'IE': 'google.ie',      # Ireland

    'ID': 'www.google.co.id',  # Indonesia

    'IN': 'google.co.in',   # India

    'IE': 'www.google.ie',  # Ireland

    'MY': 'google.com.my',  # Malaysia

    'IN': 'www.google.co.in',  # India

    'NZ': 'google.co.nz',   # New Zealand

    'MY': 'www.google.com.my',  # Malaysia

    'PH': 'google.com.ph',  # Philippines

    'NZ': 'www.google.co.nz',  # New Zealand

    'SG': 'google.com.sg',  # Singapore

    'PH': 'www.google.com.ph',  # Philippines

    # 'US': 'google.us',    # United States, redirect to .com

    'SG': 'www.google.com.sg',  # Singapore

    'ZA': 'google.co.za',   # South Africa

    # 'US': 'www.google.us',  # United States, redirect to .com

    'AR': 'google.com.ar',  # Argentina

    'ZA': 'www.google.co.za',  # South Africa

    'CL': 'google.cl',      # Chile

    'AR': 'www.google.com.ar',  # Argentina

    'ES': 'google.es',      # Spain

    'CL': 'www.google.cl',  # Chile

    'MX': 'google.com.mx',  # Mexico

    'ES': 'www.google.es',  # Spain

    'EE': 'google.ee',      # Estonia

    'MX': 'www.google.com.mx',  # Mexico

    'FI': 'google.fi',      # Finland

    'EE': 'www.google.ee',  # Estonia

    'BE': 'google.be',      # Belgium

    'FI': 'www.google.fi',  # Finland

    'FR': 'google.fr',      # France

    'BE': 'www.google.be',  # Belgium

    'IL': 'google.co.il',   # Israel

    'FR': 'www.google.fr',  # France

    'HR': 'google.hr',      # Croatia

    'IL': 'www.google.co.il',  # Israel

    'HU': 'google.hu',      # Hungary

    'HR': 'www.google.hr',  # Croatia

    'IT': 'google.it',      # Italy

    'HU': 'www.google.hu',  # Hungary

    'JP': 'google.co.jp',   # Japan

    'IT': 'www.google.it',  # Italy

    'KR': 'google.co.kr',   # South Korea

    'JP': 'www.google.co.jp',  # Japan

    'LT': 'google.lt',      # Lithuania

    'KR': 'www.google.co.kr',  # South Korea

    'LV': 'google.lv',      # Latvia

    'LT': 'www.google.lt',  # Lithuania

    'NO': 'google.no',      # Norway

    'LV': 'www.google.lv',  # Latvia

    'NL': 'google.nl',      # Netherlands

    'NO': 'www.google.no',  # Norway

    'PL': 'google.pl',      # Poland

    'NL': 'www.google.nl',  # Netherlands

    'BR': 'google.com.br',  # Brazil

    'PL': 'www.google.pl',  # Poland

    'PT': 'google.pt',      # Portugal

    'BR': 'www.google.com.br',  # Brazil

    'RO': 'google.ro',      # Romania

    'PT': 'www.google.pt',  # Portugal

    'RU': 'google.ru',      # Russia

    'RO': 'www.google.ro',  # Romania

    'SK': 'google.sk',      # Slovakia

    'RU': 'www.google.ru',  # Russia

    'SI': 'google.si',      # Slovenia

    'SK': 'www.google.sk',  # Slovakia

    'SE': 'google.se',      # Sweden

    'SI': 'www.google.si',  # Slovenia

    'TH': 'google.co.th',   # Thailand

    'SE': 'www.google.se',  # Sweden

    'TR': 'google.com.tr',  # Turkey

    'TH': 'www.google.co.th',  # Thailand

    'UA': 'google.com.ua',  # Ukraine

    'TR': 'www.google.com.tr',  # Turkey

    # 'CN': 'google.cn',    # China, only from China ?

    'UA': 'www.google.com.ua',  # Ukraine

    'HK': 'google.com.hk',  # Hong Kong

    # 'CN': 'www.google.cn',  # China, only from China ?

    'TW': 'google.com.tw'   # Taiwan

    'HK': 'www.google.com.hk',  # Hong Kong

    'TW': 'www.google.com.tw'  # Taiwan

}

}

# osm

time_range_dict = {

url_map = 'https://www.openstreetmap.org/'\

    'day': 'd',

    + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

# search-url

search_path = '/search'

search_url = ('https://{hostname}' +

              search_path +

              '?{query}&start={offset}&gws_rd=cr&gbv=1&lr={lang}&hl={lang_short}&ei=x')

time_range_search = "&tbs=qdr:{range}"

time_range_dict = {'day': 'd',

    'week': 'w',

    'week': 'w',

    'month': 'm',

    'month': 'm',

                   'year': 'y'}

    'year': 'y'

}

# other URLs

# Filter results. 0: None, 1: Moderate, 2: Strict

map_hostname_start = 'maps.google.'

filter_mapping = {

maps_path = '/maps'

    0 : 'off',

redirect_path = '/url'

    1 : 'medium',

images_path = '/images'

    2 : 'high'

supported_languages_url = 'https://www.google.com/preferences?#languages'

}

# specific xpath variables

# specific xpath variables

results_xpath = '//div[contains(@class, "ZINbbc")]'

# ------------------------

url_xpath = './/div[@class="kCrYT"][1]/a/@href'

title_xpath = './/div[@class="kCrYT"][1]/a/div[1]'

# google results are grouped into <div class="g" ../>

content_xpath = './/div[@class="kCrYT"][2]//div[contains(@class, "BNeawe")]//div[contains(@class, "BNeawe")]'

results_xpath = '//div[@class="g"]'

suggestion_xpath = '//div[contains(@class, "ZINbbc")][last()]//div[@class="rVLSBd"]/a//div[contains(@class, "BNeawe")]'

spelling_suggestion_xpath = '//div[@id="scc"]//a'

# google *sections* are no usual *results*, we ignore them

g_section_with_header='./g-section-with-header'

# map : detail location

map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'

# the title is a h3 tag relative to the result group

map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span'

title_xpath = './/h3[1]'

map_website_url_xpath = 'h3[2]/a/@href'

map_website_title_xpath = 'h3[2]'

# in the result group there is <div class="r" ../> it's first child is a <a

# href=...> (on some results, the <a> is the first "descendant", not ""child")

# map : near the location

href_xpath = './/div[@class="r"]//a/@href'

map_near = 'table[@class="ts"]//tr'

map_near_title = './/h4'

# in the result group there is <div class="s" ../> containing he *content*

map_near_url = './/h4/a/@href'

content_xpath = './/div[@class="s"]'

map_near_phone = './/span[@class="nobr"]'

# images

images_xpath = './/div/a'

image_url_xpath = './@href'

image_img_src_xpath = './img/@src'

# property names

# FIXME : no translation

property_address = "Address"

property_phone = "Phone number"

# remove google-specific tracking-url

def parse_url(url_string, google_hostname):

    # sanity check

    if url_string is None:

        return url_string

    # normal case

    parsed_url = urlparse(url_string)

    if (parsed_url.netloc in [google_hostname, '']

            and parsed_url.path == redirect_path):

        query = dict(parse_qsl(parsed_url.query))

        return query['q']

    else:

        return url_string

# Suggestions are links placed in a *card-section*, we extract only the text

# from the links not the links itself.

suggestion_xpath = '//div[contains(@class, "card-section")]//a'

# Since google does *auto-correction* on the first query these are not really

# *spelling suggestions*, we use them anyway.

spelling_suggestion_xpath = '//div[@class="med"]/p/a'

# returns extract_text on the first result selected by the xpath or None

def extract_text_from_dom(result, xpath):

def extract_text_from_dom(result, xpath):

    """returns extract_text on the first result selected by the xpath or None"""

    r = eval_xpath(result, xpath)

    r = eval_xpath(result, xpath)

    if len(r) > 0:

    if len(r) > 0:

        return extract_text(r[0])

        return extract_text(r[0])

    return None

    return None

def get_lang_country(params, lang_list, custom_aliases):

# do search-request

    """Returns a tuple with *langauage* on its first and *country* on its second

def request(query, params):

    position."""

    offset = (params['pageno'] - 1) * 10

    language = params['language']

    if language == 'all':

    if params['language'] == 'all' or params['language'] == 'en-US':

        language = 'en-US'

        language = 'en-GB'

    else:

        language = match_language(params['language'], supported_languages, language_aliases)

    language_array = language.split('-')

    language_array = language.split('-')

    if params['language'].find('-') > 0:

        country = params['language'].split('-')[1]

    if len(language_array) == 2:

    elif len(language_array) == 2:

        country = language_array[1]

        country = language_array[1]

    else:

    else:

        country = 'US'

        country = language_array[0].upper()

    url_lang = 'lang_' + language

    language = match_language(language, lang_list, custom_aliases)

    lang_country = '%s-%s' % (language, country)

    if lang_country == 'en-EN':

        lang_country = 'en'

    if use_locale_domain:

    return language, country, lang_country

        google_hostname = country_to_hostname.get(country.upper(), default_hostname)

    else:

        google_hostname = default_hostname

    # original format: ID=3e2b6616cee08557:TM=5556667580:C=r:IP=4.1.12.5-:S=23ASdf0soFgF2d34dfgf-_22JJOmHdfgg

    params['cookies']['GOOGLE_ABUSE_EXEMPTION'] = 'x'

    params['url'] = search_url.format(offset=offset,

                                      query=urlencode({'q': query}),

                                      hostname=google_hostname,

                                      lang=url_lang,

                                      lang_short=language)

    if params['time_range'] in time_range_dict:

        params['url'] += time_range_search.format(range=time_range_dict[params['time_range']])

    params['headers']['Accept-Language'] = language + ',' + language + '-' + country

def request(query, params):

    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

    """Google search request"""

    params['google_hostname'] = google_hostname

    offset = (params['pageno'] - 1) * 10

    language, country, lang_country = get_lang_country(

        # pylint: disable=undefined-variable

        params, supported_languages, language_aliases

    )

    subdomain = 'www.' + google_domains.get(country.upper(), 'google.com')

    # https://www.google.de/search?q=corona&hl=de-DE&lr=lang_de&start=0&tbs=qdr%3Ad&safe=medium

    query_url = 'https://'+ subdomain + '/search' + "?" + urlencode({

        'q': query,

        'hl': lang_country,

        'lr': "lang_" + language,

        'ie': "utf8",

        'oe': "utf8",

        'start': offset,

    })

    return params

    if params['time_range'] in time_range_dict:

        query_url += '&' + urlencode({'tbs': 'qdr:' + time_range_dict[params['time_range']]})

    if params['safesearch']:

        query_url += '&' + urlencode({'safe': filter_mapping[params['safesearch']]})

    params['url'] = query_url

    logger.debug("query_url --> %s", query_url)

    # en-US,en;q=0.8,en;q=0.5

    params['headers']['Accept-Language'] = (

        lang_country + ',' + language + ';q=0.8,' + language + ';q=0.5'

        )

    logger.debug("HTTP header Accept-Language --> %s",

                 params['headers']['Accept-Language'])

    params['headers']['Accept'] = (

        'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'

        )

    #params['google_subdomain'] = subdomain

    return params

# get response from search-request

def response(resp):

def response(resp):

    """Get response from google's search request"""

    results = []

    results = []

    # detect google sorry

    # detect google sorry

    if resp_url.path.startswith('/sorry'):

    if resp_url.path.startswith('/sorry'):

        raise RuntimeWarning(gettext('CAPTCHA required'))

        raise RuntimeWarning(gettext('CAPTCHA required'))

    # which hostname ?

    # which subdomain ?

    google_hostname = resp.search_params.get('google_hostname')

    # subdomain = resp.search_params.get('google_subdomain')

    google_url = "https://" + google_hostname

    # convert the text to dom

    # convert the text to dom

    dom = html.fromstring(resp.text)

    dom = html.fromstring(resp.text)

    instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')

    # results --> answer

    if instant_answer:

    answer = eval_xpath(dom, '//div[contains(@class, "LGOjhe")]//text()')

        results.append({'answer': u' '.join(instant_answer)})

    if answer:

        results.append({'answer': ' '.join(answer)})

    else:

        logger.debug("did not found 'answer'")

    # results --> number_of_results

    try:

    try:

        results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]

        _txt = eval_xpath(dom, '//div[@id="result-stats"]//text()')[0]

                          .split()[1].replace(',', ''))

        _digit = ''.join([n for n in _txt if n.isdigit()])

        results.append({'number_of_results': results_num})

        number_of_results = int(_digit)

    except:

        results.append({'number_of_results': number_of_results})

        pass

    except Exception as e:  # pylint: disable=broad-except

        logger.debug("did not 'number_of_results'")

        logger.error(e, exc_info=True)

    # parse results

    # parse results

    for result in eval_xpath(dom, results_xpath):

    for result in eval_xpath(dom, results_xpath):

        try:

            title = extract_text(eval_xpath(result, title_xpath)[0])

            url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)

            parsed_url = urlparse(url, google_hostname)

            # map result

        # google *sections*

            if parsed_url.netloc == google_hostname:

        if extract_text(eval_xpath(result, g_section_with_header)):

                # TODO fix inside links

            logger.debug("ingoring <g-section-with-header>")

            continue

            continue

                # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):

                #     print "yooooo"*30

                #     x = eval_xpath(result, map_near)

                #     if len(x) > 0:

                #         # map : near the location

                #         results = results + parse_map_near(parsed_url, x, google_hostname)

                #     else:

                #         # map : detail about a location

                #         results = results + parse_map_detail(parsed_url, result, google_hostname)

                # # google news

                # elif parsed_url.path == search_path:

                #     # skipping news results

                #     pass

                # # images result

                # elif parsed_url.path == images_path:

                #     # only thumbnail image provided,

                #     # so skipping image results

                #     # results = results + parse_images(result, google_hostname)

                #     pass

            else:

        try:

                # normal result

            title = extract_text(eval_xpath(result, title_xpath)[0])

            url = eval_xpath(result, href_xpath)[0]

            content = extract_text_from_dom(result, content_xpath)

            content = extract_text_from_dom(result, content_xpath)

                if content is None:

            results.append({

                    continue

                'url':      url,

                # append result

                results.append({'url': url,

                'title':    title,

                'title':    title,

                'content':  content

                'content':  content

                })

                })

        except:

        except Exception as e:  # pylint: disable=broad-except

            logger.debug('result parse error in:\n%s', etree.tostring(result, pretty_print=True))

            logger.error(e, exc_info=True)

            #from lxml import etree

            #logger.debug(etree.tostring(result, pretty_print=True))

            #import pdb

            #pdb.set_trace()

            continue

            continue

    # parse suggestion

    # parse suggestion

    # return results

    # return results

    return results

    return results

def parse_images(result, google_hostname):

    results = []

    for image in eval_xpath(result, images_xpath):

        url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)

        img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])

        # append result

        results.append({'url': url,

                        'title': '',

                        'content': '',

                        'img_src': img_src,

                        'template': 'images.html'

                        })

    return results

def parse_map_near(parsed_url, x, google_hostname):

    results = []

    for result in x:

        title = extract_text_from_dom(result, map_near_title)

        url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname)

        attributes = []

        phone = extract_text_from_dom(result, map_near_phone)

        add_attributes(attributes, property_phone, phone, 'tel:' + phone)

        results.append({'title': title,

                        'url': url,

                        'content': attributes_to_html(attributes)

                        })

    return results

def parse_map_detail(parsed_url, result, google_hostname):

    results = []

    # try to parse the geoloc

    m = re.search(r'@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path)

    if m is None:

        m = re.search(r'll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query)

    if m is not None:

        # geoloc found (ignored)

        lon = float(m.group(2))  # noqa

        lat = float(m.group(1))  # noqa

        zoom = int(m.group(3))  # noqa

        # attributes

        attributes = []

        address = extract_text_from_dom(result, map_address_xpath)

        phone = extract_text_from_dom(result, map_phone_xpath)

        add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon))

        add_attributes(attributes, property_phone, phone, 'tel:' + phone)

        # title / content / url

        website_title = extract_text_from_dom(result, map_website_title_xpath)

        content = extract_text_from_dom(result, content_xpath)

        website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname)

        # add a result if there is a website

        if website_url is not None:

            results.append({'title': website_title,

                            'content': (content + '<br />' if content is not None else '')

                            + attributes_to_html(attributes),

                            'url': website_url

                            })

    return results

def add_attributes(attributes, name, value, url):

    if value is not None and len(value) > 0:

        attributes.append({'label': name, 'value': value, 'url': url})

def attributes_to_html(attributes):

    retval = '<table class="table table-striped">'

    for a in attributes:

        value = a.get('value')

        if 'url' in a:

            value = '<a href="' + a.get('url') + '">' + value + '</a>'

        retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>'

    retval = retval + '</table>'

    return retval

# get supported languages from their site

# get supported languages from their site

def _fetch_supported_languages(resp):

def _fetch_supported_languages(resp):

    supported_languages = {}

    ret_val = {}

    dom = html.fromstring(resp.text)

    dom = html.fromstring(resp.text)

    options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

    for option in options:

        code = eval_xpath(option, './@value')[0].split('_')[-1]

        name = eval_xpath(option, './@data-name')[0].title()

        supported_languages[code] = {"name": name}

    return supported_languages

    radio_buttons = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lang"]')

    for x in radio_buttons:

        name = x.get("data-name")