Merge pull request #357 from asciimoo/google_engine

# @stable      no (HTML can change)

# @parse       url, title, content, suggestion

import re

from urllib import urlencode

from urlparse import urlparse, parse_qsl

from lxml import html

    'TW': 'www.google.com.tw'  # Taiwan

}

# osm

url_map = 'https://www.openstreetmap.org/'\

    + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

# search-url

search_path = '/search'

maps_path = '/maps/'

redirect_path = '/url'

images_path = '/images'

search_url = ('https://{hostname}' +

              search_path +

              '?{query}&start={offset}&gbv=1')

# other URLs

map_hostname_start = 'maps.google.'

maps_path = '/maps'

redirect_path = '/url'

images_path = '/images'

# specific xpath variables

results_xpath = '//li[@class="g"]'

url_xpath = './/h3/a/@href'

content_misc_xpath = './/div[@class="f slp"]'

suggestion_xpath = '//p[@class="_Bmc"]'

# map : detail location

map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'

map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span'

map_website_url_xpath = 'h3[2]/a/@href'

map_website_title_xpath = 'h3[2]'

# map : near the location

map_near = 'table[@class="ts"]//tr'

map_near_title = './/h4'

map_near_url = './/h4/a/@href'

map_near_phone = './/span[@class="nobr"]'

# images

images_xpath = './/div/a'

image_url_xpath = './@href'

image_img_src_xpath = './img/@src'

# property names

# FIXME : no translation

property_address = "Address"

property_phone = "Phone number"

# cookies

pref_cookie = ''

nid_cookie = {}

# remove google-specific tracking-url

def parse_url(url_string, google_hostname):

    # sanity check

    if url_string is None:

        return url_string

    # normal case

    parsed_url = urlparse(url_string)

    if (parsed_url.netloc in [google_hostname, '']

            and parsed_url.path == redirect_path):

        if len(language_array) == 2:

            country = language_array[1]

        else:

            country = '  '

            country = 'US'

        language = language_array[0] + ',' + language_array[0] + '-' + country

    if use_locale_domain:

        try:

            url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)

            parsed_url = urlparse(url, google_hostname)

            if (parsed_url.netloc == google_hostname

                and (parsed_url.path == search_path

                     or parsed_url.path.startswith(maps_path))):

                # remove the link to google news and google maps

                # FIXME : sometimes the URL is https://maps.google.*/maps

                # no consequence, the result trigger an exception after which is ignored

                continue

            # map result

            if ((parsed_url.netloc == google_hostname and parsed_url.path.startswith(maps_path))

               or (parsed_url.netloc.startswith(map_hostname_start))):

                x = result.xpath(map_near)

                if len(x) > 0:

                    # map : near the location

                    results = results + parse_map_near(parsed_url, x, google_hostname)

                else:

                    # map : detail about a location

                    results = results + parse_map_detail(parsed_url, result, google_hostname)

            # google news

            elif (parsed_url.netloc == google_hostname

                  and parsed_url.path == search_path):

                # skipping news results

                pass

            # images result

            if (parsed_url.netloc == google_hostname

            elif (parsed_url.netloc == google_hostname

                  and parsed_url.path == images_path):

                # only thumbnail image provided,

                # so skipping image results

                # results = results + parse_images(result, google_hostname)

                pass

            else:

                # normal result

                content = extract_text_from_dom(result, content_xpath)

                # append result

                results.append({'url': url,

                                'title': title,

                                'content': content})

        except Exception:

                                'content': content

                                })

        except:

            continue

    # parse suggestion

                        'title': '',

                        'content': '',

                        'img_src': img_src,

                        'template': 'images.html'})

                        'template': 'images.html'

                        })

    return results

def parse_map_near(parsed_url, x, google_hostname):

    results = []

    for result in x:

        title = extract_text_from_dom(result, map_near_title)

        url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname)

        attributes = []

        phone = extract_text_from_dom(result, map_near_phone)

        add_attributes(attributes, property_phone, phone, 'tel:' + phone)

        results.append({'title': title,

                        'url': url,

                        'content': attributes_to_html(attributes)

                        })

    return results

def parse_map_detail(parsed_url, result, google_hostname):

    results = []

    # try to parse the geoloc

    m = re.search('@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path)

    if m is None:

        m = re.search('ll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query)

    if m is not None:

        # geoloc found (ignored)

        lon = float(m.group(2))  # noqa

        lat = float(m.group(1))  # noqa

        zoom = int(m.group(3))  # noqa

        # attributes

        attributes = []

        address = extract_text_from_dom(result, map_address_xpath)

        phone = extract_text_from_dom(result, map_phone_xpath)

        add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon))

        add_attributes(attributes, property_phone, phone, 'tel:' + phone)

        # title / content / url

        website_title = extract_text_from_dom(result, map_website_title_xpath)

        content = extract_text_from_dom(result, content_xpath)

        website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname)

        # add a result if there is a website

        if website_url is not None:

            results.append({'title': website_title,

                            'content': (content + '<br />' if content is not None else '')

                            + attributes_to_html(attributes),

                            'url': website_url

                            })

    return results

def add_attributes(attributes, name, value, url):

    if value is not None and len(value) > 0:

        attributes.append({'label': name, 'value': value, 'url': url})

def attributes_to_html(attributes):

    retval = '<table class="table table-striped">'

    for a in attributes:

        value = a.get('value')

        if 'url' in a:

            value = '<a href="' + a.get('url') + '">' + value + '</a>'

        retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>'

    retval = retval + '</table>'

    return retval