Merge pull request #483 from misnyo/master

 @parse       url, title, content, publishedDate, embedded

"""

import re

from StringIO import StringIO

from json import loads

from lxml import etree

from urllib import urlencode, quote_plus

from dateutil import parser

from searx import logger

from searx.poolrequests import get as http_get

# engine dependent config

categories = ['music']

paging = True

# api-key

guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'

# search-url

url = 'https://api.soundcloud.com/'

search_url = url + 'search?{query}'\

    'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'

def get_client_id():

    response = http_get("https://soundcloud.com")

    rx_namespace = {"re": "http://exslt.org/regular-expressions"}

    if response.ok:

        tree = etree.parse(StringIO(response.content), etree.HTMLParser())

        script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)

        app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

        # extracts valid app_js urls from soundcloud.com content

        for app_js_url in app_js_urls:

            # gets app_js and searches for the clientid

            response = http_get(app_js_url)

            if response.ok:

                cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)

                if cids is not None and len(cids.groups()):

                    return cids.groups()[0]

    logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

    return ""

# api-key

guest_client_id = get_client_id()

# do search-request

def request(query, params):

    offset = (params['pageno'] - 1) * 20