Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Unverified Commit 36600118 authored by Alexandre Flament's avatar Alexandre Flament Committed by GitHub
Browse files

Merge pull request #2372 from dalf/remove-broken-engines 

[remove] remove searchcode_doc and twitter
parents 3c4a9c11 1c9e7cef

searx/engines/searchcode_doc.py

deleted100644 → 0
+0 −49
Original line number Original line Diff line number Diff line 
"""

 Searchcode (It)



 @website     https://searchcode.com/

 @provide-api yes (https://searchcode.com/api/)



 @using-api   yes

 @results     JSON

 @stable      yes

 @parse       url, title, content

"""



from json import loads

from urllib.parse import urlencode



# engine dependent config

categories = ['it']

paging = True



# search-url

url = 'https://searchcode.com/'

search_url = url + 'api/search_IV/?{query}&p={pageno}'





# do search-request

def request(query, params):

    params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)



    return params





# get response from search-request

def response(resp):

    results = []



    search_results = loads(resp.text)



    # parse results

    for result in search_results.get('results', []):

        href = result['url']

        title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])



        # append result

        results.append({'url': href,

                        'title': title,

                        'content': result['description']})



    # return results

    return results

searx/engines/twitter.py

deleted100644 → 0
+0 −87
Original line number Original line Diff line number Diff line 
"""

 Twitter (Social media)



 @website     https://twitter.com/

 @provide-api yes (https://dev.twitter.com/docs/using-search)



 @using-api   no

 @results     HTML (using search portal)

 @stable      no (HTML can change)

 @parse       url, title, content



 @todo        publishedDate

"""



from urllib.parse import urlencode, urljoin

from lxml import html

from datetime import datetime

from searx.utils import extract_text



# engine dependent config

categories = ['social media']

language_support = True



# search-url

base_url = 'https://twitter.com/'

search_url = base_url + 'search?'



# specific xpath variables

results_xpath = '//li[@data-item-type="tweet"]'

avatar_xpath = './/img[contains(@class, "avatar")]/@src'

link_xpath = './/small[@class="time"]//a'

title_xpath = './/span[contains(@class, "username")]'

content_xpath = './/p[contains(@class, "tweet-text")]'

timestamp_xpath = './/span[contains(@class,"_timestamp")]'





# do search-request

def request(query, params):

    params['url'] = search_url + urlencode({'q': query})



    # set language if specified

    if params['language'] != 'all':

        params['cookies']['lang'] = params['language'].split('-')[0]

    else:

        params['cookies']['lang'] = 'en'



    return params





# get response from search-request

def response(resp):

    results = []



    dom = html.fromstring(resp.text)



    # parse results

    for tweet in dom.xpath(results_xpath):

        try:

            link = tweet.xpath(link_xpath)[0]

            content = extract_text(tweet.xpath(content_xpath)[0])

            img_src = tweet.xpath(avatar_xpath)[0]

            img_src = img_src.replace('_bigger', '_normal')

        except Exception:

            continue



        url = urljoin(base_url, link.attrib.get('href'))

        title = extract_text(tweet.xpath(title_xpath))



        pubdate = tweet.xpath(timestamp_xpath)

        if len(pubdate) > 0:

            timestamp = float(pubdate[0].attrib.get('data-time'))

            publishedDate = datetime.fromtimestamp(timestamp, None)

            # append result

            results.append({'url': url,

                            'title': title,

                            'content': content,

                            'img_src': img_src,

                            'publishedDate': publishedDate})

        else:

            # append result

            results.append({'url': url,

                            'title': title,

                            'content': content,

                            'img_src': img_src})



    # return results

    return results

searx/settings.yml

+0 −8
Original line number Original line Diff line number Diff line
@@ -694,10 +694,6 @@ engines: 
    engine : stackoverflow

    engine : stackoverflow

    shortcut : st

    shortcut : st





  - name : searchcode doc

    engine : searchcode_doc

    shortcut : scd



  - name : searchcode code

  - name : searchcode code

    engine : searchcode_code

    engine : searchcode_code

    shortcut : scc

    shortcut : scc
@@ -764,10 +760,6 @@ engines: 
    categories : onions

    categories : onions

    shortcut : tch

    shortcut : tch





  - name : twitter

    engine : twitter

    shortcut : tw



# maybe in a fun category

# maybe in a fun category

#  - name : uncyclopedia

#  - name : uncyclopedia

#    engine : mediawiki

#    engine : mediawiki