Merge pull request #677 from pydo/feature/seedpeer-engine-integration (e7d005c6) · Commits · e / infra / spot

AUTHORS.rst

+1 −0

Original line number	Diff line number	Diff line
		@@ -58,3 +58,4 @@ generally made searx better:
		- marc @a01200356
		- Harry Wood @harry-wood
		- Thomas Renard @threnard
		- Pydo `<https://github.com/pydo>`_

searx/engines/seedpeer.py

0 → 100644

+78 −0

Original line number	Diff line number	Diff line
		# Seedpeer (Videos, Music, Files)
		#
		# @website http://seedpeer.eu
		# @provide-api no (nothing found)
		#
		# @using-api no
		# @results HTML (using search portal)
		# @stable yes (HTML can change)
		# @parse url, title, content, seed, leech, magnetlink

		from urlparse import urljoin
		from cgi import escape
		from urllib import quote
		from lxml import html
		from operator import itemgetter
		from searx.engines.xpath import extract_text


		url = 'http://www.seedpeer.eu/'
		search_url = url + 'search/{search_term}/7/{page_no}.html'
		# specific xpath variables
		torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
		alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
		title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
		alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
		seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
		alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
		peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
		alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
		age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
		alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
		size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
		alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'


		# do search-request
		def request(query, params):
		params['url'] = search_url.format(search_term=quote(query),
		page_no=params['pageno'] - 1)
		return params


		# get response from search-request
		def response(resp):
		results = []
		dom = html.fromstring(resp.text)
		torrent_links = dom.xpath(torrent_xpath)
		if len(torrent_links) > 0:
		seeds = dom.xpath(seeds_xpath)
		peers = dom.xpath(peers_xpath)
		titles = dom.xpath(title_xpath)
		sizes = dom.xpath(size_xpath)
		ages = dom.xpath(age_xpath)
		else: # under ~5 results uses a different xpath
		torrent_links = dom.xpath(alternative_torrent_xpath)
		seeds = dom.xpath(alternative_seeds_xpath)
		peers = dom.xpath(alternative_peers_xpath)
		titles = dom.xpath(alternative_title_xpath)
		sizes = dom.xpath(alternative_size_xpath)
		ages = dom.xpath(alternative_age_xpath)
		# return empty array if nothing is found
		if not torrent_links:
		return []

		# parse results
		for index, result in enumerate(torrent_links):
		link = result.attrib.get('href')
		href = urljoin(url, link)
		results.append({'url': href,
		'title': titles[index].text_content(),
		'content': '{}, {}'.format(sizes[index], ages[index]),
		'seed': seeds[index],
		'leech': peers[index],

		'template': 'torrent.html'})

		# return results sorted by seeder
		return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

+6 −0

Original line number	Diff line number	Diff line
		@@ -495,6 +495,12 @@ engines:
		timeout: 6.0
		categories : science

		- name : seedpeer
		engine : seedpeer
		shortcut: speu
		categories: files, music, videos
		disabled: True

		- name : dictzone
		engine : dictzone
		shortcut : dc

tests/unit/engines/seedpeer_fixture.html

0 → 100644

+110 −0

Original line number	Diff line number	Diff line
		<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
		<html xmlns="http://www.w3.org/1999/xhtml">
		<head>
		</head>
		<body>
		<div id="header">
		<div id="whoIsYou">
		<a href="/lang.php"><small>SeedPeer in your own language?</small></a>   <a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
		</div>

		<script type="text/javascript">
		whoIsYou();
		</script>
		<div id="search">
		<form action="/search.php" method="get">
		<input id="topsearchbar" name="search" value="narcos season 2" />
		<input type="submit" class="searchbutton" value="Torrents" />
		<input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
		</form>
		<div id="suggestion"></div>
		</div>
		<div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
		<div id="subtext"><a href="/">Home</a> > <a href="/search.html">Torrent search</a> > Narcos season 2 \| page 1</div>
		</div>
		<div id="nav">
		<ul>
		<!--
		<li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>

		-->
		<li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
		<ul>
		<li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
		<li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
		<li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
		<li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
		<li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
		<li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
		<li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
		</ul>
		</li>
		<li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
		<li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
		<li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
		<li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
		<li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
		<ul>
		<li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
		<li><a title="Contact Us" href="/contact.html">Contact</a></li>
		<li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li>
		<li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
		<li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
		</ul>
		</li>
		<!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
		</ul>
		<div class="clear"></div>
		</div>
		<div id="body"><div id="pageTop"></div>
		<div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
		<span style="float:right">
		<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> \|
		<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
		</span>
		<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a> 2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
		<span style="float:right">
		<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> \|
		<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
		</span>
		<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
		<table width="100%">
		<tr>
		<td width="30%">
		<h2>Torrents Download</h2>
		<a href="/">Torrent search</a><br />
		<a href="/browse.html">Browse categories</a><br />
		<a href="/verified.html">Verified Torrents</a><br />
		<a href="/order-date.html">Today's torrents</a><br />
		<a href="/yesterday.html">Yesterday's torrents</a><br />
		<a href="/stats.html">Statistics</a><br />
		<br />
		<a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
		</td>
		<td width="30%"><h2>Cool Stuff</h2>
		<a href="/promotional.php">Promotional</a><br />
		<a href="/contact.html">Advertising Information</a><br />
		<strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
		<a href="http://www.utorrent.com">µTorrent Client</a><br />
		<a href="/blog">Seedpeer Blog</a><br />
		</td>
		<td width="30%"><h2>Links</h2>
		<a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
		<a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
		<a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
		<a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
		<a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
		<a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
		<a href="https://monova.org" target="_blank">Monova</a><br />
		<a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
		<a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
		</td>
		<td width="10%"><div id="bottomlogo"></div></td>
		</tr>
		</table>
		<br />
		<br />
		</div>
		</div>
		</body>
		</html>
		No newline at end of file

tests/unit/engines/test_seedpeer.py

0 → 100644

+51 −0

Original line number	Diff line number	Diff line
		import mock
		from collections import defaultdict
		from searx.engines import seedpeer
		from searx.testing import SearxTestCase
		from datetime import datetime


		class TestSeedPeerEngine(SearxTestCase):

		html = ''
		with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
		html += fixture.read()

		def test_request(self):
		query = 'test_query'
		dicto = defaultdict(dict)
		dicto['pageno'] = 1
		params = seedpeer.request(query, dicto)
		self.assertIn('url', params)
		self.assertIn(query, params['url'])
		self.assertIn('seedpeer.eu', params['url'])

		def test_response_raises_attr_error_on_empty_response(self):
		self.assertRaises(AttributeError, seedpeer.response, None)
		self.assertRaises(AttributeError, seedpeer.response, [])
		self.assertRaises(AttributeError, seedpeer.response, '')
		self.assertRaises(AttributeError, seedpeer.response, '[]')

		def test_response_returns_empty_list(self):
		response = mock.Mock(text='<html></html>')
		self.assertEqual(seedpeer.response(response), [])

		def test_response_returns_all_results(self):
		response = mock.Mock(text=self.html)
		results = seedpeer.response(response)
		self.assertTrue(isinstance(results, list))
		self.assertEqual(len(results), 2)

		def test_response_returns_correct_results(self):
		response = mock.Mock(text=self.html)
		results = seedpeer.response(response)
		self.assertEqual(
		results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
		)
		self.assertEqual(
		results[0]['url'],
		'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
		)
		self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
		self.assertEqual(results[0]['seed'], '861')
		self.assertEqual(results[0]['leech'], '332')