Merge pull request #1544 from MarcAbonce/youtube_fix (ca09d910) · Commits · e / infra / spot

searx/engines/youtube_noapi.py

+34 −36

Original line number	Original line	Diff line number	Diff line
	@@ -8,7 +8,8 @@
	# @stable no		# @stable no
	# @parse url, title, content, publishedDate, thumbnail, embedded		# @parse url, title, content, publishedDate, thumbnail, embedded

	from lxml import html		from functools import reduce
			from json import loads
	from searx.engines.xpath import extract_text		from searx.engines.xpath import extract_text
	from searx.utils import list_get		from searx.utils import list_get
	from searx.url_utils import quote_plus		from searx.url_utils import quote_plus
	@@ -34,20 +35,6 @@ embedded_url = '<iframe width="540" height="304" ' +\

	base_youtube_url = 'https://www.youtube.com/watch?v='		base_youtube_url = 'https://www.youtube.com/watch?v='

	# specific xpath variables
	results_xpath = "//ol/li/div[contains(@class, 'yt-lockup yt-lockup-tile yt-lockup-video vve-check')]"
	url_xpath = './/h3/a/@href'
	title_xpath = './/div[@class="yt-lockup-content"]/h3/a'
	content_xpath = './/div[@class="yt-lockup-content"]/div[@class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2"]'


	# returns extract_text on the first result selected by the xpath or None
	def extract_text_from_dom(result, xpath):
	r = result.xpath(xpath)
	if len(r) > 0:
	return extract_text(r[0])
	return None


	# do search-request		# do search-request
	def request(query, params):		def request(query, params):
	@@ -63,18 +50,29 @@ def request(query, params):
	def response(resp):		def response(resp):
	results = []		results = []

	dom = html.fromstring(resp.text)		results_data = resp.text[resp.text.find('ytInitialData'):]
			results_data = results_data[results_data.find('{'):results_data.find(';\n')]

	# parse results		results_json = loads(results_data) if results_data else {}
	for result in dom.xpath(results_xpath):		sections = results_json.get('contents', {})\
	videoid = list_get(result.xpath('@data-context-item-id'), 0)		.get('twoColumnSearchResultsRenderer', {})\
			.get('primaryContents', {})\
			.get('sectionListRenderer', {})\
			.get('contents', [])

			for section in sections:
			for video_container in section.get('itemSectionRenderer', {}).get('contents', []):
			video = video_container.get('videoRenderer', {})
			videoid = video.get('videoId')
	if videoid is not None:		if videoid is not None:
	url = base_youtube_url + videoid		url = base_youtube_url + videoid
	thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg'		thumbnail = 'https://i.ytimg.com/vi/' + videoid + '/hqdefault.jpg'
			title = video.get('title', {}).get('simpleText', videoid)
	title = extract_text_from_dom(result, title_xpath) or videoid		description_snippet = video.get('descriptionSnippet', {})
	content = extract_text_from_dom(result, content_xpath)		if 'runs' in description_snippet:
			content = reduce(lambda a, b: a + b.get('text', ''), description_snippet.get('runs'), '')
			else:
			content = description_snippet.get('simpleText', '')
	embedded = embedded_url.format(videoid=videoid)		embedded = embedded_url.format(videoid=videoid)

	# append result		# append result

tests/unit/engines/test_youtube_noapi.py

+56 −106

Original line number	Original line	Diff line number	Diff line
	@@ -46,121 +46,71 @@ class TestYoutubeNoAPIEngine(SearxTestCase):
	self.assertEqual(youtube_noapi.response(response), [])		self.assertEqual(youtube_noapi.response(response), [])

	html = """		html = """
	<ol id="item-section-063864" class="item-section">		<div></div>
	<li>		<script>
	<div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile"		window["ytInitialData"] = {
	data-context-item-id="DIVZCPfAOeM"		"contents": {
	data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB">		"twoColumnSearchResultsRenderer": {
	<div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto">		"primaryContents": {
	<a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link"		"sectionListRenderer": {
	data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA">		"contents": [
	<div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg"		{
	width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a>		"itemSectionRenderer": {
	<span class="thumb-menu dark-overflow-action-menu video-actions">		"contents": [
	</span>		{
	</div>		"videoRenderer": {
	<div class="yt-lockup-content">		"videoId": "DIVZCPfAOeM",
	<h3 class="yt-lockup-title">		"title": {
	<a href="/watch?v=DIVZCPfAOeM"		"simpleText": "Title"
	class="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link"		},
	data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"		"descriptionSnippet": {
	title="Top Speed Test Kawasaki Ninja H2 (Thailand) By. MEHAY SUPERBIKE"		"runs": [
	aria-describedby="description-id-259079" rel="spf-prefetch" dir="ltr">		{
	Title		"text": "Des"
	</a>		},
	<span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span>		{
	</h3>		"text": "cription"
	<div class="yt-lockup-byline">de		}
	<a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard"		]
	data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA"		}
	data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta">		}
	<ul class="yt-lockup-meta-info">		},
	<li>il y a 20 heures</li>		{
	<li>8 424 vues</li>		"videoRenderer": {
	</ul>		"videoId": "9C_HReR_McQ",
	</div>		"title": {
	<div class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2" dir="ltr">		"simpleText": "Title"
	Description		},
	</div>		"descriptionSnippet": {
	<div class="yt-lockup-badges">		"simpleText": "Description"
	<ul class="yt-badge-list ">		}
	<li class="yt-badge-item" >		}
	<span class="yt-badge">Nouveauté</span>		}
	</li>		]
	<li class="yt-badge-item" ><span class="yt-badge " >HD</span></li>		}
	</ul>		}
	</div>		]
	<div class="yt-lockup-action-menu yt-uix-menu-container">		}
	<div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded"		}
	data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu">		}
	</div>		}
	</div>		};
	</div>		</script>
	</div>
	</div>
	</li>
	</ol>
	"""		"""
	response = mock.Mock(text=html)		response = mock.Mock(text=html)
	results = youtube_noapi.response(response)		results = youtube_noapi.response(response)
	self.assertEqual(type(results), list)		self.assertEqual(type(results), list)
	self.assertEqual(len(results), 1)		self.assertEqual(len(results), 2)
	self.assertEqual(results[0]['title'], 'Title')		self.assertEqual(results[0]['title'], 'Title')
	self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM')		self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM')
	self.assertEqual(results[0]['content'], 'Description')		self.assertEqual(results[0]['content'], 'Description')
	self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg')		self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg')
	self.assertTrue('DIVZCPfAOeM' in results[0]['embedded'])		self.assertTrue('DIVZCPfAOeM' in results[0]['embedded'])
			self.assertEqual(results[1]['title'], 'Title')
	html = """		self.assertEqual(results[1]['url'], 'https://www.youtube.com/watch?v=9C_HReR_McQ')
	<ol id="item-section-063864" class="item-section">		self.assertEqual(results[1]['content'], 'Description')
	<li>		self.assertEqual(results[1]['thumbnail'], 'https://i.ytimg.com/vi/9C_HReR_McQ/hqdefault.jpg')
	<div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile"		self.assertTrue('9C_HReR_McQ' in results[1]['embedded'])
	data-context-item-id="DIVZCPfAOeM"
	data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB">
	<div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto">
	<a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link"
	data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA">
	<div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg"
	width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a>
	<span class="thumb-menu dark-overflow-action-menu video-actions">
	</span>
	</div>
	<div class="yt-lockup-content">
	<h3 class="yt-lockup-title">
	<span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span>
	</h3>
	<div class="yt-lockup-byline">de
	<a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard"
	data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA"
	data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta">
	<ul class="yt-lockup-meta-info">
	<li>il y a 20 heures</li>
	<li>8 424 vues</li>
	</ul>
	</div>
	<div class="yt-lockup-badges">
	<ul class="yt-badge-list ">
	<li class="yt-badge-item" >
	<span class="yt-badge">Nouveauté</span>
	</li>
	<li class="yt-badge-item" ><span class="yt-badge " >HD</span></li>
	</ul>
	</div>
	<div class="yt-lockup-action-menu yt-uix-menu-container">
	<div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded"
	data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu">
	</div>
	</div>
	</div>
	</div>
	</div>
	</li>
	</ol>
	"""
	response = mock.Mock(text=html)
	results = youtube_noapi.response(response)
	self.assertEqual(type(results), list)
	self.assertEqual(len(results), 1)

	html = """		html = """
	<ol id="item-section-063864" class="item-section">		<ol id="item-section-063864" class="item-section">