[fix] replace the dead google images ajax api with a working one (439cf055) · Commits · e / infra / spot

searx/engines/google_images.py

+29 −30

Original line number	Diff line number	Diff line
		@@ -2,41 +2,42 @@
		Google (Images)

		@website https://www.google.com
		@provide-api yes (https://developers.google.com/web-search/docs/),
		deprecated!
		@provide-api yes (https://developers.google.com/custom-search/)

		@using-api yes
		@results JSON
		@stable yes (but deprecated)
		@results HTML chunk
		@stable no
		@parse url, title, img_src
		"""

		from urllib import urlencode, unquote
		from urllib import urlencode
		from urlparse import parse_qs
		from json import loads
		from lxml import html

		# engine dependent config
		categories = ['images']
		paging = True
		safesearch = True

		# search-url
		url = 'https://ajax.googleapis.com/'
		search_url = url + 'ajax/services/search/images?v=1.0&start={offset}&rsz=large&safe={safesearch}&filter=off&{query}'
		search_url = 'https://www.google.com/search'\
		'?{query}'\
		'&tbm=isch'\
		'&ijn=1'\
		'&start={offset}'


		# do search-request
		def request(query, params):
		offset = (params['pageno'] - 1) * 8

		if params['safesearch'] == 0:
		safesearch = 'off'
		else:
		safesearch = 'on'
		offset = (params['pageno'] - 1) * 100

		params['url'] = search_url.format(query=urlencode({'q': query}),
		offset=offset,
		safesearch=safesearch)

		if safesearch and params['safesearch']:
		params['url'] += '&' + urlencode({'safe': 'active'})

		return params


		@@ -44,30 +45,28 @@ def request(query, params):
		def response(resp):
		results = []

		search_res = loads(resp.text)

		# return empty array if there are no results
		if not search_res.get('responseData', {}).get('results'):
		return []
		dom = html.fromstring(resp.text)

		# parse results
		for result in search_res['responseData']['results']:
		href = result['originalContextUrl']
		title = result['title']
		if 'url' not in result:
		continue
		thumbnail_src = result['tbUrl']
		for result in dom.xpath('//div[@data-ved]'):
		data_url = result.xpath('./a/@href')[0]
		data_query = {k: v[0] for k, v in parse_qs(data_url.split('?', 1)[1]).iteritems()}

		metadata = loads(result.xpath('./div[@class="rg_meta"]/text()')[0])

		thumbnail_src = metadata['tu']

		# http to https
		thumbnail_src = thumbnail_src.replace("http://", "https://")

		# append result
		results.append({'url': href,
		'title': title,
		'content': result['content'],
		'thumbnail_src': thumbnail_src,
		'img_src': unquote(result['url']),
		results.append({'url': data_query['imgrefurl'],
		'title': metadata['pt'],
		'content': metadata['s'],
		'thumbnail_src': metadata['tu'],
		'img_src': data_query['imgurl'],
		'template': 'images.html'})

		# return results
		print len(results)
		return results

searx/tests/engines/test_google_images.py

+28 −83

Original line number	Diff line number	Diff line
		@@ -10,15 +10,15 @@ class TestGoogleImagesEngine(SearxTestCase):
		query = 'test_query'
		dicto = defaultdict(dict)
		dicto['pageno'] = 1
		dicto['safesearch'] = 1
		params = google_images.request(query, dicto)
		self.assertIn('url', params)
		self.assertIn(query, params['url'])
		self.assertIn('googleapis.com', params['url'])
		self.assertIn('safe=on', params['url'])
		self.assertIn('safe=active', params['url'])

		dicto['safesearch'] = 0
		params = google_images.request(query, dicto)
		self.assertIn('safe=off', params['url'])
		self.assertNotIn('safe', params['url'])

		def test_response(self):
		self.assertRaises(AttributeError, google_images.response, None)
		@@ -26,88 +26,33 @@ class TestGoogleImagesEngine(SearxTestCase):
		self.assertRaises(AttributeError, google_images.response, '')
		self.assertRaises(AttributeError, google_images.response, '[]')

		response = mock.Mock(text='{}')
		response = mock.Mock(text='<div></div>')
		self.assertEqual(google_images.response(response), [])

		response = mock.Mock(text='{"data": []}')
		self.assertEqual(google_images.response(response), [])

		json = """
		{
		"responseData": {
		"results": [
		{
		"GsearchResultClass": "GimageSearch",
		"width": "400",
		"height": "400",
		"imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
		"tbWidth": "124",
		"tbHeight": "124",
		"unescapedUrl": "http://unescaped.url.jpg",
		"url": "http://image.url.jpg",
		"visibleUrl": "insolitebuzz.fr",
		"title": "This is the title",
		"titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
		"originalContextUrl": "http://this.is.the.url",
		"content": "<b>test</b>",
		"contentNoFormatting": "test",
		"tbUrl": "http://thumbnail.url"
		}
		]
		},
		"responseDetails": null,
		"responseStatus": 200
		}
		"""
		response = mock.Mock(text=json)
		html = """
		<div style="display:none">
		<div eid="fWhnVq4Shqpp3pWo4AM" id="isr_scm_1" style="display:none"></div>
		<div data-cei="fWhnVq4Shqpp3pWo4AM" class="rg_add_chunk"><!--m-->
		<div class="rg_di rg_el ivg-i" data-ved="0ahUKEwjuxPWQts3JAhUGVRoKHd4KCjwQMwgDKAAwAA">
		<a href="/imgres?imgurl=http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png&imgrefurl=http://www.clker.com/clipart-south-arrow.html&h=598&w=504&tbnid=bQWQ9wz9loJmjM:&docid=vlONkeBtERfDuM&ei=fWhnVq4Shqpp3pWo4AM&tbm=isch" jsaction="fire.ivg_o;mouseover:str.hmov;mouseout:str.hmou" class="rg_l"><img data-src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRsxy3gKnEX0lrwwpRxdPWyLJ8iZ--PXZ-ThbBA2_xXDG_bdQutMQ" data-sz="f" name="bQWQ9wz9loJmjM:" class="rg_i" alt="Image result for south" jsaction="load:str.tbn" onload="google.aft&&google.aft(this)">
		<div class="_aOd rg_ilm">
		<div class="rg_ilmbg"><span class="rg_ilmn"> 504 × 598 - clker.com </span>
		</div>
		</div>
		</a>
		<div class="rg_meta">
		{"id":"bQWQ9wz9loJmjM:","isu":"clker.com","ity":"png","md":"/search?tbs\u003dsbi:AMhZZit7u1mHyop9pQisu-5idR-8W_1Itvwc3afChmsjQYPx_1yYMzBvUZgtkcGoojqekKZ-6n_1rjX9ySH0OWA_1eO5OijFY6BBDw_1GApr6xxb1bXJcBcj-DiguMoXWW7cZSG7MRQbwnI5SoDZNXcv_1xGszy886I7NVb_1oRKSliTHtzqbXAxhvYreM","msu":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAQSEgltBZD3DP2WgiG-U42R4G0RFw","oh":598,"os":"13KB","ow":504,"pt":"South Arrow Clip Art at Clker.com - vector clip art online ...","rid":"vlONkeBtERfDuM","s":"Download this image as:","sc":1,"si":"/search?q\u003dsouth\u0026biw\u003d1364\u0026bih\u003d235\u0026tbm\u003disch\u0026tbs\u003dsimg:CAESEgltBZD3DP2WgiG-U42R4G0RFw","th":245,"tu":"https://thumbnail.url/","tw":206}
		</div>
		</div><!--n--><!--m-->
		</div>
		</div>
		""" # noqa
		response = mock.Mock(text=html)
		results = google_images.response(response)
		self.assertEqual(type(results), list)
		self.assertEqual(len(results), 1)
		self.assertEqual(results[0]['title'], 'This is the title')
		self.assertEqual(results[0]['url'], 'http://this.is.the.url')
		self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url')
		self.assertEqual(results[0]['img_src'], 'http://image.url.jpg')
		self.assertEqual(results[0]['content'], '<b>test</b>')

		json = """
		{
		"responseData": {
		"results": [
		{
		"GsearchResultClass": "GimageSearch",
		"width": "400",
		"height": "400",
		"imageId": "ANd9GcQbYb9FJuAbG_hT4i8FeC0O0x-P--EHdzgRIF9ao97nHLl7C2mREn6qTQ",
		"tbWidth": "124",
		"tbHeight": "124",
		"unescapedUrl": "http://unescaped.url.jpg",
		"visibleUrl": "insolitebuzz.fr",
		"title": "This is the title",
		"titleNoFormatting": "Petit test sympa qui rend fou tout le monde ! A faire",
		"originalContextUrl": "http://this.is.the.url",
		"content": "<b>test</b>",
		"contentNoFormatting": "test",
		"tbUrl": "http://thumbnail.url"
		}
		]
		},
		"responseDetails": null,
		"responseStatus": 200
		}
		"""
		response = mock.Mock(text=json)
		results = google_images.response(response)
		self.assertEqual(type(results), list)
		self.assertEqual(len(results), 0)

		json = """
		{
		"responseData": {},
		"responseDetails": null,
		"responseStatus": 200
		}
		"""
		response = mock.Mock(text=json)
		results = google_images.response(response)
		self.assertEqual(type(results), list)
		self.assertEqual(len(results), 0)
		self.assertEqual(results[0]['title'], u'South Arrow Clip Art at Clker.com - vector clip art online ...')
		self.assertEqual(results[0]['url'], 'http://www.clker.com/clipart-south-arrow.html')
		self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url/')
		self.assertEqual(results[0]['img_src'], 'http://www.clker.com/cliparts/H/X/l/b/0/0/south-arrow-hi.png')
		self.assertEqual(results[0]['content'], 'Download this image as:')