Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5cea4f94 authored by Adam Tauber's avatar Adam Tauber
Browse files

[fix] prevent google engine to redirect 

nid/pref cookies are also removed
parent 029291ec

searx/engines/google.py

+1 −29
Original line number Diff line number Diff line
@@ -13,7 +13,6 @@ from cgi import escape 
from urllib import urlencode

from urlparse import urlparse, parse_qsl

from lxml import html, etree

from searx.poolrequests import get

from searx.engines.xpath import extract_text, extract_url

from searx.search import logger
@@ -91,7 +90,7 @@ url_map = 'https://www.openstreetmap.org/'\ 
search_path = '/search'

search_url = ('https://{hostname}' +

              search_path +

              '?{query}&start={offset}&gbv=1')

              '?{query}&start={offset}&gbv=1&gws_rd=cr')



# other URLs

map_hostname_start = 'maps.google.'
@@ -129,27 +128,6 @@ image_img_src_xpath = './img/@src' 
property_address = "Address"

property_phone = "Phone number"



# cookies

pref_cookie = ''

nid_cookie = {}





# see https://support.google.com/websearch/answer/873?hl=en

def get_google_pref_cookie():

    global pref_cookie

    if pref_cookie == '':

        resp = get('https://www.google.com/ncr', allow_redirects=False)

        pref_cookie = resp.cookies["PREF"]

    return pref_cookie





def get_google_nid_cookie(google_hostname):

    global nid_cookie

    if google_hostname not in nid_cookie:

        resp = get('https://' + google_hostname)

        nid_cookie[google_hostname] = resp.cookies.get("NID", None)

    return nid_cookie[google_hostname]





# remove google-specific tracking-url

def parse_url(url_string, google_hostname):
@@ -201,12 +179,6 @@ def request(query, params): 


    params['headers']['Accept-Language'] = language

    params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

    if google_hostname == default_hostname:

        try:

            params['cookies']['PREF'] = get_google_pref_cookie()

        except:

            logger.warning('cannot fetch PREF cookie')

    params['cookies']['NID'] = get_google_nid_cookie(google_hostname)



    params['google_hostname'] = google_hostname

searx/tests/engines/test_google.py

+1 −5
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ from searx.testing import SearxTestCase 
class TestGoogleEngine(SearxTestCase):



    def mock_response(self, text):

        response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1')

        response = mock.Mock(text=text, url='https://www.google.com/search?q=test&start=0&gbv=1&gws_rd=cr')

        response.search_params = mock.Mock()

        response.search_params.get = mock.Mock(return_value='www.google.com')

        return response
@@ -23,16 +23,12 @@ class TestGoogleEngine(SearxTestCase): 
        self.assertIn('url', params)

        self.assertIn(query, params['url'])

        self.assertIn('google.fr', params['url'])

        self.assertNotIn('PREF', params['cookies'])

        self.assertIn('NID', params['cookies'])

        self.assertIn('fr', params['headers']['Accept-Language'])



        dicto['language'] = 'all'

        params = google.request(query, dicto)

        self.assertIn('google.com', params['url'])

        self.assertIn('en', params['headers']['Accept-Language'])

        # self.assertIn('PREF', params['cookies'])

        self.assertIn('NID', params['cookies'])



    def test_response(self):

        self.assertRaises(AttributeError, google.response, None)