Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3b1122c5 authored by rachmadani haryono's avatar rachmadani haryono Committed by Alexandre Flament
Browse files

[fix] fix duden engine (#1594)

parent 87baa74a

searx/engines/duden.py

+11 −8
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ 
from lxml import html, etree

import re

from searx.engines.xpath import extract_text

from searx.url_utils import quote

from searx.url_utils import quote, urljoin

from searx import logger



categories = ['general']
@@ -20,7 +20,7 @@ language_support = False 


# search-url

base_url = 'https://www.duden.de/'

search_url = base_url + 'suchen/dudenonline/{query}?page={offset}'

search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}'





def request(query, params):
@@ -35,6 +35,10 @@ def request(query, params): 
    '''



    offset = (params['pageno'] - 1)

    if offset == 0:

        search_url_fmt = base_url + 'suchen/dudenonline/{query}'

        params['url'] = search_url_fmt.format(query=quote(query))

    else:

        params['url'] = search_url.format(offset=offset, query=quote(query))

    return params
@@ -58,12 +62,11 @@ def response(resp): 
        logger.debug("Couldn't read number of results.")

        pass



    for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'):

    for result in dom.xpath('//section[not(contains(@class, "essay"))]'):

        try:

            logger.debug("running for %s" % str(result))

            link = result.xpath('.//h2/a')[0]

            url = link.attrib.get('href')

            title = result.xpath('string(.//h2/a)')

            url = result.xpath('.//h2/a')[0].get('href')

            url = urljoin(base_url, url)

            title = result.xpath('string(.//h2/a)').strip()

            content = extract_text(result.xpath('.//p'))

            # append result

            results.append({'url': url,

tests/unit/engines/test_duden.py

+17 −11
Original line number Diff line number Diff line
@@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase): 
    def test_request(self):

        query = 'Haus'

        dic = defaultdict(dict)

        dic['pageno'] = 1

        data = [

            [1, 'https://www.duden.de/suchen/dudenonline/Haus'],

            [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1']

        ]

        for page_no, exp_res in data:

            dic['pageno'] = page_no

            params = duden.request(query, dic)

            self.assertTrue('url' in params)

            self.assertTrue(query in params['url'])

            self.assertTrue('duden.de' in params['url'])

            self.assertEqual(params['url'], exp_res)



    def test_response(self):

        resp = mock.Mock(text='<html></html>')

        self.assertEqual(duden.response(resp), [])



        html = """

        <section class="wide">

        <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2>

        <p>This is the <strong>content</strong></p>

        <a href="https://this.is.the.url/">Zum vollst&auml;ndigen Artikel</a>

        <section class="vignette">

            <h2"> <a href="/rechtschreibung/Haus">

                <strong>This is the title also here</strong>

            </a> </h2>

          <p>This is the content</p>

        </section>

        """



        resp = mock.Mock(text=html)

        results = duden.response(resp)
@@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase): 


        # testing result (dictionary entry)

        r = results[0]

        self.assertEqual(r['url'], 'https://this.is.the.url/')

        self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus')

        self.assertEqual(r['title'], 'This is the title also here')

        self.assertEqual(r['content'], 'This is the content')