Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c80e82a8 authored by Markus Heiser's avatar Markus Heiser
Browse files

[mod] DuckDuckGo: reversed engineered & upgrade to data_type: traits_v1



Partial reverse engineering of the DuckDuckGo (DDG) engines including a
improved language and region handling based on the enigne.traits_v1 data.

- DDG Lite
- DDG Instant Answer API
- DDG Images
- DDG Weather

docs/src/searx.engine.duckduckgo.rst:
  Online documentation of the DDG engines (make docs.live)

searx/data/engine_traits.json
  Add data type "traits_v1" generated by the fetch_traits() functions from:

  - "duckduckgo" (WEB),
  - "duckduckgo images" and
  - "duckduckgo weather"

  and remove data from obsolete data type "supported_languages".

searx/autocomplete.py:
  Reversed engineered Autocomplete from DDG.  Supports DDG's languages.

searx/engines/duckduckgo.py:
  - fetch_traits():  Fetch languages & regions from DDG.

  - get_ddg_lang(): Get DDG's language identifier from SearXNG's locale.  DDG
    defines its languages by region codes.  DDG-Lite does not offer a language
    selection to the user, only a region can be selected by the user.

  - Cache ``vqd`` value: The vqd value depends on the query string and is needed
    for the follow up pages or the images loaded by a XMLHttpRequest (DDG
    images).  The ``vqd`` value of a search term is stored for 10min in the
    redis DB.

  - DDG Lite engine: reversed engineered request method with improved Language
    and region support and better ``vqd`` handling.

searx/engines/duckduckgo_definitions.py: DDG Instant Answer API
  The *instant answers* API does not support languages, or at least we could not
  find out how language support should work.  It seems that most of the features
  are based on English terms.

searx/engines/duckduckgo_images.py: DDG Images
  Reversed engineered request method.  Improved language and region handling
  based on cookies and the enigne.traits_v1 data.  Response: add image format to
  the result list

searx/engines/duckduckgo_weather.py: DDG Weather
  Improved language and region handling based on cookies and the
  enigne.traits_v1 data.

Signed-off-by: default avatarMarkus Heiser <markus.heiser@darmarit.de>
parent e9afc4f8
Loading
Loading
Loading
Loading
+22 −0
Original line number Diff line number Diff line
.. _duckduckgo engines:

=================
DukcDukGo engines
=================

.. contents:: Contents
   :depth: 2
   :local:
   :backlinks: entry

.. automodule:: searx.engines.duckduckgo
   :members:

.. automodule:: searx.engines.duckduckgo_images
   :members:

.. automodule:: searx.engines.duckduckgo_definitions
   :members:

.. automodule:: searx.engines.duckduckgo_weather
   :members:
+17 −7
Original line number Diff line number Diff line
@@ -61,14 +61,24 @@ def dbpedia(query, _lang):
    return results


def duckduckgo(query, _lang):
    # duckduckgo autocompleter
    url = 'https://ac.duckduckgo.com/ac/?{0}&type=list'
def duckduckgo(query, sxng_locale):
    """Autocomplete from DuckDuckGo. Supports DuckDuckGo's languages"""

    resp = loads(get(url.format(urlencode(dict(q=query)))).text)
    if len(resp) > 1:
        return resp[1]
    return []
    traits = engines['duckduckgo'].traits
    args = {
        'q': query,
        'kl': traits.get_region(sxng_locale, traits.all_locale),
    }

    url = 'https://duckduckgo.com/ac/?type=list&' + urlencode(args)
    resp = get(url)

    ret_val = []
    if resp.ok:
        j = resp.json()
        if len(j) > 1:
            ret_val = j[1]
    return ret_val


def google(query, lang):
+300 −207
Original line number Diff line number Diff line
@@ -471,11 +471,106 @@
      "zh_TW"
    ]
  },
  "ddg definitions": {
  "duckduckgo": {
    "all_locale": "wt-wt",
    "custom": {},
    "data_type": "supported_languages",
    "languages": {},
    "custom": {
      "lang_region": {
        "ar-DZ": "ar_DZ",
        "ar-JO": "ar_JO",
        "ar-SA": "ar_SA",
        "bn-IN": "bn_IN",
        "de-CH": "de_CH",
        "en-AU": "en_AU",
        "en-CA": "en_CA",
        "en-GB": "en_GB",
        "es-AR": "es_AR",
        "es-CL": "es_CL",
        "es-CO": "es_CO",
        "es-CR": "es_CR",
        "es-EC": "es_EC",
        "es-MX": "es_MX",
        "es-PE": "es_PE",
        "es-UY": "es_UY",
        "es-VE": "es_VE",
        "fr-BE": "fr_BE",
        "fr-CA": "fr_CA",
        "fr-CH": "fr_CH",
        "nl-BE": "nl_BE",
        "pt-BR": "pt_BR"
      }
    },
    "data_type": "traits_v1",
    "languages": {
      "af": "af_ZA",
      "ar": "ar_EG",
      "ast": "ast_ES",
      "az_Latn": "az_AZ",
      "be": "be_BY",
      "bg": "bg_BG",
      "bn": "bn_BD",
      "br": "br_FR",
      "bs_Latn": "bs_BA",
      "ca": "ca_ES",
      "cs": "cs_CZ",
      "cy": "cy_GB",
      "da": "da_DK",
      "de": "de_DE",
      "el": "el_GR",
      "en": "en_US",
      "eo": "eo_XX",
      "es": "es_ES",
      "et": "et_EE",
      "eu": "eu_ES",
      "fa": "fa_IR",
      "fi": "fi_FI",
      "fil": "tl_PH",
      "fr": "fr_FR",
      "ga": "ga_IE",
      "gd": "gd_GB",
      "gl": "gl_ES",
      "he": "he_IL",
      "hi": "hi_IN",
      "hr": "hr_HR",
      "hu": "hu_HU",
      "hy": "hy_AM",
      "id": "id_ID",
      "is": "is_IS",
      "it": "it_IT",
      "ja": "ja_JP",
      "kab": "kab_DZ",
      "kn": "kn_IN",
      "ko": "ko_KR",
      "ku": "ku",
      "kw": "kw_GB",
      "lt": "lt_LT",
      "lv": "lv_LV",
      "ml": "ml_IN",
      "mr": "mr_IN",
      "ms": "ms_MY",
      "nb": "nb_NO",
      "nl": "nl_NL",
      "nn": "nn_NO",
      "pl": "pl_PL",
      "pt": "pt_PT",
      "ro": "ro_RO",
      "ru": "ru_RU",
      "sc": "sc_IT",
      "si": "si_LK",
      "sk": "sk_SK",
      "sl": "sl_SI",
      "sq": "sq_AL",
      "sr_Cyrl": "sr_RS",
      "sv": "sv_SE",
      "ta": "ta_IN",
      "te": "te_IN",
      "th": "th_TH",
      "tr": "tr_TR",
      "uk": "uk_UA",
      "ur": "ur_PK",
      "vi": "vi_VN",
      "zh_Hans": "zh_CN",
      "zh_Hant": "zh_TW"
    },
    "regions": {
      "ar-SA": "xa-ar",
      "bg-BG": "bg-bg",
@@ -539,77 +634,108 @@
      "zh-HK": "hk-tzh",
      "zh-TW": "tw-tzh"
    },
    "supported_languages": [
      "ar-XA",
      "bg-BG",
      "ca-CT",
      "ca-ES",
      "cs-CZ",
      "da-DK",
      "de-AT",
      "de-CH",
      "de-DE",
      "el-GR",
      "en-AU",
      "en-CA",
      "en-ID",
      "en-IE",
      "en-IL",
      "en-IN",
      "en-MY",
      "en-NZ",
      "en-PH",
      "en-PK",
      "en-SG",
      "en-TH",
      "en-UK",
      "en-US",
      "en-VN",
      "en-ZA",
      "es-AR",
      "es-CL",
      "es-CO",
      "es-ES",
      "es-MX",
      "es-PE",
      "es-US",
      "et-EE",
      "fi-FI",
      "fr-BE",
      "fr-CA",
      "fr-CH",
      "fr-FR",
      "hr-HR",
      "hu-HU",
      "it-IT",
      "jp-JP",
      "kr-KR",
      "lt-LT",
      "lv-LV",
      "nl-BE",
      "nl-NL",
      "no-NO",
      "pl-PL",
      "pt-BR",
      "pt-PT",
      "ro-RO",
      "ru-RU",
      "sk-SK",
      "sl-SL",
      "sv-SE",
      "tr-TR",
      "tzh-HK",
      "tzh-TW",
      "uk-UA",
      "wt-WT",
      "zh-CN"
    ]
    "supported_languages": {}
  },
  "duckduckgo": {
  "duckduckgo images": {
    "all_locale": "wt-wt",
    "custom": {},
    "data_type": "supported_languages",
    "languages": {},
    "custom": {
      "lang_region": {
        "ar-DZ": "ar_DZ",
        "ar-JO": "ar_JO",
        "ar-SA": "ar_SA",
        "bn-IN": "bn_IN",
        "de-CH": "de_CH",
        "en-AU": "en_AU",
        "en-CA": "en_CA",
        "en-GB": "en_GB",
        "es-AR": "es_AR",
        "es-CL": "es_CL",
        "es-CO": "es_CO",
        "es-CR": "es_CR",
        "es-EC": "es_EC",
        "es-MX": "es_MX",
        "es-PE": "es_PE",
        "es-UY": "es_UY",
        "es-VE": "es_VE",
        "fr-BE": "fr_BE",
        "fr-CA": "fr_CA",
        "fr-CH": "fr_CH",
        "nl-BE": "nl_BE",
        "pt-BR": "pt_BR"
      }
    },
    "data_type": "traits_v1",
    "languages": {
      "af": "af_ZA",
      "ar": "ar_EG",
      "ast": "ast_ES",
      "az_Latn": "az_AZ",
      "be": "be_BY",
      "bg": "bg_BG",
      "bn": "bn_BD",
      "br": "br_FR",
      "bs_Latn": "bs_BA",
      "ca": "ca_ES",
      "cs": "cs_CZ",
      "cy": "cy_GB",
      "da": "da_DK",
      "de": "de_DE",
      "el": "el_GR",
      "en": "en_US",
      "eo": "eo_XX",
      "es": "es_ES",
      "et": "et_EE",
      "eu": "eu_ES",
      "fa": "fa_IR",
      "fi": "fi_FI",
      "fil": "tl_PH",
      "fr": "fr_FR",
      "ga": "ga_IE",
      "gd": "gd_GB",
      "gl": "gl_ES",
      "he": "he_IL",
      "hi": "hi_IN",
      "hr": "hr_HR",
      "hu": "hu_HU",
      "hy": "hy_AM",
      "id": "id_ID",
      "is": "is_IS",
      "it": "it_IT",
      "ja": "ja_JP",
      "kab": "kab_DZ",
      "kn": "kn_IN",
      "ko": "ko_KR",
      "ku": "ku",
      "kw": "kw_GB",
      "lt": "lt_LT",
      "lv": "lv_LV",
      "ml": "ml_IN",
      "mr": "mr_IN",
      "ms": "ms_MY",
      "nb": "nb_NO",
      "nl": "nl_NL",
      "nn": "nn_NO",
      "pl": "pl_PL",
      "pt": "pt_PT",
      "ro": "ro_RO",
      "ru": "ru_RU",
      "sc": "sc_IT",
      "si": "si_LK",
      "sk": "sk_SK",
      "sl": "sl_SI",
      "sq": "sq_AL",
      "sr_Cyrl": "sr_RS",
      "sv": "sv_SE",
      "ta": "ta_IN",
      "te": "te_IN",
      "th": "th_TH",
      "tr": "tr_TR",
      "uk": "uk_UA",
      "ur": "ur_PK",
      "vi": "vi_VN",
      "zh_Hans": "zh_CN",
      "zh_Hant": "zh_TW"
    },
    "regions": {
      "ar-SA": "xa-ar",
      "bg-BG": "bg-bg",
@@ -673,77 +799,108 @@
      "zh-HK": "hk-tzh",
      "zh-TW": "tw-tzh"
    },
    "supported_languages": [
      "ar-XA",
      "bg-BG",
      "ca-CT",
      "ca-ES",
      "cs-CZ",
      "da-DK",
      "de-AT",
      "de-CH",
      "de-DE",
      "el-GR",
      "en-AU",
      "en-CA",
      "en-ID",
      "en-IE",
      "en-IL",
      "en-IN",
      "en-MY",
      "en-NZ",
      "en-PH",
      "en-PK",
      "en-SG",
      "en-TH",
      "en-UK",
      "en-US",
      "en-VN",
      "en-ZA",
      "es-AR",
      "es-CL",
      "es-CO",
      "es-ES",
      "es-MX",
      "es-PE",
      "es-US",
      "et-EE",
      "fi-FI",
      "fr-BE",
      "fr-CA",
      "fr-CH",
      "fr-FR",
      "hr-HR",
      "hu-HU",
      "it-IT",
      "jp-JP",
      "kr-KR",
      "lt-LT",
      "lv-LV",
      "nl-BE",
      "nl-NL",
      "no-NO",
      "pl-PL",
      "pt-BR",
      "pt-PT",
      "ro-RO",
      "ru-RU",
      "sk-SK",
      "sl-SL",
      "sv-SE",
      "tr-TR",
      "tzh-HK",
      "tzh-TW",
      "uk-UA",
      "wt-WT",
      "zh-CN"
    ]
    "supported_languages": {}
  },
  "duckduckgo images": {
  "duckduckgo weather": {
    "all_locale": "wt-wt",
    "custom": {},
    "data_type": "supported_languages",
    "languages": {},
    "custom": {
      "lang_region": {
        "ar-DZ": "ar_DZ",
        "ar-JO": "ar_JO",
        "ar-SA": "ar_SA",
        "bn-IN": "bn_IN",
        "de-CH": "de_CH",
        "en-AU": "en_AU",
        "en-CA": "en_CA",
        "en-GB": "en_GB",
        "es-AR": "es_AR",
        "es-CL": "es_CL",
        "es-CO": "es_CO",
        "es-CR": "es_CR",
        "es-EC": "es_EC",
        "es-MX": "es_MX",
        "es-PE": "es_PE",
        "es-UY": "es_UY",
        "es-VE": "es_VE",
        "fr-BE": "fr_BE",
        "fr-CA": "fr_CA",
        "fr-CH": "fr_CH",
        "nl-BE": "nl_BE",
        "pt-BR": "pt_BR"
      }
    },
    "data_type": "traits_v1",
    "languages": {
      "af": "af_ZA",
      "ar": "ar_EG",
      "ast": "ast_ES",
      "az_Latn": "az_AZ",
      "be": "be_BY",
      "bg": "bg_BG",
      "bn": "bn_BD",
      "br": "br_FR",
      "bs_Latn": "bs_BA",
      "ca": "ca_ES",
      "cs": "cs_CZ",
      "cy": "cy_GB",
      "da": "da_DK",
      "de": "de_DE",
      "el": "el_GR",
      "en": "en_US",
      "eo": "eo_XX",
      "es": "es_ES",
      "et": "et_EE",
      "eu": "eu_ES",
      "fa": "fa_IR",
      "fi": "fi_FI",
      "fil": "tl_PH",
      "fr": "fr_FR",
      "ga": "ga_IE",
      "gd": "gd_GB",
      "gl": "gl_ES",
      "he": "he_IL",
      "hi": "hi_IN",
      "hr": "hr_HR",
      "hu": "hu_HU",
      "hy": "hy_AM",
      "id": "id_ID",
      "is": "is_IS",
      "it": "it_IT",
      "ja": "ja_JP",
      "kab": "kab_DZ",
      "kn": "kn_IN",
      "ko": "ko_KR",
      "ku": "ku",
      "kw": "kw_GB",
      "lt": "lt_LT",
      "lv": "lv_LV",
      "ml": "ml_IN",
      "mr": "mr_IN",
      "ms": "ms_MY",
      "nb": "nb_NO",
      "nl": "nl_NL",
      "nn": "nn_NO",
      "pl": "pl_PL",
      "pt": "pt_PT",
      "ro": "ro_RO",
      "ru": "ru_RU",
      "sc": "sc_IT",
      "si": "si_LK",
      "sk": "sk_SK",
      "sl": "sl_SI",
      "sq": "sq_AL",
      "sr_Cyrl": "sr_RS",
      "sv": "sv_SE",
      "ta": "ta_IN",
      "te": "te_IN",
      "th": "th_TH",
      "tr": "tr_TR",
      "uk": "uk_UA",
      "ur": "ur_PK",
      "vi": "vi_VN",
      "zh_Hans": "zh_CN",
      "zh_Hant": "zh_TW"
    },
    "regions": {
      "ar-SA": "xa-ar",
      "bg-BG": "bg-bg",
@@ -807,71 +964,7 @@
      "zh-HK": "hk-tzh",
      "zh-TW": "tw-tzh"
    },
    "supported_languages": [
      "ar-XA",
      "bg-BG",
      "ca-CT",
      "ca-ES",
      "cs-CZ",
      "da-DK",
      "de-AT",
      "de-CH",
      "de-DE",
      "el-GR",
      "en-AU",
      "en-CA",
      "en-ID",
      "en-IE",
      "en-IL",
      "en-IN",
      "en-MY",
      "en-NZ",
      "en-PH",
      "en-PK",
      "en-SG",
      "en-TH",
      "en-UK",
      "en-US",
      "en-VN",
      "en-ZA",
      "es-AR",
      "es-CL",
      "es-CO",
      "es-ES",
      "es-MX",
      "es-PE",
      "es-US",
      "et-EE",
      "fi-FI",
      "fr-BE",
      "fr-CA",
      "fr-CH",
      "fr-FR",
      "hr-HR",
      "hu-HU",
      "it-IT",
      "jp-JP",
      "kr-KR",
      "lt-LT",
      "lv-LV",
      "nl-BE",
      "nl-NL",
      "no-NO",
      "pl-PL",
      "pt-BR",
      "pt-PT",
      "ro-RO",
      "ru-RU",
      "sk-SK",
      "sl-SL",
      "sv-SE",
      "tr-TR",
      "tzh-HK",
      "tzh-TW",
      "uk-UA",
      "wt-WT",
      "zh-CN"
    ]
    "supported_languages": {}
  },
  "google": {
    "all_locale": "ZZ",
+257 −86

File changed.

Preview size limit exceeded, changes collapsed.

+22 −14
Original line number Diff line number Diff line
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""DuckDuckGo (Instant Answer API)
"""
DuckDuckGo Instant Answer API
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The `DDG-API <https://duckduckgo.com/api>`__ is no longer documented but from
reverse engineering we can see that some services (e.g. instant answers) still
in use from the DDG search engine.

As far we can say the *instant answers* API does not support languages, or at
least we could not find out how language support should work.  It seems that
most of the features are based on English terms.

"""

import json
from typing import TYPE_CHECKING

from urllib.parse import urlencode, urlparse, urljoin
from lxml import html

from searx.data import WIKIDATA_UNITS
from searx.engines.duckduckgo import language_aliases
from searx.engines.duckduckgo import (  # pylint: disable=unused-import
    fetch_traits,
    _fetch_supported_languages,
    supported_languages_url,
)
from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function
from searx.utils import extract_text, html_to_text, get_string_replaces_function
from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom

if TYPE_CHECKING:
    import logging

    logger: logging.Logger

# about
about = {
    "website": 'https://duckduckgo.com/',
@@ -38,7 +48,7 @@ replace_http_by_https = get_string_replaces_function({'http:': 'https:'})


def is_broken_text(text):
    """duckduckgo may return something like "<a href="xxxx">http://somewhere Related website<a/>"
    """duckduckgo may return something like ``<a href="xxxx">http://somewhere Related website<a/>``

    The href URL is broken, the "Related website" may contains some HTML.

@@ -63,8 +73,6 @@ def result_to_text(text, htmlResult):

def request(query, params):
    params['url'] = URL.format(query=urlencode({'q': query}))
    language = match_language(params['language'], supported_languages, language_aliases)
    language = language.split('-')[0]
    return params


@@ -72,7 +80,7 @@ def response(resp):
    # pylint: disable=too-many-locals, too-many-branches, too-many-statements
    results = []

    search_res = json.loads(resp.text)
    search_res = resp.json()

    # search_res.get('Entity') possible values (not exhaustive) :
    # * continent / country / department / location / waterfall
@@ -236,7 +244,7 @@ def unit_to_str(unit):


def area_to_str(area):
    """parse {'unit': 'http://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}"""
    """parse ``{'unit': 'https://www.wikidata.org/entity/Q712226', 'amount': '+20.99'}``"""
    unit = unit_to_str(area.get('unit'))
    if unit is not None:
        try:
Loading