Merge pull request #1652 from return42/mod-qwant

Improve qwant engine

4 files changed, 361 insertions, 206 deletions

diff --git a/searx/data/engines_languages.json b/searx/data/engines_languages.json
index 1d4d9b2e..3ee6fad7 100644
--- a/searx/data/engines_languages.json
+++ b/searx/data/engines_languages.json
@@ -1396,170 +1396,155 @@
     "sv",
     "zh"
   ],
-  "qwant": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant images": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant news": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
-  "qwant videos": [
-    "bg-BG",
-    "ca-ES",
-    "cs-CZ",
-    "da-DK",
-    "de-AT",
-    "de-CH",
-    "de-DE",
-    "el-GR",
-    "en-AU",
-    "en-CA",
-    "en-GB",
-    "en-IE",
-    "en-MY",
-    "en-NZ",
-    "en-US",
-    "es-AR",
-    "es-CL",
-    "es-ES",
-    "es-MX",
-    "et-EE",
-    "fi-FI",
-    "fr-BE",
-    "fr-CA",
-    "fr-CH",
-    "fr-FR",
-    "hu-HU",
-    "it-CH",
-    "it-IT",
-    "ko-KR",
-    "nb-NO",
-    "nl-BE",
-    "nl-NL",
-    "pl-PL",
-    "pt-PT",
-    "ro-RO",
-    "sv-SE",
-    "th-TH",
-    "zh-CN",
-    "zh-HK"
-  ],
+  "qwant": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
+  "qwant images": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
+  "qwant news": {
+    "ca-ES": "ca_ES",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pt-PT": "pt_PT"
+  },
+  "qwant videos": {
+    "bg-BG": "bg_BG",
+    "ca-ES": "ca_ES",
+    "cs-CZ": "cs_CZ",
+    "da-DK": "da_DK",
+    "de-AT": "de_AT",
+    "de-CH": "de_CH",
+    "de-DE": "de_DE",
+    "el-GR": "el_GR",
+    "en-AU": "en_AU",
+    "en-CA": "en_CA",
+    "en-GB": "en_GB",
+    "en-IE": "en_IE",
+    "en-MY": "en_MY",
+    "en-NZ": "en_NZ",
+    "en-US": "en_US",
+    "es-AR": "es_AR",
+    "es-CL": "es_CL",
+    "es-ES": "es_ES",
+    "es-MX": "es_MX",
+    "et-EE": "et_EE",
+    "fi-FI": "fi_FI",
+    "fr-BE": "fr_BE",
+    "fr-CA": "fr_CA",
+    "fr-CH": "fr_CH",
+    "fr-FR": "fr_FR",
+    "hu-HU": "hu_HU",
+    "it-CH": "it_CH",
+    "it-IT": "it_IT",
+    "ko-KR": "ko_KR",
+    "nb-NO": "nb_NO",
+    "nl-BE": "nl_BE",
+    "nl-NL": "nl_NL",
+    "pl-PL": "pl_PL",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro_RO",
+    "sv-SE": "sv_SE",
+    "th-TH": "th_TH",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK"
+  },
   "startpage": {
     "af": {
       "alias": "afrikaans"
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index a1799491..6de2176d 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -9,16 +9,16 @@ https://www.qwant.com/ queries.
 This implementation is used by different qwant engines in the settings.yml::
 
   - name: qwant
-    categories: general
+    qwant_categ: web
     ...
   - name: qwant news
-    categories: news
+    qwant_categ: news
     ...
   - name: qwant images
-    categories: images
+    qwant_categ: images
     ...
   - name: qwant videos
-    categories: videos
+    qwant_categ: videos
     ...
 
 """
@@ -30,11 +30,11 @@ from datetime import (
 from json import loads
 from urllib.parse import urlencode
 from flask_babel import gettext
+import babel
 
-from searx.utils import match_language
 from searx.exceptions import SearxEngineAPIException
 from searx.network import raise_for_httperror
-
+from searx.locales import get_engine_locale
 
 # about
 about = {
@@ -50,13 +50,20 @@ about = {
 categories = []
 paging = True
 supported_languages_url = about['website']
+qwant_categ = None  # web|news|inages|videos
 
-category_to_keyword = {
-    'general': 'web',
-    'news': 'news',
-    'images': 'images',
-    'videos': 'videos',
-}
+safesearch = True
+safe_search_map = {0: '&safesearch=0', 1: '&safesearch=1', 2: '&safesearch=2'}
+
+# fmt: off
+qwant_news_locales = [
+    'ca_ad', 'ca_es', 'ca_fr', 'co_fr', 'de_at', 'de_ch', 'de_de', 'en_au',
+    'en_ca', 'en_gb', 'en_ie', 'en_my', 'en_nz', 'en_us', 'es_ad', 'es_ar',
+    'es_cl', 'es_co', 'es_es', 'es_mx', 'es_pe', 'eu_es', 'eu_fr', 'fc_ca',
+    'fr_ad', 'fr_be', 'fr_ca', 'fr_ch', 'fr_fr', 'it_ch', 'it_it', 'nl_be',
+    'nl_nl', 'pt_ad', 'pt_pt',
+]
+# fmt: on
 
 # search-url
 url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={offset}'
@@ -64,10 +71,13 @@ url = 'https://api.qwant.com/v3/search/{keyword}?{query}&count={count}&offset={o
 
 def request(query, params):
     """Qwant search request"""
-    keyword = category_to_keyword[categories[0]]
+
+    if not query:
+        return None
+
     count = 10  # web: count must be equal to 10
 
-    if keyword == 'images':
+    if qwant_categ == 'images':
         count = 50
         offset = (params['pageno'] - 1) * count
         # count + offset must be lower than 250
@@ -78,22 +88,18 @@ def request(query, params):
         offset = min(offset, 40)
 
     params['url'] = url.format(
-        keyword=keyword,
+        keyword=qwant_categ,
         query=urlencode({'q': query}),
         offset=offset,
         count=count,
     )
 
-    # add language tag
-    if params['language'] == 'all':
-        params['url'] += '&locale=en_US'
-    else:
-        language = match_language(
-            params['language'],
-            supported_languages,
-            language_aliases,
-        )
-        params['url'] += '&locale=' + language.replace('-', '_')
+    # add quant's locale
+    q_locale = get_engine_locale(params['language'], supported_languages, default='en_US')
+    params['url'] += '&locale=' + q_locale
+
+    # add safesearch option
+    params['url'] += safe_search_map.get(params['safesearch'], '')
 
     params['raise_for_httperror'] = False
     return params
@@ -103,7 +109,6 @@ def response(resp):
     """Get response from Qwant's search request"""
     # pylint: disable=too-many-locals, too-many-branches, too-many-statements
 
-    keyword = category_to_keyword[categories[0]]
     results = []
 
     # load JSON result
@@ -125,7 +130,7 @@ def response(resp):
     # raise for other errors
     raise_for_httperror(resp)
 
-    if keyword == 'web':
+    if qwant_categ == 'web':
         # The WEB query contains a list named 'mainline'.  This list can contain
         # different result types (e.g. mainline[0]['type'] returns type of the
         # result items in mainline[0]['items']
@@ -136,7 +141,7 @@ def response(resp):
         # result['items'].
         mainline = data.get('result', {}).get('items', [])
         mainline = [
-            {'type': keyword, 'items': mainline},
+            {'type': qwant_categ, 'items': mainline},
         ]
 
     # return empty array if there are no results
@@ -146,7 +151,7 @@ def response(resp):
     for row in mainline:
 
         mainline_type = row.get('type', 'web')
-        if mainline_type != keyword:
+        if mainline_type != qwant_categ:
             continue
 
         if mainline_type == 'ads':
@@ -238,19 +243,43 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
 def _fetch_supported_languages(resp):
-    # list of regions is embedded in page as a js object
-    response_text = resp.text
-    response_text = response_text[response_text.find('INITIAL_PROPS') :]
-    response_text = response_text[response_text.find('{') : response_text.find('</script>')]
-
-    regions_json = loads(response_text)
-
-    supported_languages = []
-    for country, langs in regions_json['locales'].items():
-        for lang in langs['langs']:
-            lang_code = "{lang}-{country}".format(lang=lang, country=country)
-            supported_languages.append(lang_code)
+
+    text = resp.text
+    text = text[text.find('INITIAL_PROPS') :]
+    text = text[text.find('{') : text.find('</script>')]
+
+    q_initial_props = loads(text)
+    q_locales = q_initial_props.get('locales')
+    q_valid_locales = []
+
+    for country, v in q_locales.items():
+        for lang in v['langs']:
+            _locale = "{lang}_{country}".format(lang=lang, country=country)
+
+            if qwant_categ == 'news' and _locale.lower() not in qwant_news_locales:
+                # qwant-news does not support all locales from qwant-web:
+                continue
+
+            q_valid_locales.append(_locale)
+
+    supported_languages = {}
+
+    for q_locale in q_valid_locales:
+        try:
+            locale = babel.Locale.parse(q_locale, sep='_')
+        except babel.core.UnknownLocaleError:
+            print("ERROR: can't determine babel locale of quant's locale %s" % q_locale)
+            continue
+
+        # note: supported_languages (dict)
+        #
+        #   dict's key is a string build up from a babel.Locale object / the
+        #   notation 'xx-XX' (and 'xx') conforms to SearXNG's locale (and
+        #   language) notation and dict's values are the locale strings used by
+        #   the engine.
+
+        searxng_locale = locale.language + '-' + locale.territory  # --> params['language']
+        supported_languages[searxng_locale] = q_locale
 
     return supported_languages
diff --git a/searx/locales.py b/searx/locales.py
index 3deff686..227c9505 100644
--- a/searx/locales.py
+++ b/searx/locales.py
@@ -10,6 +10,8 @@ import pathlib
 
 from babel import Locale
 from babel.support import Translations
+import babel.languages
+import babel.core
 import flask_babel
 import flask
 from flask.ctx import has_request_context
@@ -150,3 +152,138 @@ def locales_initialize(directory=None):
             LOCALE_NAMES[tag] = get_locale_descr(locale, dirname)
             if locale.text_direction == 'rtl':
                 RTL_LOCALES.add(tag)
+
+
+def get_engine_locale(searxng_locale, engine_locales, default=None):
+    """Return engine's language (aka locale) string that best fits to argument
+    ``searxng_locale``.
+
+    Argument ``engine_locales`` is a python dict that maps *SearXNG locales* to
+    corresponding *engine locales*:
+
+      <engine>: {
+          # SearXNG string : engine-string
+          'ca-ES'          : 'ca_ES',
+          'fr-BE'          : 'fr_BE',
+          'fr-CA'          : 'fr_CA',
+          'fr-CH'          : 'fr_CH',
+          'fr'             : 'fr_FR',
+          ...
+          'pl-PL'          : 'pl_PL',
+          'pt-PT'          : 'pt_PT'
+      }
+
+    .. hint::
+
+       The *SearXNG locale* string has to be known by babel!
+
+    If there is no direct 1:1 mapping, this functions tries to narrow down
+    engine's language (locale).  If no value can be determined by these
+    approximation attempts the ``default`` value is returned.
+
+    Assumptions:
+
+    A. When user select a language the results should be optimized according to
+       the selected language.
+
+    B. When user select a language and a territory the results should be
+       optimized with first priority on terrirtory and second on language.
+
+    First approximation rule (*by territory*):
+
+      When the user selects a locale with terrirtory (and a language), the
+      territory has priority over the language.  If any of the offical languages
+      in the terrirtory is supported by the engine (``engine_locales``) it will
+      be used.
+
+    Second approximation rule (*by language*):
+
+      If "First approximation rule" brings no result or the user selects only a
+      language without a terrirtory.  Check in which territories the language
+      has an offical status and if one of these territories is supported by the
+      engine.
+
+    """
+    # pylint: disable=too-many-branches
+
+    engine_locale = engine_locales.get(searxng_locale)
+
+    if engine_locale is not None:
+        # There was a 1:1 mapping (e.g. "fr-BE --> fr_BE" or "fr --> fr_FR"), no
+        # need to narrow language nor territory.
+        return engine_locale
+
+    locale = babel.Locale.parse(searxng_locale, sep='-')
+
+    # SearXNG's selected locale is not supported by the engine ..
+
+    if locale.territory:
+        # Try to narrow by *offical* languages in the territory (??-XX).
+
+        for official_language in babel.languages.get_official_languages(locale.territory, de_facto=True):
+            searxng_locale = official_language + '-' + locale.territory
+            engine_locale = engine_locales.get(searxng_locale)
+            if engine_locale is not None:
+                return engine_locale
+
+    # Engine does not support one of the offical languages in the territory or
+    # there is only a language selected without a territory.
+
+    # Now lets have a look if the searxng_lang (the language selected by the
+    # user) is a offical language in other territories.  If so, check if
+    # engine does support the searxng_lang in this other territory.
+
+    if locale.language:
+
+        searxng_lang = locale.language
+        if locale.script:
+            searxng_lang += '_' + locale.script
+
+        terr_lang_dict = {}
+        for territory, langs in babel.core.get_global("territory_languages").items():
+            if not langs.get(searxng_lang, {}).get('official_status'):
+                continue
+            terr_lang_dict[territory] = langs.get(searxng_lang)
+
+        # first: check fr-FR, de-DE .. is supported by the engine
+
+        territory = locale.language.upper()
+        if terr_lang_dict.get(territory):
+            searxng_locale = locale.language + '-' + territory
+            engine_locale = engine_locales.get(searxng_locale)
+            if engine_locale is not None:
+                return engine_locale
+
+        # second: sort by population_percent and take first match
+
+        # drawback of "population percent": if there is a terrirtory with a
+        #   small number of people (e.g 100) but the majority speaks the
+        #   language, then the percentage migth be 100% (--> 100 people) but in
+        #   a different terrirtory with more people (e.g. 10.000) where only 10%
+        #   speak the language the total amount of speaker is higher (--> 200
+        #   people).
+        #
+        #   By example: The population of Saint-Martin is 33.000, of which 100%
+        #   speak French, but this is less than the 30% of the approximately 2.5
+        #   million Belgian citizens
+        #
+        #   - 'fr-MF', 'population_percent': 100.0, 'official_status': 'official'
+        #   - 'fr-BE', 'population_percent': 38.0, 'official_status': 'official'
+
+        terr_lang_list = []
+        for k, v in terr_lang_dict.items():
+            terr_lang_list.append((k, v))
+
+        for territory, _lang in sorted(terr_lang_list, key=lambda item: item[1]['population_percent'], reverse=True):
+            searxng_locale = locale.language + '-' + territory
+            engine_locale = engine_locales.get(searxng_locale)
+            if engine_locale is not None:
+                return engine_locale
+
+    # No luck: narrow by "language from territory" and "territory from language"
+    # does not fit to a locale supported by the engine.
+
+    if engine_locale is None:
+        engine_locale = default
+
+    return default
diff --git a/searx/settings.yml b/searx/settings.yml
index 6cae30c4..9530a442 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -1198,6 +1198,7 @@ engines:
       results: HTML
 
   - name: qwant
+    qwant_categ: web
     engine: qwant
     shortcut: qw
     categories: [general, web]
@@ -1206,6 +1207,7 @@ engines:
       rosebud: *test_rosebud
 
   - name: qwant news
+    qwant_categ: news
     engine: qwant
     shortcut: qwn
     categories: news
@@ -1213,6 +1215,7 @@ engines:
     network: qwant
 
   - name: qwant images
+    qwant_categ: images
     engine: qwant
     shortcut: qwi
     categories: [images, web]
@@ -1220,6 +1223,7 @@ engines:
     network: qwant
 
   - name: qwant videos
+    qwant_categ: videos
     engine: qwant
     shortcut: qwv
     categories: [videos, web]