[enh] settings.yml: add use_default_settings option (2nd version)

tests/unit/settings/syntaxerror_settings.yml

@@ -0,0 +1,2 @@
+Test:
+  **********

tests/unit/settings/user_settings.yml

@@ -0,0 +1,111 @@
+general:
+    debug : False
+    instance_name : "searx"
+
+search:
+    safe_search : 0
+    autocomplete : ""
+    default_lang : ""
+    ban_time_on_fail : 5
+    max_ban_time_on_fail : 120
+
+server:
+    port : 9000
+    bind_address : "0.0.0.0"
+    secret_key : "user_settings_secret"
+    base_url : False
+    image_proxy : False
+    http_protocol_version : "1.0"
+    method: "POST"
+    default_http_headers:
+        X-Content-Type-Options : nosniff
+        X-XSS-Protection : 1; mode=block
+        X-Download-Options : noopen
+        X-Robots-Tag : noindex, nofollow
+        Referrer-Policy : no-referrer
+
+ui:
+    static_path : ""
+    templates_path : ""
+    default_theme : oscar
+    default_locale : ""
+    theme_args :
+        oscar_style : logicodev
+
+engines:
+  - name : wikidata
+    engine : wikidata
+    shortcut : wd
+    timeout : 3.0
+    weight : 2
+
+  - name : wikibooks
+    engine : mediawiki
+    shortcut : wb
+    categories : general
+    base_url : "https://{language}.wikibooks.org/"
+    number_of_results : 5
+    search_type : text
+
+  - name : wikinews
+    engine : mediawiki
+    shortcut : wn
+    categories : news
+    base_url : "https://{language}.wikinews.org/"
+    number_of_results : 5
+    search_type : text
+
+  - name : wikiquote
+    engine : mediawiki
+    shortcut : wq
+    categories : general
+    base_url : "https://{language}.wikiquote.org/"
+    number_of_results : 5
+    search_type : text
+
+locales:
+    en : English
+    ar : العَرَبِيَّة (Arabic)
+    bg : Български (Bulgarian)
+    bo : བོད་སྐད་ (Tibetian)
+    ca : Català (Catalan)
+    cs : Čeština (Czech)
+    cy : Cymraeg (Welsh)
+    da : Dansk (Danish)
+    de : Deutsch (German)
+    el_GR : Ελληνικά (Greek_Greece)
+    eo : Esperanto (Esperanto)
+    es : Español (Spanish)
+    et : Eesti (Estonian)
+    eu : Euskara (Basque)
+    fa_IR : (fārsī) فارسى (Persian)
+    fi : Suomi (Finnish)
+    fil : Wikang Filipino (Filipino)
+    fr : Français (French)
+    gl : Galego (Galician)
+    he : עברית (Hebrew)
+    hr : Hrvatski (Croatian)
+    hu : Magyar (Hungarian)
+    ia : Interlingua (Interlingua)
+    it : Italiano (Italian)
+    ja : 日本語 (Japanese)
+    lt : Lietuvių (Lithuanian)
+    nl : Nederlands (Dutch)
+    nl_BE : Vlaams (Dutch_Belgium)
+    oc : Lenga D'òc (Occitan)
+    pl : Polski (Polish)
+    pt : Português (Portuguese)
+    pt_BR : Português (Portuguese_Brazil)
+    ro : Română (Romanian)
+    ru : Русский (Russian)
+    sk : Slovenčina (Slovak)
+    sl : Slovenski (Slovene)
+    sr : српски (Serbian)
+    sv : Svenska (Swedish)
+    te : తెలుగు (telugu)
+    ta : தமிழ் (Tamil)
+    tr : Türkçe (Turkish)
+    uk : українська мова (Ukrainian)
+    vi : tiếng việt (Vietnamese)
+    zh : 中文 (Chinese)
+    zh_TW : 國語 (Taiwanese Mandarin)

tests/unit/settings/user_settings_keep_only.yml

@@ -0,0 +1,14 @@
+use_default_settings:
+    engines:
+        keep_only:
+            - wikibooks
+            - wikinews
+server:
+    secret_key: "user_secret_key"
+    bind_address: "0.0.0.0"
+    default_http_headers:
+        Custom-Header: Custom-Value
+engines:
+    - name: wikipedia
+    - name: newengine
+      engine: dummy

tests/unit/settings/user_settings_remove.yml

@@ -0,0 +1,10 @@
+use_default_settings:
+    engines:
+        remove:
+            - wikibooks
+            - wikinews
+server:
+    secret_key: "user_secret_key"
+    bind_address: "0.0.0.0"
+    default_http_headers:
+        Custom-Header: Custom-Value

tests/unit/settings/user_settings_remove2.yml

@@ -0,0 +1,15 @@
+use_default_settings:
+    engines:
+        remove:
+            - wikibooks
+            - wikinews
+server:
+    secret_key: "user_secret_key"
+    bind_address: "0.0.0.0"
+    default_http_headers:
+        Custom-Header: Custom-Value
+engines:
+    - name: wikipedia
+      tokens: ['secret_token']
+    - name: newengine
+      engine: dummy

tests/unit/settings/user_settings_simple.yml

@@ -0,0 +1,6 @@
+use_default_settings: True
+server:
+    secret_key: "user_secret_key"
+    bind_address: "0.0.0.0"
+    default_http_headers:
+        Custom-Header: Custom-Value

tests/unit/test_settings_loader.py

@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+from os.path import dirname, join, abspath
+from unittest.mock import patch
+
+from searx.testing import SearxTestCase
+from searx.exceptions import SearxSettingsException
+from searx import settings_loader
+
+
+test_dir = abspath(dirname(__file__))
+
+
+class TestLoad(SearxTestCase):
+
+    def test_load_zero(self):
+        with self.assertRaises(SearxSettingsException):
+            settings_loader.load_yaml('/dev/zero')
+
+        with self.assertRaises(SearxSettingsException):
+            settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
+
+        with self.assertRaises(SearxSettingsException):
+            settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
+
+    def test_check_settings_yml(self):
+        self.assertIsNone(settings_loader.check_settings_yml('/dev/zero'))
+
+        bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
+        self.assertEqual(settings_loader.check_settings_yml(bad_settings_path), bad_settings_path)
+
+
+class TestDefaultSettings(SearxTestCase):
+
+    def test_load(self):
+        settings, msg = settings_loader.load_settings(load_user_setttings=False)
+        self.assertTrue(msg.startswith('load the default settings from'))
+        self.assertFalse(settings['general']['debug'])
+        self.assertTrue(isinstance(settings['general']['instance_name'], str))
+        self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
+        self.assertTrue(isinstance(settings['server']['port'], int))
+        self.assertTrue(isinstance(settings['server']['bind_address'], str))
+        self.assertTrue(isinstance(settings['engines'], list))
+        self.assertTrue(isinstance(settings['locales'], dict))
+        self.assertTrue(isinstance(settings['doi_resolvers'], dict))
+        self.assertTrue(isinstance(settings['default_doi_resolver'], str))
+
+
+class TestUserSettings(SearxTestCase):
+
+    def test_is_use_default_settings(self):
+        self.assertFalse(settings_loader.is_use_default_settings({}))
+        self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': True}))
+        self.assertTrue(settings_loader.is_use_default_settings({'use_default_settings': {}}))
+        with self.assertRaises(ValueError):
+            self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 1}))
+        with self.assertRaises(ValueError):
+            self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
+
+    def test_user_settings_not_found(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': '/dev/null'}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('load the default settings from'))
+            self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
+
+    def test_user_settings(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('merge the default settings'))
+            self.assertEqual(settings['server']['secret_key'], "user_secret_key")
+            self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
+
+    def test_user_settings_remove(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('merge the default settings'))
+            self.assertEqual(settings['server']['secret_key'], "user_secret_key")
+            self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
+            engine_names = [engine['name'] for engine in settings['engines']]
+            self.assertNotIn('wikinews', engine_names)
+            self.assertNotIn('wikibooks', engine_names)
+            self.assertIn('wikipedia', engine_names)
+
+    def test_user_settings_remove2(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('merge the default settings'))
+            self.assertEqual(settings['server']['secret_key'], "user_secret_key")
+            self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
+            engine_names = [engine['name'] for engine in settings['engines']]
+            self.assertNotIn('wikinews', engine_names)
+            self.assertNotIn('wikibooks', engine_names)
+            self.assertIn('wikipedia', engine_names)
+            wikipedia = list(filter(lambda engine: (engine.get('name')) == 'wikipedia', settings['engines']))
+            self.assertEqual(wikipedia[0]['engine'], 'wikipedia')
+            self.assertEqual(wikipedia[0]['tokens'], ['secret_token'])
+            newengine = list(filter(lambda engine: (engine.get('name')) == 'newengine', settings['engines']))
+            self.assertEqual(newengine[0]['engine'], 'dummy')
+
+    def test_user_settings_keep_only(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('merge the default settings'))
+            engine_names = [engine['name'] for engine in settings['engines']]
+            self.assertEqual(engine_names, ['wikibooks', 'wikinews', 'wikipedia', 'newengine'])
+            # wikipedia has been removed, then added again with the "engine" section of user_settings_keep_only.yml
+            self.assertEqual(len(settings['engines'][2]), 1)
+
+    def test_custom_settings(self):
+        with patch.dict(settings_loader.environ,
+                        {'SEARX_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}):
+            settings, msg = settings_loader.load_settings()
+            self.assertTrue(msg.startswith('load the user settings from'))
+            self.assertEqual(settings['server']['port'], 9000)
+            self.assertEqual(settings['server']['secret_key'], "user_settings_secret")
+            engine_names = [engine['name'] for engine in settings['engines']]
+            self.assertEqual(engine_names, ['wikidata', 'wikibooks', 'wikinews', 'wikiquote'])