[mod] revision of the settings_loader

The intention of this PR is to modernize the settings_loader implementations.
The concept is old (remember, this is partly from 2014), back then we only had
one config file, meanwhile we have had a folder with config files for a very
long time.  Callers can now load a YAML configuration from this folder as
follows ::

    settings_loader.get_yaml_cfg('my-config.yml')

- BTW this is a fix of #3557.

- Further the `existing_filename_or_none` construct dates back to times when
  there was not yet a `pathlib.Path` in all Python versions we supported in the
  past.

- Typehints have been added wherever appropriate

At the same time, this patch should also be downward compatible and not
introduce a new environment variable. The localization of the folder with the
configurations is further based on:

    SEARXNG_SETTINGS_PATH (wich defaults to /etc/searxng/settings.yml)

Which means, the default config folder is `/etc/searxng/`.

ATTENTION: intended functional changes!

 If SEARXNG_SETTINGS_PATH was set and pointed to a not existing file, the
 previous implementation silently loaded the default configuration.  This
 behavior has been changed: if the file or folder does not exist, an
 EnvironmentError exception will be thrown in future.

Closes: https://github.com/searxng/searxng/issues/3557
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

tests/unit/test_settings_loader.py

@@ -1,7 +1,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # pylint: disable=missing-module-docstring
 
-from os.path import dirname, join, abspath
+from pathlib import Path
+
+import os
 from unittest.mock import patch
 
 from searx.exceptions import SearxSettingsException
@@ -9,7 +11,8 @@ from searx import settings_loader
 from tests import SearxTestCase
 
 
-test_dir = abspath(dirname(__file__))
+def _settings(f_name):
+    return str(Path(__file__).parent.absolute() / "settings" / f_name)
 
 
 class TestLoad(SearxTestCase):  # pylint: disable=missing-class-docstring
@@ -18,16 +21,9 @@ class TestLoad(SearxTestCase):  # pylint: disable=missing-class-docstring
             settings_loader.load_yaml('/dev/zero')
 
         with self.assertRaises(SearxSettingsException):
-            settings_loader.load_yaml(join(test_dir, '/settings/syntaxerror_settings.yml'))
+            settings_loader.load_yaml(_settings("syntaxerror_settings.yml"))
 
-        with self.assertRaises(SearxSettingsException):
-            settings_loader.load_yaml(join(test_dir, '/settings/empty_settings.yml'))
-
-    def test_existing_filename_or_none(self):
-        self.assertIsNone(settings_loader.existing_filename_or_none('/dev/zero'))
-
-        bad_settings_path = join(test_dir, 'settings/syntaxerror_settings.yml')
-        self.assertEqual(settings_loader.existing_filename_or_none(bad_settings_path), bad_settings_path)
+        self.assertEqual(settings_loader.load_yaml(_settings("empty_settings.yml")), {})
 
 
 class TestDefaultSettings(SearxTestCase):  # pylint: disable=missing-class-docstring
@@ -55,24 +51,22 @@ class TestUserSettings(SearxTestCase):  # pylint: disable=missing-class-docstrin
             self.assertFalse(settings_loader.is_use_default_settings({'use_default_settings': 0}))
 
     def test_user_settings_not_found(self):
-        with patch.dict(settings_loader.environ, {'SEARXNG_SETTINGS_PATH': '/dev/null'}):
-            settings, msg = settings_loader.load_settings()
-            self.assertTrue(msg.startswith('load the default settings from'))
-            self.assertEqual(settings['server']['secret_key'], "ultrasecretkey")
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("not_exists.yml")}):
+            with self.assertRaises(EnvironmentError):
+                _s, _m = settings_loader.load_settings()
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': "/folder/not/exists"}):
+            with self.assertRaises(EnvironmentError):
+                _s, _m = settings_loader.load_settings()
 
     def test_user_settings(self):
-        with patch.dict(
-            settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'settings/user_settings_simple.yml')}
-        ):
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("user_settings_simple.yml")}):
             settings, msg = settings_loader.load_settings()
             self.assertTrue(msg.startswith('merge the default settings'))
             self.assertEqual(settings['server']['secret_key'], "user_secret_key")
             self.assertEqual(settings['server']['default_http_headers']['Custom-Header'], "Custom-Value")
 
     def test_user_settings_remove(self):
-        with patch.dict(
-            settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove.yml')}
-        ):
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("user_settings_remove.yml")}):
             settings, msg = settings_loader.load_settings()
             self.assertTrue(msg.startswith('merge the default settings'))
             self.assertEqual(settings['server']['secret_key'], "user_secret_key")
@@ -83,9 +77,7 @@ class TestUserSettings(SearxTestCase):  # pylint: disable=missing-class-docstrin
             self.assertIn('wikipedia', engine_names)
 
     def test_user_settings_remove2(self):
-        with patch.dict(
-            settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'settings/user_settings_remove2.yml')}
-        ):
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("user_settings_remove2.yml")}):
             settings, msg = settings_loader.load_settings()
             self.assertTrue(msg.startswith('merge the default settings'))
             self.assertEqual(settings['server']['secret_key'], "user_secret_key")
@@ -101,9 +93,7 @@ class TestUserSettings(SearxTestCase):  # pylint: disable=missing-class-docstrin
             self.assertEqual(newengine[0]['engine'], 'dummy')
 
     def test_user_settings_keep_only(self):
-        with patch.dict(
-            settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'settings/user_settings_keep_only.yml')}
-        ):
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("user_settings_keep_only.yml")}):
             settings, msg = settings_loader.load_settings()
             self.assertTrue(msg.startswith('merge the default settings'))
             engine_names = [engine['name'] for engine in settings['engines']]
@@ -112,9 +102,7 @@ class TestUserSettings(SearxTestCase):  # pylint: disable=missing-class-docstrin
             self.assertEqual(len(settings['engines'][2]), 1)
 
     def test_custom_settings(self):
-        with patch.dict(
-            settings_loader.environ, {'SEARXNG_SETTINGS_PATH': join(test_dir, 'settings/user_settings.yml')}
-        ):
+        with patch.dict(os.environ, {'SEARXNG_SETTINGS_PATH': _settings("user_settings.yml")}):
             settings, msg = settings_loader.load_settings()
             self.assertTrue(msg.startswith('load the user settings from'))
             self.assertEqual(settings['server']['port'], 9000)