[mod] remove option ui.static_use_hash (cache busting) (#5004)

Cache busting has caused serious problems for users in the past, here are two
examples:

- https://github.com/searxng/searxng/issues/4419
- https://github.com/searxng/searxng/issues/4481

And it makes development and deployment significantly more complex because it
binds the client side to the server side:

- https://github.com/searxng/searxng/pull/4466

In the light of a decoupled development of the WEB clients from the server side:

- https://github.com/searxng/searxng/pull/4988

is it appropriate to abandon this feature. In fact,  it has been ineffective
since #4436 anyway.

However, the benefit has always been questionable, since at best only a few kB
of data are saved (at least in the context of an image_proxy, the effect is below
the detection limit). Ultimately, the client is responsible for caching.

Related: https://github.com/searxng/searxng/issues?q=label%3A%22clear%20browser%20cache%22

Closes: https://github.com/searxng/searxng/pull/4466
Closes: https://github.com/searxng/searxng/issues/1326
Closes: https://github.com/searxng/searxng/issues/964

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/settings.yml

@@ -119,8 +119,6 @@ valkey:
 ui:
   # Custom static path - leave it blank if you didn't change
   static_path: ""
-  # Is overwritten by ${SEARXNG_STATIC_USE_HASH}.
-  static_use_hash: false
   # Custom templates path - leave it blank if you didn't change
   templates_path: ""
   # query_in_title: When true, the result page's titles contains the query

searx/settings_defaults.py

@@ -194,7 +194,6 @@ SCHEMA = {
     },
     'ui': {
         'static_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'static')),
-        'static_use_hash': SettingsValue(bool, False, 'SEARXNG_STATIC_USE_HASH'),
         'templates_path': SettingsDirectoryValue(str, os.path.join(searx_dir, 'templates')),
         'default_theme': SettingsValue(str, 'simple'),
         'default_locale': SettingsValue(str, ''),

searx/webapp.py

@@ -76,7 +76,6 @@ from searx.engines import (
 from searx import webutils
 from searx.webutils import (
     highlight_content,
-    get_static_files,
     get_result_templates,
     get_themes,
     exception_classname_to_text,
@@ -131,7 +130,6 @@ warnings.simplefilter("always")
 
 # about static
 logger.debug('static directory is %s', settings['ui']['static_path'])
-static_files = get_static_files(settings['ui']['static_path'])
 
 # about templates
 logger.debug('templates directory is %s', settings['ui']['templates_path'])
@@ -239,25 +237,44 @@ def get_result_template(theme_name: str, template_name: str):
     return 'result_templates/' + template_name
 
 
+_STATIC_FILES: list[str] = []
+
+
 def custom_url_for(endpoint: str, **values):
-    suffix = ""
-    if endpoint == 'static' and values.get('filename'):
-        file_hash = static_files.get(values['filename'])
-        if not file_hash:
+    global _STATIC_FILES  # pylint: disable=global-statement
+    if not _STATIC_FILES:
+        _STATIC_FILES = webutils.get_static_file_list()
+
+    if endpoint == "static" and values.get("filename"):
+
+        # We need to verify the "filename" argument: in the jinja templates
+        # there could be call like:
+        #     url_for('static', filename='img/favicon.png')
+        # which should map to:
+        #     static/themes/<theme_name>/img/favicon.png
+
+        arg_filename = values["filename"]
+        if arg_filename not in _STATIC_FILES:
             # try file in the current theme
-            theme_name = sxng_request.preferences.get_value('theme')
-            filename_with_theme = "themes/{}/{}".format(theme_name, values['filename'])
-            file_hash = static_files.get(filename_with_theme)
-            if file_hash:
-                values['filename'] = filename_with_theme
-        if get_setting('ui.static_use_hash') and file_hash:
-            suffix = "?" + file_hash
-    if endpoint == 'info' and 'locale' not in values:
-        locale = sxng_request.preferences.get_value('locale')
-        if infopage.INFO_PAGES.get_page(values['pagename'], locale) is None:
+            theme_name = sxng_request.preferences.get_value("theme")
+            arg_filename = f"themes/{theme_name}/{arg_filename}"
+            if arg_filename in _STATIC_FILES:
+                values["filename"] = arg_filename
+
+    if endpoint == "info" and "locale" not in values:
+
+        # We need to verify the "locale" argument: in the jinja templates there
+        # could be call like:
+        #     url_for('info', pagename='about')
+        # which should map to:
+        #     info/<locale>/about
+
+        locale = sxng_request.preferences.get_value("locale")
+        if infopage.INFO_PAGES.get_page(values["pagename"], locale) is None:
             locale = infopage.INFO_PAGES.locale_default
-        values['locale'] = locale
-    return url_for(endpoint, **values) + suffix
+        values["locale"] = locale
+
+    return url_for(endpoint, **values)
 
 
 def image_proxify(url: str):

searx/webutils.py

@@ -12,14 +12,15 @@ import re
 import itertools
 import json
 from datetime import datetime, timedelta
-from typing import Iterable, List, Tuple, Dict, TYPE_CHECKING
+from typing import Iterable, List, Tuple, TYPE_CHECKING
 
 from io import StringIO
 from codecs import getincrementalencoder
 
 from flask_babel import gettext, format_date  # type: ignore
 
-from searx import logger, settings
+from searx import logger, get_setting
+
 from searx.engines import DEFAULT_CATEGORY
 
 if TYPE_CHECKING:
@@ -177,30 +178,22 @@ def get_themes(templates_path):
     return os.listdir(templates_path)
 
 
-def get_hash_for_file(file: pathlib.Path) -> str:
-    m = hashlib.sha1()
-    with file.open('rb') as f:
-        m.update(f.read())
-    return m.hexdigest()
+def get_static_file_list() -> list[str]:
+    file_list = []
+    static_path = pathlib.Path(str(get_setting("ui.static_path")))
 
-
-def get_static_files(static_path: str) -> Dict[str, str]:
-    static_files: Dict[str, str] = {}
-    static_path_path = pathlib.Path(static_path)
-
-    def walk(path: pathlib.Path):
-        for file in path.iterdir():
-            if file.name.startswith('.'):
+    def _walk(path: pathlib.Path):
+        for f in path.iterdir():
+            if f.name.startswith('.'):
                 # ignore hidden file
                 continue
-            if file.is_file():
-                static_files[str(file.relative_to(static_path_path))] = get_hash_for_file(file)
-            if file.is_dir() and file.name not in ('node_modules', 'src'):
-                # ignore "src" and "node_modules" directories
-                walk(file)
+            if f.is_file():
+                file_list.append(str(f.relative_to(static_path)))
+            if f.is_dir():
+                _walk(f)
 
-    walk(static_path_path)
-    return static_files
+    _walk(static_path)
+    return file_list
 
 
 def get_result_templates(templates_path):
@@ -331,7 +324,7 @@ def group_engines_in_tab(engines: Iterable[Engine]) -> List[Tuple[str, Iterable[
     def engine_sort_key(engine):
         return (engine.about.get('language', ''), engine.name)
 
-    tabs = list(settings['categories_as_tabs'].keys())
+    tabs = list(get_setting('categories_as_tabs').keys())
     subgroups = itertools.groupby(sorted(engines, key=get_subgroup), get_subgroup)
     sorted_groups = sorted(((name, list(engines)) for name, engines in subgroups), key=group_sort_key)