[fix] prevent multiple, parallel initializations of tables in the cache DB (#4991)

Depending on the respective runtime behavior, it could happen that the initial
loading of the DB tables in the cache was performed multiple times and in
parallel. The concurrent accesses then led to the `sqlite3.OperationalError:
database is locked` exception as in #4951.

Since this problem depends significantly on the runtimes (e.g., how long it
takes to retrieve the content for a table), this error could not be observed in
all installations.

Closes: https://github.com/searxng/searxng/issues/4951

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2025-07-09 17:32:10 +02:00 committed by GitHub
parent f798ddd492
commit 2fe8540903
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 13 additions and 4 deletions

View file

@ -24,8 +24,9 @@ class CurrenciesDB:
def init(self):
if self.cache.properties("currencies loaded") != "OK":
self.load()
# To avoid parallel initializations, the property is set first
self.cache.properties.set("currencies loaded", "OK")
self.load()
# F I X M E:
# do we need a maintenance .. rember: database is stored
# in /tmp and will be rebuild during the reboot anyway

View file

@ -7,7 +7,6 @@ import typing
__all__ = ["TrackerPatternsDB"]
import re
import pathlib
from collections.abc import Iterator
from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
@ -22,7 +21,6 @@ class TrackerPatternsDB:
# pylint: disable=missing-class-docstring
ctx_name = "data_tracker_patterns"
json_file = pathlib.Path(__file__).parent / "tracker_patterns.json"
CLEAR_LIST_URL = [
# ClearURL rule lists, the first one that responds HTTP 200 is used
@ -42,8 +40,9 @@ class TrackerPatternsDB:
def init(self):
if self.cache.properties("tracker_patterns loaded") != "OK":
self.load()
# To avoid parallel initializations, the property is set first
self.cache.properties.set("tracker_patterns loaded", "OK")
self.load()
# F I X M E:
# do we need a maintenance .. rember: database is stored
# in /tmp and will be rebuild during the reboot anyway

View file

@ -13,6 +13,7 @@ from searx.data import TRACKER_PATTERNS
from . import Plugin, PluginInfo
if typing.TYPE_CHECKING:
import flask
from searx.search import SearchWithPlugins
from searx.extended_types import SXNG_Request
from searx.result_types import Result, LegacyResult
@ -37,6 +38,10 @@ class SXNGPlugin(Plugin):
preference_section="privacy",
)
def init(self, app: "flask.Flask") -> bool:
TRACKER_PATTERNS.init()
return True
def on_result(self, request: "SXNG_Request", search: "SearchWithPlugins", result: Result) -> bool:
result.filter_urls(self.filter_url_field)

View file

@ -24,6 +24,10 @@ class OnlineCurrencyProcessor(OnlineProcessor):
engine_type = 'online_currency'
def initialize(self):
CURRENCIES.init()
super().initialize()
def get_params(self, search_query, engine_category):
"""Returns a set of :ref:`request params <engine request online_currency>`
or ``None`` if search query does not match to :py:obj:`parser_re`."""