[mod] data: implement a simple currencies (SQL) database (#4836)

To reduce the memory footprint, this patch no longer loads the JSON data
completely into memory.  Instead, there is an SQL database based on
`ExpireCacheSQLite`.

The class CurrenciesDB is a simple DB application that encapsulates the
DB (queries and initialization) and provides convenient methods like
`name_to_iso4217` and `iso4217_to_name`.

Related:

- https://github.com/searxng/searxng/discussions/1892
- https://github.com/searxng/searxng/pull/3458#issuecomment-2900807671
- https://github.com/searxng/searxng/pull/4650

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2025-05-25 10:40:57 +02:00 committed by GitHub
parent e46187e3ce
commit 848c8d0544
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 106 additions and 35 deletions

View file

@ -4,21 +4,17 @@
make data.all
"""
from __future__ import annotations
__all__ = ["ahmia_blacklist_loader"]
import json
from pathlib import Path
import typing
from searx import logger
from .core import log, data_dir
from .currencies import CurrenciesDB
log = logger.getChild("data")
data_dir = Path(__file__).parent
CURRENCIES: dict[str, typing.Any]
CURRENCIES: CurrenciesDB
USER_AGENTS: dict[str, typing.Any]
EXTERNAL_URLS: dict[str, typing.Any]
WIKIDATA_UNITS: dict[str, typing.Any]
@ -29,7 +25,7 @@ ENGINE_TRAITS: dict[str, typing.Any]
LOCALES: dict[str, typing.Any]
lazy_globals = {
"CURRENCIES": None,
"CURRENCIES": CurrenciesDB(),
"USER_AGENTS": None,
"EXTERNAL_URLS": None,
"WIKIDATA_UNITS": None,
@ -41,7 +37,6 @@ lazy_globals = {
}
data_json_files = {
"CURRENCIES": "currencies.json",
"USER_AGENTS": "useragents.json",
"EXTERNAL_URLS": "external_urls.json",
"WIKIDATA_UNITS": "wikidata_units.json",
@ -63,6 +58,7 @@ def __getattr__(name):
return data
log.debug("init searx.data.%s", name)
with open(data_dir / data_json_files[name], encoding='utf-8') as f:
lazy_globals[name] = json.load(f)