From 43ade054b19501d00ff7a75e3f7599522a4ccdf0 Mon Sep 17 00:00:00 2001 From: Bnyro Date: Tue, 24 Jun 2025 22:34:24 +0200 Subject: [PATCH] [feat] cache: implement redis cache as alternative to sqlite --- searx/cache.py | 110 ++++++++++++++++++++++++++++++++++++++++++++- searx/settings.yml | 2 +- 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/searx/cache.py b/searx/cache.py index 16386838f..f87888625 100644 --- a/searx/cache.py +++ b/searx/cache.py @@ -28,6 +28,8 @@ import msgspec from searx import sqlitedb from searx import logger from searx import get_setting +from searx import redisdb, redislib +from searx.sqlitedb import SQLiteProperties log = logger.getChild("cache") @@ -149,7 +151,7 @@ class ExpireCache(abc.ABC): @abc.abstractmethod def get(self, key: str, default=None, ctx: str | None = None) -> typing.Any: - """Return *value* of *key*. If key is unset, ``None`` is returned.""" + """Return *value* of *key*. If key is unset, ``default`` is returned.""" @abc.abstractmethod def maintenance(self, force: bool = False, truncate: bool = False) -> bool: @@ -164,6 +166,10 @@ class ExpireCache(abc.ABC): password has changed. """ + @abc.abstractmethod + def pairs(self, ctx: str) -> Iterator[tuple[str, typing.Any]]: + """Iterate over key/value pairs from table given by argument ``ctx``.""" + @abc.abstractmethod def state(self) -> ExpireCacheStats: """Returns a :py:obj:`ExpireCacheStats`, which provides information @@ -179,7 +185,7 @@ class ExpireCache(abc.ABC): types could be implemented in the future, e.g. a Valkey (Redis) adapter. """ - return ExpireCacheSQLite(cfg) + return ExpireCacheRedis(cfg) @staticmethod def normalize_name(name: str) -> str: @@ -418,3 +424,103 @@ class ExpireCacheSQLite(sqlitedb.SQLiteAppl, ExpireCache): for row in self.DB.execute(f"SELECT key, value, expire FROM {table}"): cached_items[table].append((row[0], self.deserialize(row[1]), row[2])) return ExpireCacheStats(cached_items=cached_items) + + +class ExpireCacheRedis(ExpireCache): + """Key/value database implementation using Redis as backend. + + Redis is especially made for the purpose of storing key/value pairs in a + database, so it might have some small performance bit over using the SQLite + backend. + + Please note that this engine doesn't use ``db_url``. Instead it uses + the url set with the key ``redis.url`` in ``settings.yml``!""" + + EXPIRE_CACHE_KEY_PREFIX = "ExpireCache" + + def __init__(self, cfg: ExpireCacheCfg): + """An instance of the SQLite expire cache is build up from a + :py:obj:`config `.""" + + self.cfg = cfg + self.properties = SQLiteProperties(cfg.db_url) + + def _get_cache_hash_key(self, key: str, ctx: str | None) -> str: + """Get the hash key that is used to uniquely identify the given pair (key, cfg) + in the cache.""" + hash = self.secret_hash(key) + return f"{self.EXPIRE_CACHE_KEY_PREFIX}|{ctx or ''}|{hash}" + + def _get_keys_iterator(self): + """Get an iterator over the keys in this cache.""" + c = redisdb.client() + + for key in list(c.scan_iter()): + key = str(key) + key_parts = key.split("|", 2) + if len(key_parts) != 3: + continue + + prefix, _ctx, _keyhash = key_parts + if prefix == self.EXPIRE_CACHE_KEY_PREFIX: + yield key + + def set(self, key: str, value: typing.Any, expire: int | None, ctx: str | None = None) -> bool: + """Set *key* to *value*. To set a timeout on key use argument + ``expire`` (in sec.). If expire is unset the default is taken from + :py:obj:`ExpireCacheCfg.MAXHOLD_TIME`. After the timeout has expired, + the key will automatically be deleted. + """ + c = redisdb.client() + cache_key = self._get_cache_hash_key(key, ctx) + + value = self.serialize(value) + c.set(cache_key, value, ex=expire) + + def get(self, key: str, default=None, ctx: str | None = None) -> typing.Any: + """Return *value* of *key*. If key is unset, ``default`` is returned.""" + c = redisdb.client() + cache_key = self._get_cache_hash_key(key, ctx) + + value = c.get(cache_key) + if value or value == b'': + return self.deserialize(value) + + return default + + def maintenance(self, force: bool = False, truncate: bool = False) -> bool: + """Performs maintenance on the cache. + + Redis automatically removes expired entries, so only ``truncate`` has an effect, + i.e. it deletes all entries. + """ + c = redisdb.client() + + for key in self._get_keys_iterator(): + c.delete(key) + + def pairs(self, ctx: str): + found_pairs = [] + for key in self._get_keys_iterator(): + _prefix, key_ctx, _keyhash = key.split("|", 2) + if key_ctx == ctx: + value = self.get(key, ctx) + found_pairs.append((key, value)) + return found_pairs + + def state(self) -> ExpireCacheStats: + """Returns a :py:obj:`ExpireCacheStats`, which provides information + about the status of the cache.""" + + ctxs = {} + for key in self._get_keys_iterator(): + _prefix, ctx, _keyhash = key.split("|", 2) + + if ctx not in ctxs: + ctxs[ctx] = [] + + value = self.get(key) + # querying the expiry time is not supported, thus it's always set to None + ctxs[ctx].push((key, value, None)) + + return ExpireCacheStats(ctxs) diff --git a/searx/settings.yml b/searx/settings.yml index 76200d4be..6ab787492 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -113,7 +113,7 @@ server: redis: # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}. # https://docs.searxng.org/admin/settings/settings_redis.html#settings-redis - url: false + url: redis://127.0.0.1:6379 ui: # Custom static path - leave it blank if you didn't change