mirror of
https://github.com/searxng/searxng.git
synced 2025-07-19 03:09:25 +02:00
[mod] migrate from Redis to Valkey
This patch migrates from `redis==5.2.1` [1] to `valkey==6.1.0` [2].
The migration to valkey is necessary because the company behind Redis has decided
to abandon the open source license. After experiencing a drop in user numbers,
they now want to run it under a dual license again. But this move demonstrates
once again how unreliable the company is and how it treats open source
developers.
To review first, read the docs::
$ make docs.live
Follow the instructions to remove redis:
- http://0.0.0.0:8000/admin/settings/settings_redis.html
Config and install a local valkey DB:
- http://0.0.0.0:8000/admin/settings/settings_valkey.html
[1] https://pypi.org/project/redis/
[2] https://pypi.org/project/valkey/
Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
HLFH
Markus Heiser
parent
fe52290e65
commit
36538e6a20
43 changed files with 468 additions and 724 deletions
|
|
@ -12,11 +12,11 @@ from ._helpers import too_many_requests
|
|||
|
||||
__all__ = ['dump_request', 'get_network', 'get_real_ip', 'too_many_requests']
|
||||
|
||||
redis_client = None
|
||||
valkey_client = None
|
||||
cfg = None
|
||||
|
||||
|
||||
def init(_cfg, _redis_client):
|
||||
global redis_client, cfg # pylint: disable=global-statement
|
||||
redis_client = _redis_client
|
||||
def init(_cfg, _valkey_client):
|
||||
global valkey_client, cfg # pylint: disable=global-statement
|
||||
valkey_client = _valkey_client
|
||||
cfg = _cfg
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ Method ``ip_limit``
|
|||
|
||||
The ``ip_limit`` method counts request from an IP in *sliding windows*. If
|
||||
there are to many requests in a sliding window, the request is evaluated as a
|
||||
bot request. This method requires a redis DB and needs a HTTP X-Forwarded-For_
|
||||
header. To take privacy only the hash value of an IP is stored in the redis DB
|
||||
bot request. This method requires a valkey DB and needs a HTTP X-Forwarded-For_
|
||||
header. To take privacy only the hash value of an IP is stored in the valkey DB
|
||||
and at least for a maximum of 10 minutes.
|
||||
|
||||
The :py:obj:`.link_token` method can be used to investigate whether a request is
|
||||
|
|
@ -46,8 +46,8 @@ import flask
|
|||
import werkzeug
|
||||
|
||||
from searx.extended_types import SXNG_Request
|
||||
from searx import redisdb
|
||||
from searx.redislib import incr_sliding_window, drop_counter
|
||||
from searx import valkeydb
|
||||
from searx.valkeylib import incr_sliding_window, drop_counter
|
||||
|
||||
from . import link_token
|
||||
from . import config
|
||||
|
|
@ -97,14 +97,14 @@ def filter_request(
|
|||
) -> werkzeug.Response | None:
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
redis_client = redisdb.client()
|
||||
valkey_client = valkeydb.client()
|
||||
|
||||
if network.is_link_local and not cfg['botdetection.ip_limit.filter_link_local']:
|
||||
logger.debug("network %s is link-local -> not monitored by ip_limit method", network.compressed)
|
||||
return None
|
||||
|
||||
if request.args.get('format', 'html') != 'html':
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.API_WINDOW:' + network.compressed, API_WINDOW)
|
||||
c = incr_sliding_window(valkey_client, 'ip_limit.API_WINDOW:' + network.compressed, API_WINDOW)
|
||||
if c > API_MAX:
|
||||
return too_many_requests(network, "too many request in API_WINDOW")
|
||||
|
||||
|
|
@ -114,12 +114,12 @@ def filter_request(
|
|||
|
||||
if not suspicious:
|
||||
# this IP is no longer suspicious: release ip again / delete the counter of this IP
|
||||
drop_counter(redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed)
|
||||
drop_counter(valkey_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed)
|
||||
return None
|
||||
|
||||
# this IP is suspicious: count requests from this IP
|
||||
c = incr_sliding_window(
|
||||
redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed, SUSPICIOUS_IP_WINDOW
|
||||
valkey_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed, SUSPICIOUS_IP_WINDOW
|
||||
)
|
||||
if c > SUSPICIOUS_IP_MAX:
|
||||
logger.error("BLOCK: too many request from %s in SUSPICIOUS_IP_WINDOW (redirect to /)", network)
|
||||
|
|
@ -127,22 +127,22 @@ def filter_request(
|
|||
response.headers["Cache-Control"] = "no-store, max-age=0"
|
||||
return response
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
c = incr_sliding_window(valkey_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
if c > BURST_MAX_SUSPICIOUS:
|
||||
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX_SUSPICIOUS)")
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
c = incr_sliding_window(valkey_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
if c > LONG_MAX_SUSPICIOUS:
|
||||
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX_SUSPICIOUS)")
|
||||
|
||||
return None
|
||||
|
||||
# vanilla limiter without extensions counts BURST_MAX and LONG_MAX
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
c = incr_sliding_window(valkey_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
if c > BURST_MAX:
|
||||
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX)")
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
c = incr_sliding_window(valkey_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
if c > LONG_MAX:
|
||||
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX)")
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ a ping by request a static URL.
|
|||
|
||||
.. note::
|
||||
|
||||
This method requires a redis DB and needs a HTTP X-Forwarded-For_ header.
|
||||
This method requires a valkey DB and needs a HTTP X-Forwarded-For_ header.
|
||||
|
||||
To get in use of this method a flask URL route needs to be added:
|
||||
|
||||
|
|
@ -45,8 +45,8 @@ import string
|
|||
import random
|
||||
|
||||
from searx import logger
|
||||
from searx import redisdb
|
||||
from searx.redislib import secret_hash
|
||||
from searx import valkeydb
|
||||
from searx.valkeylib import secret_hash
|
||||
from searx.extended_types import SXNG_Request
|
||||
|
||||
from ._helpers import (
|
||||
|
|
@ -76,17 +76,17 @@ def is_suspicious(network: IPv4Network | IPv6Network, request: SXNG_Request, ren
|
|||
:py:obj:`PING_LIVE_TIME`.
|
||||
|
||||
"""
|
||||
redis_client = redisdb.client()
|
||||
if not redis_client:
|
||||
valkey_client = valkeydb.client()
|
||||
if not valkey_client:
|
||||
return False
|
||||
|
||||
ping_key = get_ping_key(network, request)
|
||||
if not redis_client.get(ping_key):
|
||||
if not valkey_client.get(ping_key):
|
||||
logger.info("missing ping (IP: %s) / request: %s", network.compressed, ping_key)
|
||||
return True
|
||||
|
||||
if renew:
|
||||
redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
valkey_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
|
||||
logger.debug("found ping for (client) network %s -> %s", network.compressed, ping_key)
|
||||
return False
|
||||
|
|
@ -98,9 +98,9 @@ def ping(request: SXNG_Request, token: str):
|
|||
The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
|
||||
|
||||
"""
|
||||
from . import redis_client, cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
||||
from . import valkey_client, cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
||||
|
||||
if not redis_client:
|
||||
if not valkey_client:
|
||||
return
|
||||
if not token_is_valid(token):
|
||||
return
|
||||
|
|
@ -110,7 +110,7 @@ def ping(request: SXNG_Request, token: str):
|
|||
|
||||
ping_key = get_ping_key(network, request)
|
||||
logger.debug("store ping_key for (client) network %s (IP %s) -> %s", network.compressed, real_ip, ping_key)
|
||||
redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
valkey_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
|
||||
|
||||
def get_ping_key(network: IPv4Network | IPv6Network, request: SXNG_Request) -> str:
|
||||
|
|
@ -134,21 +134,21 @@ def token_is_valid(token) -> bool:
|
|||
|
||||
def get_token() -> str:
|
||||
"""Returns current token. If there is no currently active token a new token
|
||||
is generated randomly and stored in the redis DB.
|
||||
is generated randomly and stored in the valkey DB.
|
||||
|
||||
- :py:obj:`TOKEN_LIVE_TIME`
|
||||
- :py:obj:`TOKEN_KEY`
|
||||
|
||||
"""
|
||||
redis_client = redisdb.client()
|
||||
if not redis_client:
|
||||
# This function is also called when limiter is inactive / no redis DB
|
||||
valkey_client = valkeydb.client()
|
||||
if not valkey_client:
|
||||
# This function is also called when limiter is inactive / no valkey DB
|
||||
# (see render function in webapp.py)
|
||||
return '12345678'
|
||||
token = redis_client.get(TOKEN_KEY)
|
||||
token = valkey_client.get(TOKEN_KEY)
|
||||
if token:
|
||||
token = token.decode('UTF-8')
|
||||
else:
|
||||
token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
|
||||
redis_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
|
||||
valkey_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
|
||||
return token
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue