[fix] correct determination of the IP for the request

For correct determination of the IP to the request the function
botdetection.get_real_ip() is implemented.  This fonction is used in the
ip_limit and link_token method of the botdetection and it is used in the
self_info plugin.

A documentation about the X-Forwarded-For header has been added.

[1] https://github.com/searxng/searxng/pull/2357#issuecomment-1566211059

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

searx/botdetection/__init__.py

@@ -2,11 +2,25 @@
 # lint: pylint
 """.. _botdetection src:
 
-Bot detection methods
----------------------
+X-Forwarded-For
+===============
 
-The methods implemented in this python package are use by the :ref:`limiter src`.
+.. attention::
+
+   A correct setup of the HTTP request headers ``X-Forwarded-For`` and
+   ``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
+
+   - `NGINX RequestHeader`_
+   - `Apache RequestHeader`_
+
+.. _NGINX RequestHeader:
+    https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
+.. _Apache RequestHeader:
+    https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
+
+.. autofunction:: searx.botdetection.get_real_ip
 
 """
 
 from ._helpers import dump_request
+from ._helpers import get_real_ip

searx/botdetection/ip_limit.py

@@ -49,7 +49,7 @@ from searx import logger
 from searx.redislib import incr_sliding_window, drop_counter
 
 from . import link_token
-from ._helpers import too_many_requests
+from ._helpers import too_many_requests, get_real_ip
 
 
 logger = logger.getChild('botdetection.ip_limit')
@@ -89,9 +89,7 @@ def filter_request(request: flask.Request, cfg: config.Config) -> Optional[werkz
     # pylint: disable=too-many-return-statements
     redis_client = redisdb.client()
 
-    client_ip = request.headers.get('X-Forwarded-For', '')
-    if not client_ip:
-        logger.error("missing HTTP header X-Forwarded-For")
+    client_ip = get_real_ip(request)
 
     if request.args.get('format', 'html') != 'html':
         c = incr_sliding_window(redis_client, 'ip_limit.API_WONDOW:' + client_ip, API_WONDOW)

searx/botdetection/limiter.toml

@@ -1,3 +1,8 @@
 [botdetection.ip_limit]
 
-link_token = false
+link_token = false
+
+[real_ip]
+
+# Number of values to trust for X-Forwarded-For.
+x_for = 1

searx/botdetection/link_token.py

@@ -43,6 +43,7 @@ import flask
 from searx import logger
 from searx import redisdb
 from searx.redislib import secret_hash
+from ._helpers import get_real_ip
 
 TOKEN_LIVE_TIME = 600
 """Livetime (sec) of limiter's CSS token."""
@@ -73,7 +74,7 @@ def is_suspicious(request: flask.Request, renew: bool = False):
     if not redis_client.get(ping_key):
         logger.warning(
             "missing ping (IP: %s) / request: %s",
-            request.headers.get('X-Forwarded-For', ''),
+            get_real_ip(request),
             ping_key,
         )
         return True
@@ -111,9 +112,7 @@ def get_ping_key(request: flask.Request):
         PING_KEY
         + "["
         + secret_hash(
-            request.headers.get('X-Forwarded-For', '')
-            + request.headers.get('Accept-Language', '')
-            + request.headers.get('User-Agent', '')
+            get_real_ip(request) + request.headers.get('Accept-Language', '') + request.headers.get('User-Agent', '')
         )
         + "]"
     )