Icycoide/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng.git synced 2025-08-02 10:02:20 +02:00
Code Issues Projects Releases Packages Wiki Activity

[mod] botdetection: HTTP Fetch Metadata Request Headers

Browse source 
HTTP Fetch Metadata Request Headers [1][2] are used to detect bot requests. Bots
with invalid *Fetch Metadata* will be redirected to the intro (`index`)  page.

[1] https://www.w3.org/TR/fetch-metadata/
[2] https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2024-10-27 13:17:40 +01:00 • committed by Markus Heiser
parent 8ef5fbca4e
commit fe08bb1d90
4 changed files with 74 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
searx/limiter.py
View file

@ -112,6 +112,7 @@ from searx.botdetection import (
http_accept_encoding,
http_accept_language,
http_user_agent,
http_sec_fetch,
ip_limit,
ip_lists,
get_network,
@ -179,16 +180,17 @@ def filter_request(request: SXNG_Request) -> werkzeug.Response | None:
logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
# methods applied on /
# methods applied on all requests
for func in [
http_user_agent,
]:
val = func.filter_request(network, request, cfg)
if val is not None:
logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
return val
# methods applied on /search
# methods applied on /search requests
if request.path == '/search':
@ -197,11 +199,14 @@ def filter_request(request: SXNG_Request) -> werkzeug.Response | None:
http_accept_encoding,
http_accept_language,
http_user_agent,
http_sec_fetch,
ip_limit,
]:
val = func.filter_request(network, request, cfg)
if val is not None:
logger.debug(f"NOT OK ({func.__name__}): {network}: %s", dump_request(sxng_request))
return val
logger.debug(f"OK {network}: %s", dump_request(sxng_request))
return None