From 01be2612ab845771929181592931464f179357ea Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Fri, 4 Jul 2025 14:35:28 +0200 Subject: [PATCH] [mod] container: replace uWSGI with Granian (#4820) * [mod] container: replace uWSGI with Granian The configuration in Granian is handled with ENVs, much more convenient and practical for updating. The settings have been tested for over two months in a production instance, being usable on small to somewhat large instances without having to modify anything. It also removes the patch functions and ENVs abstraction from the entrypoint, this makes it possible to run the container with immutable configuration. In some setups, It may be desired to have the volumes/files under a specific uid/gid (other than searxng:searxng), if the entrypoint has root permissions it will chown automatically on every start, which may not be desired. Explicitly setting the new ENV `FORCE_OWNERSHIP=false` will prevent ownership from being modified. No manual migration is necessary **unless** the user has changed the default uWSGI configuration or has a very specific setup. Closes https://github.com/searxng/searxng/issues/4894 Closes https://github.com/searxng/searxng/issues/4818 Closes https://github.com/searxng/searxng/issues/4802 Supersedes https://github.com/searxng/searxng/pull/4596 Related https://github.com/searxng/searxng/discussions/4479 * [mod] docs: add container/granian All container documentation has been recreated. A new documentation page has been created for Granian. * [enh] misc: apply suggestions Minor documentation changes. Suggested https://github.com/searxng/searxng/pull/4820#discussion_r2134539259 Suggested https://github.com/searxng/searxng/pull/4820#discussion_r2134538610 Suggested https://github.com/searxng/searxng/pull/4820#discussion_r2134827964 Suggested https://github.com/searxng/searxng/pull/4820#discussion_r2134544300 Suggested https://github.com/searxng/searxng/pull/4820#discussion_r2149387388 --------- Signed-off-by: Markus Heiser Co-authored-by: Ivan Gabaldon Co-authored-by: Markus Heiser --- container/Dockerfile | 26 +-- container/base-builder.yml | 2 - container/base.yml | 3 - container/config/uwsgi.ini | 55 ----- container/entrypoint.sh | 68 ++---- container/template/.empty | 0 docs/admin/index.rst | 1 + docs/admin/installation-docker.rst | 342 ++++++++++++++-------------- docs/admin/installation-granian.rst | 52 +++++ docs/admin/installation-scripts.rst | 3 +- docs/admin/installation.rst | 4 +- utils/lib_sxng_container.sh | 1 - 12 files changed, 257 insertions(+), 300 deletions(-) delete mode 100644 container/config/uwsgi.ini create mode 100644 container/template/.empty create mode 100644 docs/admin/installation-granian.rst diff --git a/container/Dockerfile b/container/Dockerfile index 880ae26fc..5c4689072 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -5,7 +5,7 @@ COPY ./requirements.txt ./requirements.txt RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ - && pip install "uwsgi~=2.0" + && pip install "granian~=2.0" COPY ./searx/ ./searx/ @@ -27,13 +27,9 @@ ARG LABEL_VCS_URL="unspecified" COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ -COPY --chown=searxng:searxng ./container/config/ ./.template/ +COPY --chown=searxng:searxng ./container/template/ ./.template/ COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh -ARG TIMESTAMP_UWSGI="0" - -RUN touch -c --date=@$TIMESTAMP_UWSGI ./.template/uwsgi.ini - LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ org.opencontainers.image.created="$LABEL_DATE" \ org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ @@ -46,14 +42,18 @@ LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ org.opencontainers.image.version="$SEARXNG_GIT_VERSION" ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \ - INSTANCE_NAME="SearXNG" \ - AUTOCOMPLETE="" \ - BASE_URL="" \ - BIND_ADDRESS="[::]:8080" \ SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \ - UWSGI_SETTINGS_PATH="$CONFIG_PATH/uwsgi.ini" \ - UWSGI_WORKERS="%k" \ - UWSGI_THREADS="4" + GRANIAN_PROCESS_NAME="searxng" \ + GRANIAN_INTERFACE="wsgi" \ + GRANIAN_HOST="::" \ + GRANIAN_PORT="8080" \ + GRANIAN_WEBSOCKETS="false" \ + GRANIAN_LOOP="uvloop" \ + GRANIAN_BLOCKING_THREADS="4" \ + GRANIAN_WORKERS_KILL_TIMEOUT="30" \ + GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="300" \ + GRANIAN_STATIC_PATH_MOUNT="/usr/local/searxng/searx/static/" \ + GRANIAN_STATIC_PATH_EXPIRES="3600" VOLUME $CONFIG_PATH VOLUME $DATA_PATH diff --git a/container/base-builder.yml b/container/base-builder.yml index d6566f790..928e09232 100644 --- a/container/base-builder.yml +++ b/container/base-builder.yml @@ -8,8 +8,6 @@ contents: - python3-dev - py3-pip - brotli - # uwsgi - - libffi-dev entrypoint: command: /bin/sh -l diff --git a/container/base.yml b/container/base.yml index e0a3d9d07..0fb742595 100644 --- a/container/base.yml +++ b/container/base.yml @@ -8,9 +8,6 @@ contents: - python3 # healthcheck - wget - # uwsgi - - libxml2 - - mailcap entrypoint: command: /bin/sh -l diff --git a/container/config/uwsgi.ini b/container/config/uwsgi.ini deleted file mode 100644 index 3bfd49e72..000000000 --- a/container/config/uwsgi.ini +++ /dev/null @@ -1,55 +0,0 @@ -[uwsgi] -# Listening address -# default value: [::]:8080 (see Dockerfile) -http-socket = $(BIND_ADDRESS) - -# Who will run the code -uid = searxng -gid = searxng - -# Number of workers (usually CPU count) -# default value: %k (= number of CPU core, see Dockerfile) -workers = $(UWSGI_WORKERS) - -# Number of threads per worker -# default value: 4 (see Dockerfile) -threads = $(UWSGI_THREADS) - -# The right granted on the created socket -chmod-socket = 666 - -# Plugin to use and interpreter config -single-interpreter = true -master = true -lazy-apps = true -enable-threads = true - -# Module to import -module = searx.webapp - -# Virtualenv and python path -pythonpath = /usr/local/searxng/ -chdir = /usr/local/searxng/searx/ - -# automatically set processes name to something meaningful -auto-procname = true - -# Disable request logging for privacy -disable-logging = true -log-5xx = true - -# Set the max size of a request (request-body excluded) -buffer-size = 8192 - -# No keep alive -# See https://github.com/searx/searx-docker/issues/24 -add-header = Connection: close - -# Follow SIGTERM convention -# See https://github.com/searxng/searxng/issues/3427 -die-on-term - -# uwsgi serves the static files -static-map = /static=/usr/local/searxng/searx/static -static-gzip-all = True -offload-threads = %k diff --git a/container/entrypoint.sh b/container/entrypoint.sh index 8fbdb6947..d9b6b91de 100755 --- a/container/entrypoint.sh +++ b/container/entrypoint.sh @@ -2,6 +2,7 @@ # shellcheck shell=dash set -u +# Check if it's a valid file check_file() { local target="$1" @@ -16,6 +17,7 @@ EOF fi } +# Check if it's a valid directory check_directory() { local target="$1" @@ -47,66 +49,30 @@ EOF ;; esac - if [ "$(stat -c %U:%G "$target")" != "searxng:searxng" ]; then - if [ "$(id -u)" -eq 0 ]; then + target_ownership=$(stat -c %U:%G "$target") + + if [ "$target_ownership" != "searxng:searxng" ]; then + if [ "${FORCE_OWNERSHIP:-true}" = true ] && [ "$(id -u)" -eq 0 ]; then chown -R searxng:searxng "$target" else cat <`_ - - `Docker Cheat Sheet `_ - - `Alpine Linux `_ - `(wiki) `__ - `apt packages `_ - - Alpine's ``/bin/sh`` is :man:`dash` + - `Docker 101`_ + - `Docker cheat sheet (PDF doc)`_ + - `Podman rootless containers`_ -**If you intend to create a public instance using Docker, use our well maintained -docker container** +.. important:: -- `searxng/searxng @dockerhub`_. + Understanding container architecture basics is essential for properly + maintaining your SearXNG instance. This guide assumes familiarity with + container concepts and provides deployment steps at a high level. -.. sidebar:: hint + If you're new to containers, we recommend learning the fundamentals at + `Docker 101`_ before proceeding. - The rest of this article is of interest only to those who want to create and - maintain their own Docker images. +Container images are the basis for deployments in containerized environments, +`Docker compose`_, Kubernetes and more. -The sources are hosted at searxng-docker_ and the container includes: +.. _Container installation: -- a HTTPS reverse proxy `[caddy]`_ and -- a Redis_ DB - -The `default SearXNG setup `_ -of this container: - -- enables :ref:`limiter ` to protect against bots -- enables :ref:`image proxy ` for better privacy -- enables :ref:`cache busting ` to save bandwidth - ----- - - -Get Docker -========== - -If you plan to build and maintain a docker image by yourself, make sure you have -`Docker installed `_. On Linux don't -forget to add your user to the docker group (log out and log back in so that -your group membership is re-evaluated): - -.. code:: sh - - $ sudo usermod -a -G docker $USER - - -searxng/searxng -=============== - -.. sidebar:: ``docker run`` - - - `-\-rm `__ - automatically clean up when container exits - - `-d `__ start - detached container - - `-v `__ - mount volume ``HOST:CONTAINER`` - -The docker image is based on :origin:`Dockerfile` and available from -`searxng/searxng @dockerhub`_. Using the docker image is quite easy, for -instance you can pull the `searxng/searxng @dockerhub`_ image and deploy a local -instance using `docker run `_: - -.. code:: sh - - $ mkdir my-instance - $ cd my-instance - $ export PORT=8080 - $ docker pull searxng/searxng - $ docker run --rm \ - -d -p ${PORT}:8080 \ - -v "${PWD}/searxng:/etc/searxng" \ - -e "BASE_URL=http://localhost:$PORT/" \ - -e "INSTANCE_NAME=my-instance" \ - searxng/searxng - 2f998.... # container's ID - -The environment variables UWSGI_WORKERS and UWSGI_THREADS overwrite the default -number of UWSGI processes and UWSGI threads specified in `/etc/searxng/uwsgi.ini`. - -Open your WEB browser and visit the URL: - -.. code:: sh - - $ xdg-open "http://localhost:$PORT" - -Inside ``${PWD}/searxng``, you will find ``settings.yml`` and ``uwsgi.ini``. You -can modify these files according to your needs and restart the Docker image. - -.. code:: sh - - $ docker container restart 2f998 - -Use command ``container ls`` to list running containers, add flag `-a -`__ to list -exited containers also. With ``container stop`` a running container can be -stopped. To get rid of a container use ``container rm``: - -.. code:: sh - - $ docker container ls - CONTAINER ID IMAGE COMMAND CREATED ... - 2f998d725993 searxng/searxng "/sbin/tini -- /usr/…" 7 minutes ago ... - - $ docker container stop 2f998 - $ docker container rm 2f998 - -.. sidebar:: Warning - - This might remove all docker items, not only those from SearXNG. - -If you won't use docker anymore and want to get rid of all containers & images -use the following *prune* command: - -.. code:: sh - - $ docker stop $(docker ps -aq) # stop all containers - $ docker system prune # make some housekeeping - $ docker rmi -f $(docker images -q) # drop all images - - -shell inside container ----------------------- - -.. sidebar:: Bashism - - - `A tale of two shells: bash or dash `_ - - `How to make bash scripts work in dash `_ - - `Checking for Bashisms `_ - -To open a shell inside the container: - -.. code:: sh - - $ docker exec -it 2f998 sh - - -Build the image -=============== - -It's also possible to build SearXNG from the embedded :origin:`Dockerfile`:: - - $ git clone https://github.com/searxng/searxng.git - $ cd searxng - $ make docker.build - ... - Successfully built 49586c016434 - Successfully tagged searxng/searxng:latest - Successfully tagged searxng/searxng:1.0.0-209-9c823800-dirty - - $ docker images - REPOSITORY TAG IMAGE ID CREATED SIZE - searxng/searxng 1.0.0-209-9c823800-dirty 49586c016434 13 minutes ago 308MB - searxng/searxng latest 49586c016434 13 minutes ago 308MB - alpine 3.13 6dbb9cc54074 3 weeks ago 5.61MB - - -Command line +Installation ============ -.. sidebar:: docker run +.. _Container prerequisites: - Use flags ``-it`` for `interactive processes - `__. +Prerequisites +------------- -In the :origin:`Dockerfile` the ENTRYPOINT_ is defined as -:origin:`container/entrypoint.sh` +You need a working Docker or Podman installation on your system. Choose the +option that works best for your environment: + +- `Docker `_ (recommended for most users) +- `Podman `_ + +In the case of Docker, you need to add the user running the container to the +``docker`` group and restart the session: .. code:: sh - docker run --rm -it searxng/searxng -h + $ sudo usermod -aG docker $USER + +In the case of Podman, no additional steps are generally required, but there +are some considerations when running `Podman rootless containers`_. + +.. _Container pulling images: + +Pulling images +-------------- + +.. note:: + + DockerHub now applies rate limits to unauthenticated image pulls. If you + are affected by this, you can use the `GHCR mirror`_ instead. + +The official images are mirrored at: + +- `DockerHub mirror`_ +- `GHCR mirror`_ (GitHub Container Registry) + +Pull the latest image: + +.. code:: sh + + $ docker pull docker.io/searxng/searxng:latest + +\.\. or if you want to lock in to a specific version: + +.. code:: sh + + $ docker pull docker.io/searxng/searxng:2025.6.3-b73ac81 + +.. _Container instancing: + +Instancing +========== + +This section is intended for advanced users who need custom deployments. We +recommend using `Docker compose`_, which provides a preconfigured environment +with sensible defaults. + +Basic container instancing example: + +.. code:: sh + + # Create directories for configuration and persistent data + $ mkdir -p ./searxng/config/ ./searxng/data/ + $ cd ./searxng/ + + # Run the container + $ docker run --name searxng --replace -d \ + -p 8888:8080 \ + -v "./config/:/etc/searxng/" \ + -v "./data/:/var/cache/searxng/" \ + docker.io/searxng/searxng:latest + +This will start SearXNG in the background, accessible at http://localhost:8888 + +.. _Container management: + +Management +---------- + +List running containers: + +.. code:: sh + + $ docker container list + CONTAINER ID IMAGE ... CREATED PORTS NAMES + 37f6487c8703 ... ... 3 minutes ago 0.0.0.0:8888->8080/tcp searxng + +Access the container shell (troubleshooting): + +.. code:: sh + + $ docker container exec -it --user root searxng /bin/sh -l + 37f6487c8703:/usr/local/searxng# + +Stop and remove the container: + +.. code:: sh + + $ docker container stop searxng + $ docker container rm searxng + +.. _Container volumes: + +Volumes +======= + +Two volumes are exposed that should be mounted to preserve its contents: + +- ``/etc/searxng``: Configuration files (settings.yml, etc.) +- ``/var/cache/searxng``: Persistent data (faviconcache.db, etc.) + +.. _Container environment variables: + +Environment variables +===================== + +The following environment variables can be configured: + +- ``$SEARXNG_*``: Controls the SearXNG configuration options, look out for + environment ``$SEARXNG_*`` in :ref:`settings server` and :ref:`settings + general`. +- ``$GRANIAN_*``: Controls the :ref:`Granian server options `. +- ``$FORCE_OWNERSHIP``: Ensures mounted volumes/files are owned by the + ``searxng:searxng`` user (default: ``true``) + +Container internal paths (don't modify unless you know what you're doing): + +- ``$CONFIG_PATH``: Path to the SearXNG configuration directory (default: ``/etc/searxng``) +- ``$SEARXNG_SETTINGS_PATH``: Path to the SearXNG settings file (default: ``$CONFIG_PATH/settings.yml``) +- ``$DATA_PATH``: Path to the SearXNG data directory (default: ``/var/cache/searxng``) + +.. _Container custom images: + +Custom images +============= + +To build your own SearXNG container image from source (please note, custom +container images are not officially supported): + +.. code:: sh + + $ git clone https://github.com/searxng/searxng.git + $ cd ./searxng/ + + # Run the container build script + $ make container + + $ docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + localhost/searxng/searxng latest b14e256bfc36 14 seconds ago 201 MB + localhost/searxng/searxng 2025.5.1-b653119ab-dirty b14e256bfc36 14 seconds ago 201 MB + localhost/searxng/searxng builder 7f334c752b41 20 seconds ago 765 MB + ghcr.io/searxng/base searxng-builder 7d6b8a1bed4a 20 hours ago 625 MB + ghcr.io/searxng/base searxng 29baf9ef13ef 20 hours ago 62.5 MB diff --git a/docs/admin/installation-granian.rst b/docs/admin/installation-granian.rst new file mode 100644 index 000000000..06fd064c6 --- /dev/null +++ b/docs/admin/installation-granian.rst @@ -0,0 +1,52 @@ +.. _searxng granian: + +======= +Granian +======= + +.. _Options: https://github.com/emmett-framework/granian/blob/master/README.md#options +.. _Workers and threads: https://github.com/emmett-framework/granian/blob/master/README.md#workers-and-threads +.. _Backpressure: https://github.com/emmett-framework/granian/blob/master/README.md#backpressure +.. _Runtime mode: https://github.com/emmett-framework/granian/blob/master/README.md#runtime-mode + +.. sidebar:: further reading + + - `Options`_ + - `Workers and threads`_ + - `Backpressure`_ + - `Runtime mode`_ + +.. note:: + + Granian will be the future replacement for :ref:`searxng uwsgi` in SearXNG. + At the moment, it's only officially supported in the :ref:`installation + container`. + +.. _Granian installation: + +Installation +============ + +We only recommend installing Granian with pip, as officially documented. Run +the following command in the Python environment of the SearXNG installation: + +.. code:: sh + + $ pip install granian + +.. _Granian configuration: + +Configuration +============= + +.. note:: + + It's not advised to modify the amount of workers, expect increased resource + usage and potential issues with :ref:`botdetection`. + +Granian can be configured via option parameters and environment variables +(``$GRANIAN_*``). + +We provide sane defaults that should fit most use cases, however if you feel +you should change something, Granian documents all available parameters in the +`Options`_ section. diff --git a/docs/admin/installation-scripts.rst b/docs/admin/installation-scripts.rst index 2d43f5e37..573f82e1c 100644 --- a/docs/admin/installation-scripts.rst +++ b/docs/admin/installation-scripts.rst @@ -55,8 +55,7 @@ When all services are installed and running fine, you can add SearXNG to your HTTP server. We do not have any preferences regarding the HTTP server, you can use whatever you prefer. -We use caddy in our :ref:`docker image ` and we have -implemented installation procedures for: +We implemented installation procedures for: - :ref:`installation nginx` - :ref:`installation apache` diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 54d901f76..da603a639 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -6,13 +6,13 @@ Installation *You're spoilt for choice*, choose your preferred method of installation. -- :ref:`installation docker` +- :ref:`installation container` - :ref:`installation scripts` - :ref:`installation basic` The :ref:`installation basic` is an excellent illustration of *how a SearXNG instance is build up* (see :ref:`architecture uWSGI`). If you do not have any -special preferences, it's recommended to use the :ref:`installation docker` or the +special preferences, it's recommended to use the :ref:`installation container` or the :ref:`installation scripts`. .. attention:: diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh index b1e80717b..9abed5afb 100644 --- a/utils/lib_sxng_container.sh +++ b/utils/lib_sxng_container.sh @@ -122,7 +122,6 @@ container.build() { # shellcheck disable=SC2086 "$container_engine" $params_build \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ - --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./container/config/uwsgi.ini)" \ --build-arg="GIT_URL=$GIT_URL" \ --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \