[enh] container: tidy builds (#5086)

Building the container currently does not work properly.
When rebuilding several times with `make container`, `version_frozen.py`
is recreated, which wouldn't be an issue if the file’s timestamp was constant.
Now, when creating `version_frozen.py`, it will have the same timestamp as the
commit when it was created. (`version_frozen.py` is moved to a dedicated layer).

Reusing "builder" cache when building "dist" could be slow
(CD reports 2 seconds, but locally I've seen it take up to 10 seconds),
so the Dockerfile is now split and we save a couple steps
by importing the "builder" image directly.

The last changes made it possible to remove the layer cache in "builder",
since the overhead is now greater than building the layers from scratch.

Until now, all "dist" layers were squashed into a single layer,
which in most cases is a good idea
(except for storage/delivery pricing/overhead), but in our case,
since we manage the entire pipeline, we can ignore this
and share layers between builds.
This means (for example) that if we change files unrelated to the container
in several consecutive commits (documentation changes), we don't have to push
the entire image to registry, but only the different layers
(`version_frozen.py` in this example).
The same applies when pulling, as only the layers that have changed
compared to the local layers will be downloaded (that's the theory,
we'll see if this works as expected or if we need to tweak something else).

.dockerignore

@@ -1,44 +1,5 @@
-*~
+*
-*/*~
-*/*/*~
-*/*/*/*~
-*/*/*/*/*~
 
-# Git
+!container/entrypoint.sh
-.git
+!searx/**
-.gitignore
+!requirements*.txt
-
-# CI
-.codeclimate.yml
-.travis.yml
-.taskcluster.yml
-
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*/__pycache__/
-*/*/__pycache__/
-*/*/*/__pycache__/
-*.py[cod]
-*/*.py[cod]
-*/*/*.py[cod]
-*/*/*/*.py[cod]
-
-# node_modules
-node_modules/
-*/node_modules/
-*/*/node_modules/
-*/*/*/node_modules/
-*/*/*/*/node_modules/
-
-.tx/
-
-# to sync with .gitignore
-geckodriver.log
-.coverage
-coverage/
-cache/
-build/
-dist/
-local/
-gh-pages/
-*.egg-info/

.github/workflows/cleanup.yml

@@ -34,4 +34,4 @@ jobs:
           image-names: "cache"
           image-tags: "!searxng*"
           cut-off: "1d"
-          keep-n-most-recent: "100"
+          keep-n-most-recent: "30"

.github/workflows/container.yml

@@ -104,6 +104,8 @@ jobs:
     needs: build-base
     strategy:
       fail-fast: false
+      # Faster runners first to cache arch independent wheels
+      max-parallel: 1
       matrix:
         include:
           - arch: amd64
@@ -121,11 +123,8 @@ jobs:
       packages: write
 
     outputs:
-      version_string: ${{ steps.build.outputs.version_string }}
-      version_tag: ${{ steps.build.outputs.version_tag }}
       docker_tag: ${{ steps.build.outputs.docker_tag }}
       git_url: ${{ steps.build.outputs.git_url }}
-      git_branch: ${{ steps.build.outputs.git_branch }}
 
     steps:
       - name: Setup Python
@@ -148,9 +147,8 @@ jobs:
       - name: Setup cache container mounts
         uses: actions/cache@v4
         with:
-          # yamllint disable-line rule:line-length
+          key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
-          key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile') }}"
+          restore-keys: "container-mounts-"
-          restore-keys: "container-mounts-${{ matrix.arch }}-"
           path: |
             /var/tmp/buildah-cache/
             /var/tmp/buildah-cache-*/

container/Dockerfile

@@ -1,61 +0,0 @@
-FROM ghcr.io/searxng/base:searxng-builder AS builder
-
-COPY ./requirements*.txt ./
-
-RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
- && . ./venv/bin/activate \
- && pip install -r requirements.txt \
- && pip install -r requirements-server.txt
-
-COPY ./searx/ ./searx/
-
-ARG TIMESTAMP_SETTINGS="0"
-
-RUN python -m compileall -q searx \
- && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
- && find ./searx/static \
-    \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \
-    -type f -exec gzip -9 -k {} + -exec brotli --best {} +
-
-FROM ghcr.io/searxng/base:searxng AS dist
-
-ARG LABEL_DATE="0001-01-01T00:00:00Z"
-ARG GIT_URL="unspecified"
-ARG SEARXNG_GIT_VERSION="unspecified"
-ARG LABEL_VCS_REF="unspecified"
-ARG LABEL_VCS_URL="unspecified"
-
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
-COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
-COPY --chown=searxng:searxng ./container/template/ ./.template/
-COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
-
-LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
-      org.opencontainers.image.created="$LABEL_DATE" \
-      org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \
-      org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \
-      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
-      org.opencontainers.image.revision="$LABEL_VCS_REF" \
-      org.opencontainers.image.source="$LABEL_VCS_URL" \
-      org.opencontainers.image.title="searxng" \
-      org.opencontainers.image.url="$LABEL_VCS_URL" \
-      org.opencontainers.image.version="$SEARXNG_GIT_VERSION"
-
-ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \
-    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
-    GRANIAN_PROCESS_NAME="searxng" \
-    GRANIAN_INTERFACE="wsgi" \
-    GRANIAN_HOST="::" \
-    GRANIAN_PORT="8080" \
-    GRANIAN_WEBSOCKETS="false" \
-    GRANIAN_LOOP="uvloop" \
-    GRANIAN_BLOCKING_THREADS="4" \
-    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
-    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
-
-VOLUME $CONFIG_PATH
-VOLUME $DATA_PATH
-
-EXPOSE 8080
-
-ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

container/builder.dockerfile

@@ -0,0 +1,24 @@
+FROM ghcr.io/searxng/base:searxng-builder AS builder
+
+COPY ./requirements*.txt ./
+
+RUN --mount=type=cache,id=pip,target=/root/.cache/pip set -eux; \
+    python -m venv ./.venv/; \
+    . ./.venv/bin/activate; \
+    pip install -r ./requirements.txt -r ./requirements-server.txt
+
+COPY ./searx/ ./searx/
+
+ARG TIMESTAMP_SETTINGS="0"
+
+RUN set -eux; \
+    python -m compileall -q ./searx/; \
+    touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml; \
+    find ./searx/static/ -type f \
+        \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" \) \
+        -exec gzip -9 -k {} + \
+        -exec brotli -9 -k {} + \
+        -exec gzip --test {}.gz + \
+        -exec brotli --test {}.br +; \
+    # Move always changing files to /usr/local/searxng/
+    mv ./searx/version_frozen.py ./

container/dist.dockerfile

@@ -0,0 +1,44 @@
+FROM ghcr.io/searxng/base:searxng AS dist
+
+ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
+ARG CONTAINER_IMAGE_NAME="searxng"
+
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/.venv/ ./.venv/
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/searx/ ./searx/
+COPY --chown=searxng:searxng ./container/ ./
+COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/version_frozen.py ./searx/
+
+ARG CREATED="0001-01-01T00:00:00Z"
+ARG VERSION="unknown"
+ARG VCS_URL="unknown"
+ARG VCS_REVISION="unknown"
+
+LABEL org.opencontainers.image.created="$CREATED" \
+      org.opencontainers.image.description="SearXNG is a metasearch engine. Users are neither tracked nor profiled." \
+      org.opencontainers.image.documentation="https://docs.searxng.org/admin/installation-docker" \
+      org.opencontainers.image.licenses="AGPL-3.0-or-later" \
+      org.opencontainers.image.revision="$VCS_REVISION" \
+      org.opencontainers.image.source="$VCS_URL" \
+      org.opencontainers.image.title="SearXNG" \
+      org.opencontainers.image.url="https://searxng.org" \
+      org.opencontainers.image.version="$VERSION"
+
+ENV SEARXNG_VERSION="$VERSION" \
+    SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
+    GRANIAN_PROCESS_NAME="searxng" \
+    GRANIAN_INTERFACE="wsgi" \
+    GRANIAN_HOST="::" \
+    GRANIAN_PORT="8080" \
+    GRANIAN_WEBSOCKETS="false" \
+    GRANIAN_LOOP="uvloop" \
+    GRANIAN_BLOCKING_THREADS="4" \
+    GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
+    GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
+
+# "*_PATH" ENVs are defined in base images
+VOLUME $CONFIG_PATH
+VOLUME $DATA_PATH
+
+EXPOSE 8080
+
+ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

container/entrypoint.sh

@@ -127,4 +127,4 @@ volume_handler "$DATA_PATH"
 # Check for files
 config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"
 
-exec /usr/local/searxng/venv/bin/granian searx.webapp:app
+exec /usr/local/searxng/.venv/bin/granian searx.webapp:app

docs/admin/installation-docker.rst

@@ -80,7 +80,7 @@ Pull the latest image:
 
 .. code:: sh
 
-   $ docker pull docker.io/searxng/searxng:2025.6.3-b73ac81
+   $ docker pull docker.io/searxng/searxng:2025.8.1-3d96414
 
 .. _Container instancing:
 
@@ -119,14 +119,14 @@ List running containers:
 
    $ docker container list
    CONTAINER ID  IMAGE  ...  CREATED        PORTS                   NAMES
-   37f6487c8703  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng
+   1af574997e63  ...    ...  3 minutes ago  0.0.0.0:8888->8080/tcp  searxng
 
 Access the container shell (troubleshooting):
 
 .. code:: sh
 
    $ docker container exec -it --user root searxng /bin/sh -l
-   37f6487c8703:/usr/local/searxng#
+   1af574997e63:/usr/local/searxng#
 
 Stop and remove the container:
 
@@ -183,8 +183,8 @@ container images are not officially supported):
 
    $ docker images
    REPOSITORY                 TAG                 IMAGE ID  CREATED             SIZE
-   localhost/searxng/searxng  latest                    b14e256bfc36  14 seconds ago  201 MB
+   localhost/searxng/searxng  2025.8.1-3d96414    ...       About a minute ago  183 MB
-   localhost/searxng/searxng  2025.5.1-b653119ab-dirty  b14e256bfc36  14 seconds ago  201 MB
+   localhost/searxng/searxng  latest              ...       About a minute ago  183 MB
-   localhost/searxng/searxng  builder                   7f334c752b41  20 seconds ago  765 MB
+   localhost/searxng/searxng  builder             ...       About a minute ago  524 MB
-   ghcr.io/searxng/base       searxng-builder           7d6b8a1bed4a  20 hours ago    625 MB
+   ghcr.io/searxng/base       searxng-builder     ...       2 days ago          378 MB
-   ghcr.io/searxng/base       searxng                   29baf9ef13ef  20 hours ago    62.5 MB
+   ghcr.io/searxng/base       searxng             ...       2 days ago          42.2 MB

searx/version.py

@@ -134,9 +134,14 @@ DOCKER_TAG = "{DOCKER_TAG}"
 GIT_URL = "{GIT_URL}"
 GIT_BRANCH = "{GIT_BRANCH}"
 """
-        with open(os.path.join(os.path.dirname(__file__), "version_frozen.py"), "w", encoding="utf8") as f:
+        path = os.path.join(os.path.dirname(__file__), "version_frozen.py")
+        with open(path, "w", encoding="utf8") as f:
             f.write(python_code)
             print(f"{f.name} created")
+
+        # set file timestamp to commit timestamp
+        commit_timestamp = int(subprocess_run("git show -s --format=%ct"))
+        os.utime(path, (commit_timestamp, commit_timestamp))
     else:
         # output shell code to set the variables
         # usage: eval "$(python -m searx.version)"

utils/lib_sxng_container.sh

@@ -14,7 +14,6 @@ CONTAINER_IMAGE_NAME="searxng"
 container.build() {
     local parch=${OVERRIDE_ARCH:-$(uname -m)}
     local container_engine
-    local dockerfile
     local arch
     local variant
     local platform
@@ -42,19 +41,16 @@ container.build() {
     # Setup arch specific
     case $parch in
         "X64" | "x86_64" | "amd64")
-            dockerfile="Dockerfile"
             arch="amd64"
             variant=""
             platform="linux/$arch"
             ;;
         "ARM64" | "aarch64" | "arm64")
-            dockerfile="Dockerfile"
             arch="arm64"
             variant=""
             platform="linux/$arch"
             ;;
         "ARMV7" | "armhf" | "armv7l" | "armv7")
-            dockerfile="Dockerfile"
             arch="arm"
             variant="v7"
             platform="linux/$arch/$variant"
@@ -86,27 +82,20 @@ container.build() {
         python -m searx.version freeze
         eval "$(python -m searx.version)"
 
-        info_msg "Set \$VERSION_STRING: $VERSION_STRING"
-        info_msg "Set \$VERSION_TAG: $VERSION_TAG"
         info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
         info_msg "Set \$GIT_URL: $GIT_URL"
-        info_msg "Set \$GIT_BRANCH: $GIT_BRANCH"
 
         if [ "$container_engine" = "podman" ]; then
-            params_build_builder="build --format=oci --platform=$platform --target=builder --layers --identity-label=false"
+            params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false"
-            params_build="build --format=oci --platform=$platform --layers --squash-all --omit-history --identity-label=false"
+            params_build=$params_build_builder
         else
-            params_build_builder="build --platform=$platform --target=builder"
+            params_build_builder="build --platform=$platform"
-            params_build="build --platform=$platform --squash"
+            params_build=$params_build_builder
         fi
 
         if [ "$GITHUB_ACTIONS" = "true" ]; then
-            params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache"
-
-            # Tags
             params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
         else
-            # Tags
             params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
             params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
         fi
@@ -115,19 +104,19 @@ container.build() {
         "$container_engine" $params_build_builder \
             --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
             --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
-            --file="./container/$dockerfile" \
+            --file="./container/builder.dockerfile" \
             .
         build_msg CONTAINER "Image \"builder\" built"
 
         # shellcheck disable=SC2086
         "$container_engine" $params_build \
-            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
+            --build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
-            --build-arg="GIT_URL=$GIT_URL" \
+            --build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
-            --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
+            --build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-            --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+            --build-arg="VERSION=$DOCKER_TAG" \
-            --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
+            --build-arg="VCS_URL=$GIT_URL" \
-            --build-arg="LABEL_VCS_URL=$GIT_URL" \
+            --build-arg="VCS_REVISION=$(git rev-parse HEAD)" \
-            --file="./container/$dockerfile" \
+            --file="./container/dist.dockerfile" \
             .
         build_msg CONTAINER "Image built"
 
@@ -136,11 +125,8 @@ container.build() {
 
             # Output to GHA
             cat <<EOF >>"$GITHUB_OUTPUT"
-version_string=$VERSION_STRING
-version_tag=$VERSION_TAG
 docker_tag=$DOCKER_TAG
 git_url=$GIT_URL
-git_branch=$GIT_BRANCH
 EOF
         fi
     )