Icycoide/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng.git synced 2025-08-10 05:46:46 +02:00
Code Issues Projects Releases Packages Wiki Activity

[enh] container: tidy builds (#5086)

Browse source 
Building the container currently does not work properly.
When rebuilding several times with `make container`, `version_frozen.py`
is recreated, which wouldn't be an issue if the file’s timestamp was constant.
Now, when creating `version_frozen.py`, it will have the same timestamp as the
commit when it was created. (`version_frozen.py` is moved to a dedicated layer).

Reusing "builder" cache when building "dist" could be slow
(CD reports 2 seconds, but locally I've seen it take up to 10 seconds),
so the Dockerfile is now split and we save a couple steps
by importing the "builder" image directly.

The last changes made it possible to remove the layer cache in "builder",
since the overhead is now greater than building the layers from scratch.

Until now, all "dist" layers were squashed into a single layer,
which in most cases is a good idea
(except for storage/delivery pricing/overhead), but in our case,
since we manage the entire pipeline, we can ignore this
and share layers between builds.
This means (for example) that if we change files unrelated to the container
in several consecutive commits (documentation changes), we don't have to push
the entire image to registry, but only the different layers
(`version_frozen.py` in this example).
The same applies when pulling, as only the layers that have changed
compared to the local layers will be downloaded (that's the theory,
we'll see if this works as expected or if we need to tweak something else).
This commit is contained in:
Ivan Gabaldon 2025-08-07 10:46:26 +02:00 • committed by GitHub
parent 94256e3383
commit 3de7a6da2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 105 additions and 148 deletions
Download patch file Download diff file Expand all files Collapse all files

47
.dockerignore
View file

@ -1,44 +1,5 @@
*~
*/*~
*/*/*~
*/*/*/*~
*/*/*/*/*~
*
# Git
.git
.gitignore
# CI
.codeclimate.yml
.travis.yml
.taskcluster.yml
# Byte-compiled / optimized / DLL files
__pycache__/
*/__pycache__/
*/*/__pycache__/
*/*/*/__pycache__/
*.py[cod]
*/*.py[cod]
*/*/*.py[cod]
*/*/*/*.py[cod]
# node_modules
node_modules/
*/node_modules/
*/*/node_modules/
*/*/*/node_modules/
*/*/*/*/node_modules/
.tx/
# to sync with .gitignore
geckodriver.log
.coverage
coverage/
cache/
build/
dist/
local/
gh-pages/
*.egg-info/
!container/entrypoint.sh
!searx/**
!requirements*.txt

2
.github/workflows/cleanup.yml vendored
View file

@ -34,4 +34,4 @@ jobs:
image-names: "cache"
image-tags: "!searxng*"
cut-off: "1d"
keep-n-most-recent: "100"
keep-n-most-recent: "30"

10
.github/workflows/container.yml vendored
View file

@ -104,6 +104,8 @@ jobs:
needs: build-base
strategy:
fail-fast: false
# Faster runners first to cache arch independent wheels
max-parallel: 1
matrix:
include:
- arch: amd64
@ -121,11 +123,8 @@ jobs:
packages: write
outputs:
version_string: ${{ steps.build.outputs.version_string }}
version_tag: ${{ steps.build.outputs.version_tag }}
docker_tag: ${{ steps.build.outputs.docker_tag }}
git_url: ${{ steps.build.outputs.git_url }}
git_branch: ${{ steps.build.outputs.git_branch }}
steps:
- name: Setup Python
@ -148,9 +147,8 @@ jobs:
- name: Setup cache container mounts
uses: actions/cache@v4
with:
# yamllint disable-line rule:line-length
key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile') }}"
restore-keys: "container-mounts-${{ matrix.arch }}-"
key: "container-mounts-${{ hashFiles('./container/*.dockerfile') }}"
restore-keys: "container-mounts-"
path: |
/var/tmp/buildah-cache/
/var/tmp/buildah-cache-*/

61
container/Dockerfile
View file

@ -1,61 +0,0 @@
FROM ghcr.io/searxng/base:searxng-builder AS builder
COPY ./requirements*.txt ./
RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \
&& . ./venv/bin/activate \
&& pip install -r requirements.txt \
&& pip install -r requirements-server.txt
COPY ./searx/ ./searx/
ARG TIMESTAMP_SETTINGS="0"
RUN python -m compileall -q searx \
&& touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \
&& find ./searx/static \
\( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \
-type f -exec gzip -9 -k {} + -exec brotli --best {} +
FROM ghcr.io/searxng/base:searxng AS dist
ARG LABEL_DATE="0001-01-01T00:00:00Z"
ARG GIT_URL="unspecified"
ARG SEARXNG_GIT_VERSION="unspecified"
ARG LABEL_VCS_REF="unspecified"
ARG LABEL_VCS_URL="unspecified"
COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/
COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/
COPY --chown=searxng:searxng ./container/template/ ./.template/
COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh
LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \
org.opencontainers.image.created="$LABEL_DATE" \
org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \
org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \
org.opencontainers.image.licenses="AGPL-3.0-or-later" \
org.opencontainers.image.revision="$LABEL_VCS_REF" \
org.opencontainers.image.source="$LABEL_VCS_URL" \
org.opencontainers.image.title="searxng" \
org.opencontainers.image.url="$LABEL_VCS_URL" \
org.opencontainers.image.version="$SEARXNG_GIT_VERSION"
ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \
SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
GRANIAN_PROCESS_NAME="searxng" \
GRANIAN_INTERFACE="wsgi" \
GRANIAN_HOST="::" \
GRANIAN_PORT="8080" \
GRANIAN_WEBSOCKETS="false" \
GRANIAN_LOOP="uvloop" \
GRANIAN_BLOCKING_THREADS="4" \
GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
VOLUME $CONFIG_PATH
VOLUME $DATA_PATH
EXPOSE 8080
ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

24
container/builder.dockerfile Normal file
View file

@ -0,0 +1,24 @@
FROM ghcr.io/searxng/base:searxng-builder AS builder
COPY ./requirements*.txt ./
RUN --mount=type=cache,id=pip,target=/root/.cache/pip set -eux; \
python -m venv ./.venv/; \
. ./.venv/bin/activate; \
pip install -r ./requirements.txt -r ./requirements-server.txt
COPY ./searx/ ./searx/
ARG TIMESTAMP_SETTINGS="0"
RUN set -eux; \
python -m compileall -q ./searx/; \
touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml; \
find ./searx/static/ -type f \
\( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" \) \
-exec gzip -9 -k {} + \
-exec brotli -9 -k {} + \
-exec gzip --test {}.gz + \
-exec brotli --test {}.br +; \
# Move always changing files to /usr/local/searxng/
mv ./searx/version_frozen.py ./

44
container/dist.dockerfile Normal file
View file

@ -0,0 +1,44 @@
FROM ghcr.io/searxng/base:searxng AS dist
ARG CONTAINER_IMAGE_ORGANIZATION="searxng"
ARG CONTAINER_IMAGE_NAME="searxng"
COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/.venv/ ./.venv/
COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/searx/ ./searx/
COPY --chown=searxng:searxng ./container/ ./
COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/version_frozen.py ./searx/
ARG CREATED="0001-01-01T00:00:00Z"
ARG VERSION="unknown"
ARG VCS_URL="unknown"
ARG VCS_REVISION="unknown"
LABEL org.opencontainers.image.created="$CREATED" \
org.opencontainers.image.description="SearXNG is a metasearch engine. Users are neither tracked nor profiled." \
org.opencontainers.image.documentation="https://docs.searxng.org/admin/installation-docker" \
org.opencontainers.image.licenses="AGPL-3.0-or-later" \
org.opencontainers.image.revision="$VCS_REVISION" \
org.opencontainers.image.source="$VCS_URL" \
org.opencontainers.image.title="SearXNG" \
org.opencontainers.image.url="https://searxng.org" \
org.opencontainers.image.version="$VERSION"
ENV SEARXNG_VERSION="$VERSION" \
SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \
GRANIAN_PROCESS_NAME="searxng" \
GRANIAN_INTERFACE="wsgi" \
GRANIAN_HOST="::" \
GRANIAN_PORT="8080" \
GRANIAN_WEBSOCKETS="false" \
GRANIAN_LOOP="uvloop" \
GRANIAN_BLOCKING_THREADS="4" \
GRANIAN_WORKERS_KILL_TIMEOUT="30s" \
GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m"
# "*_PATH" ENVs are defined in base images
VOLUME $CONFIG_PATH
VOLUME $DATA_PATH
EXPOSE 8080
ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"]

2
container/entrypoint.sh
View file

@ -127,4 +127,4 @@ volume_handler "$DATA_PATH"
# Check for files
config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml"
exec /usr/local/searxng/venv/bin/granian searx.webapp:app
exec /usr/local/searxng/.venv/bin/granian searx.webapp:app

0
container/template/.empty
View file

18
docs/admin/installation-docker.rst
View file

@ -80,7 +80,7 @@ Pull the latest image:
.. code:: sh
$ docker pull docker.io/searxng/searxng:2025.6.3-b73ac81
$ docker pull docker.io/searxng/searxng:2025.8.1-3d96414
.. _Container instancing:
@ -119,14 +119,14 @@ List running containers:
$ docker container list
CONTAINER ID IMAGE ... CREATED PORTS NAMES
37f6487c8703 ... ... 3 minutes ago 0.0.0.0:8888->8080/tcp searxng
1af574997e63 ... ... 3 minutes ago 0.0.0.0:8888->8080/tcp searxng
Access the container shell (troubleshooting):
.. code:: sh
$ docker container exec -it --user root searxng /bin/sh -l
37f6487c8703:/usr/local/searxng#
1af574997e63:/usr/local/searxng#
Stop and remove the container:
@ -182,9 +182,9 @@ container images are not officially supported):
$ make container
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
localhost/searxng/searxng latest b14e256bfc36 14 seconds ago 201 MB
localhost/searxng/searxng 2025.5.1-b653119ab-dirty b14e256bfc36 14 seconds ago 201 MB
localhost/searxng/searxng builder 7f334c752b41 20 seconds ago 765 MB
ghcr.io/searxng/base searxng-builder 7d6b8a1bed4a 20 hours ago 625 MB
ghcr.io/searxng/base searxng 29baf9ef13ef 20 hours ago 62.5 MB
REPOSITORY TAG IMAGE ID CREATED SIZE
localhost/searxng/searxng 2025.8.1-3d96414 ... About a minute ago 183 MB
localhost/searxng/searxng latest ... About a minute ago 183 MB
localhost/searxng/searxng builder ... About a minute ago 524 MB
ghcr.io/searxng/base searxng-builder ... 2 days ago 378 MB
ghcr.io/searxng/base searxng ... 2 days ago 42.2 MB

7
searx/version.py
View file

@ -134,9 +134,14 @@ DOCKER_TAG = "{DOCKER_TAG}"
GIT_URL = "{GIT_URL}"
GIT_BRANCH = "{GIT_BRANCH}"
"""
with open(os.path.join(os.path.dirname(__file__), "version_frozen.py"), "w", encoding="utf8") as f:
path = os.path.join(os.path.dirname(__file__), "version_frozen.py")
with open(path, "w", encoding="utf8") as f:
f.write(python_code)
print(f"{f.name} created")
# set file timestamp to commit timestamp
commit_timestamp = int(subprocess_run("git show -s --format=%ct"))
os.utime(path, (commit_timestamp, commit_timestamp))
else:
# output shell code to set the variables
# usage: eval "$(python -m searx.version)"

38
utils/lib_sxng_container.sh
View file

@ -14,7 +14,6 @@ CONTAINER_IMAGE_NAME="searxng"
container.build() {
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local container_engine
local dockerfile
local arch
local variant
local platform
@ -42,19 +41,16 @@ container.build() {
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
dockerfile="Dockerfile"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
@ -86,27 +82,20 @@ container.build() {
python -m searx.version freeze
eval "$(python -m searx.version)"
info_msg "Set \$VERSION_STRING: $VERSION_STRING"
info_msg "Set \$VERSION_TAG: $VERSION_TAG"
info_msg "Set \$DOCKER_TAG: $DOCKER_TAG"
info_msg "Set \$GIT_URL: $GIT_URL"
info_msg "Set \$GIT_BRANCH: $GIT_BRANCH"
if [ "$container_engine" = "podman" ]; then
params_build_builder="build --format=oci --platform=$platform --target=builder --layers --identity-label=false"
params_build="build --format=oci --platform=$platform --layers --squash-all --omit-history --identity-label=false"
params_build_builder="build --format=oci --platform=$platform --layers --identity-label=false"
params_build=$params_build_builder
else
params_build_builder="build --platform=$platform --target=builder"
params_build="build --platform=$platform --squash"
params_build_builder="build --platform=$platform"
params_build=$params_build_builder
fi
if [ "$GITHUB_ACTIONS" = "true" ]; then
params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache"
# Tags
params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant"
else
# Tags
params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest"
params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG"
fi
@ -115,19 +104,19 @@ container.build() {
"$container_engine" $params_build_builder \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \
--file="./container/$dockerfile" \
--file="./container/builder.dockerfile" \
.
build_msg CONTAINER "Image \"builder\" built"
# shellcheck disable=SC2086
"$container_engine" $params_build \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--file="./container/$dockerfile" \
--build-arg="CONTAINER_IMAGE_ORGANIZATION=$CONTAINER_IMAGE_ORGANIZATION" \
--build-arg="CONTAINER_IMAGE_NAME=$CONTAINER_IMAGE_NAME" \
--build-arg="CREATED=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="VERSION=$DOCKER_TAG" \
--build-arg="VCS_URL=$GIT_URL" \
--build-arg="VCS_REVISION=$(git rev-parse HEAD)" \
--file="./container/dist.dockerfile" \
.
build_msg CONTAINER "Image built"
@ -136,11 +125,8 @@ container.build() {
# Output to GHA
cat <<EOF >>"$GITHUB_OUTPUT"
version_string=$VERSION_STRING
version_tag=$VERSION_TAG
docker_tag=$DOCKER_TAG
git_url=$GIT_URL
git_branch=$GIT_BRANCH
EOF
fi
)