searxng/searx/version.py
Ivan Gabaldon 3de7a6da2d
[enh] container: tidy builds (#5086)
Building the container currently does not work properly.
When rebuilding several times with `make container`, `version_frozen.py`
is recreated, which wouldn't be an issue if the file’s timestamp was constant.
Now, when creating `version_frozen.py`, it will have the same timestamp as the
commit when it was created. (`version_frozen.py` is moved to a dedicated layer).

Reusing "builder" cache when building "dist" could be slow
(CD reports 2 seconds, but locally I've seen it take up to 10 seconds),
so the Dockerfile is now split and we save a couple steps
by importing the "builder" image directly.

The last changes made it possible to remove the layer cache in "builder",
since the overhead is now greater than building the layers from scratch.

Until now, all "dist" layers were squashed into a single layer,
which in most cases is a good idea
(except for storage/delivery pricing/overhead), but in our case,
since we manage the entire pipeline, we can ignore this
and share layers between builds.
This means (for example) that if we change files unrelated to the container
in several consecutive commits (documentation changes), we don't have to push
the entire image to registry, but only the different layers
(`version_frozen.py` in this example).
The same applies when pulling, as only the layers that have changed
compared to the local layers will be downloaded (that's the theory,
we'll see if this works as expected or if we need to tweak something else).
2025-08-07 10:46:26 +02:00

155 lines
5.2 KiB
Python

# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=,missing-module-docstring,missing-class-docstring
import importlib
import logging
import os
import shlex
import subprocess
# fallback values
# if there is searx.version_frozen module, and it is not possible to get the git tag
VERSION_STRING = "1.0.0"
VERSION_TAG = "1.0.0"
DOCKER_TAG = "1.0.0"
GIT_URL = "unknown"
GIT_BRANCH = "unknown"
logger = logging.getLogger("searx")
SUBPROCESS_RUN_ENV = {
"PATH": os.environ["PATH"],
"LC_ALL": "C",
"LANGUAGE": "",
}
def subprocess_run(args, **kwargs):
"""Call :py:func:`subprocess.run` and return (striped) stdout. If returncode is
non-zero, raise a :py:func:`subprocess.CalledProcessError`.
"""
if not isinstance(args, (list, tuple)):
args = shlex.split(args)
kwargs["env"] = kwargs.get("env", SUBPROCESS_RUN_ENV)
kwargs["encoding"] = kwargs.get("encoding", "utf-8")
kwargs["stdout"] = subprocess.PIPE
kwargs["stderr"] = subprocess.PIPE
# raise CalledProcessError if returncode is non-zero
kwargs["check"] = True
proc = subprocess.run(args, **kwargs) # pylint: disable=subprocess-run-check
return proc.stdout.strip()
def get_git_url_and_branch():
# handle GHA directly
if "GITHUB_REPOSITORY" in os.environ and "GITHUB_REF_NAME" in os.environ:
git_url = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}"
git_branch = os.environ["GITHUB_REF_NAME"]
return git_url, git_branch
try:
ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}")
except subprocess.CalledProcessError:
ref = subprocess_run("git rev-parse --abbrev-ref master@{upstream}")
origin, git_branch = ref.split("/", 1)
git_url = subprocess_run(["git", "remote", "get-url", origin])
# get https:// url from git@ url
if git_url.startswith("git@"):
git_url = git_url.replace(":", "/", 2).replace("git@", "https://", 1)
if git_url.endswith(".git"):
git_url = git_url.replace(".git", "", 1)
return git_url, git_branch
def get_git_version():
git_commit_date_hash = subprocess_run(r"git show -s --date='format:%Y.%m.%d' --format='%cd+%h'")
# Remove leading zero from minor and patch level / replacement of PR-2122
# which depended on the git version: '2023.05.06+..' --> '2023.5.6+..'
git_commit_date_hash = git_commit_date_hash.replace('.0', '.')
tag_version = git_version = git_commit_date_hash
docker_tag = git_commit_date_hash.replace("+", "-")
# add "+dirty" suffix if there are uncommitted changes except searx/settings.yml
try:
subprocess_run("git diff --quiet -- . ':!searx/settings.yml' ':!utils/brand.env'")
except subprocess.CalledProcessError as e:
if e.returncode == 1:
git_version += "+dirty"
else:
logger.warning('"%s" returns an unexpected return code %i', e.returncode, e.cmd)
return git_version, tag_version, docker_tag
def get_information():
version_string = VERSION_STRING
version_tag = VERSION_TAG
docker_tag = DOCKER_TAG
git_url = GIT_URL
git_branch = GIT_BRANCH
try:
version_string, version_tag, docker_tag = get_git_version()
except subprocess.CalledProcessError as ex:
logger.error("Error while getting the version: %s", ex.stderr)
try:
git_url, git_branch = get_git_url_and_branch()
except subprocess.CalledProcessError as ex:
logger.error("Error while getting the git URL & branch: %s", ex.stderr)
return version_string, version_tag, docker_tag, git_url, git_branch
try:
vf = importlib.import_module('searx.version_frozen')
VERSION_STRING, VERSION_TAG, DOCKER_TAG, GIT_URL, GIT_BRANCH = (
vf.VERSION_STRING,
vf.VERSION_TAG,
vf.DOCKER_TAG,
vf.GIT_URL,
vf.GIT_BRANCH,
)
except ImportError:
VERSION_STRING, VERSION_TAG, DOCKER_TAG, GIT_URL, GIT_BRANCH = get_information()
logger.info("version: %s", VERSION_STRING)
if __name__ == "__main__":
import sys
if len(sys.argv) >= 2 and sys.argv[1] == "freeze":
VERSION_STRING, VERSION_TAG, DOCKER_TAG, GIT_URL, GIT_BRANCH = get_information()
# freeze the version (to create an archive outside a git repository)
python_code = f"""# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring
# this file is generated automatically by searx/version.py
VERSION_STRING = "{VERSION_STRING}"
VERSION_TAG = "{VERSION_TAG}"
DOCKER_TAG = "{DOCKER_TAG}"
GIT_URL = "{GIT_URL}"
GIT_BRANCH = "{GIT_BRANCH}"
"""
path = os.path.join(os.path.dirname(__file__), "version_frozen.py")
with open(path, "w", encoding="utf8") as f:
f.write(python_code)
print(f"{f.name} created")
# set file timestamp to commit timestamp
commit_timestamp = int(subprocess_run("git show -s --format=%ct"))
os.utime(path, (commit_timestamp, commit_timestamp))
else:
# output shell code to set the variables
# usage: eval "$(python -m searx.version)"
shell_code = f"""
VERSION_STRING="{VERSION_STRING}"
VERSION_TAG="{VERSION_TAG}"
DOCKER_TAG="{DOCKER_TAG}"
GIT_URL="{GIT_URL}"
GIT_BRANCH="{GIT_BRANCH}"
"""
print(shell_code)