mirror of
https://github.com/searxng/searxng.git
synced 2025-07-31 00:52:22 +02:00
[mod] add Golang ecosystem to the SearXNG toolchain
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser
Markus Heiser
parent
ae0fcf3a42
commit
20a193f04c
5 changed files with 250 additions and 216 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -9,6 +9,7 @@ geckodriver.log
|
|||
.coverage
|
||||
coverage/
|
||||
|
||||
.govm/
|
||||
.nvm/
|
||||
cache/
|
||||
build/
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -32,7 +32,7 @@ install uninstall:
|
|||
$(Q)./manage pyenv.$@
|
||||
|
||||
PHONY += clean
|
||||
clean: py.clean docs.clean node.clean nvm.clean test.clean
|
||||
clean: py.clean docs.clean node.clean nvm.clean go.clean test.clean
|
||||
$(Q)./manage build_msg CLEAN "common files"
|
||||
$(Q)find . -name '*.orig' -exec rm -f {} +
|
||||
$(Q)find . -name '*.rej' -exec rm -f {} +
|
||||
|
|
@ -57,7 +57,7 @@ test.shell:
|
|||
$(MTOOLS) \
|
||||
utils/lib.sh \
|
||||
utils/lib_sxng*.sh \
|
||||
utils/lib_go.sh \
|
||||
utils/lib_govm.sh \
|
||||
utils/lib_nvm.sh \
|
||||
utils/lib_redis.sh \
|
||||
utils/lib_valkey.sh \
|
||||
|
|
@ -82,6 +82,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover
|
|||
MANAGE += themes.all themes.simple themes.fix themes.lint themes.test
|
||||
MANAGE += static.build.commit static.build.drop static.build.restore
|
||||
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
|
||||
MANAGE += go.env.dev go.clean
|
||||
|
||||
PHONY += $(MANAGE)
|
||||
|
||||
|
|
|
|||
23
manage
23
manage
|
|
@ -32,8 +32,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
|
|||
# shellcheck source=utils/lib_sxng_test.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
|
||||
|
||||
# shellcheck source=utils/lib_go.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
|
||||
# shellcheck source=utils/lib_govm.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_govm.sh"
|
||||
|
||||
# shellcheck source=utils/lib_valkey.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
|
||||
|
|
@ -41,7 +41,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_valkey.sh"
|
|||
# shellcheck source=utils/lib_sxng_vite.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"
|
||||
|
||||
PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
|
||||
# add ./local dev tools from python (virtualenv), golang and nodejs
|
||||
PATH="${PY_ENV}/bin:${REPO_ROOT}/node_modules/.bin:${GOROOT}/bin:${GOPATH}/bin:${PATH}"
|
||||
|
||||
# config
|
||||
|
||||
|
|
@ -100,11 +101,27 @@ EOF
|
|||
static.help
|
||||
vite.help
|
||||
cat <<EOF
|
||||
dev.:
|
||||
env: enter developer environment (or exec a command in)
|
||||
environment ...
|
||||
SEARXNG_VALKEY_URL : ${SEARXNG_VALKEY_URL}
|
||||
EOF
|
||||
}
|
||||
|
||||
dev.env() {
|
||||
go.env.dev
|
||||
node.env.dev
|
||||
|
||||
export GOENV
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
export PS1="(dev.env)$ "
|
||||
bash --norc --noprofile
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
if [ "$VERBOSE" = "1" ]; then
|
||||
SPHINX_VERBOSE="-v"
|
||||
|
|
|
|||
211
utils/lib_go.sh
211
utils/lib_go.sh
|
|
@ -1,211 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
#
|
||||
# Tools to install and maintain golang [1] binaries & packages.
|
||||
#
|
||||
# [1] https://golang.org/doc/devel/release#policy
|
||||
#
|
||||
# A simple *helloworld* test with user 'my_user' :
|
||||
#
|
||||
# sudo -H adduser my_user
|
||||
# ./manage go.golang go1.17.3 my_user
|
||||
# ./manage go.install github.com/go-training/helloworld@latest my_user
|
||||
# ./manage go.bash my_user
|
||||
# $ helloword
|
||||
# Hello World!!
|
||||
#
|
||||
# Don't forget to remove 'my_user': sudo -H deluser --remove-home my_user
|
||||
|
||||
# shellcheck source=utils/lib.sh
|
||||
. /dev/null
|
||||
|
||||
# configure golang environment
|
||||
# ----------------------------
|
||||
|
||||
[[ -z "${GO_VERSION}" ]] && GO_VERSION="go1.17.3"
|
||||
|
||||
GO_DL_URL="https://golang.org/dl"
|
||||
|
||||
# implement go functions
|
||||
# -----------------------
|
||||
|
||||
go.help(){
|
||||
cat <<EOF
|
||||
go.:
|
||||
ls : list golang binary archives (stable)
|
||||
golang : (re-) install golang binary in user's \$HOME/local folder
|
||||
install : install go package in user's \$HOME/go-apps folder
|
||||
bash : start bash interpreter with golang environment sourced
|
||||
EOF
|
||||
}
|
||||
|
||||
go.ls(){
|
||||
python <<EOF
|
||||
import sys, json, requests
|
||||
resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
|
||||
for ver in json.loads(resp.text):
|
||||
if not ver['stable']:
|
||||
continue
|
||||
for f in ver['files']:
|
||||
if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
|
||||
continue
|
||||
print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
|
||||
EOF
|
||||
}
|
||||
|
||||
go.ver_info(){
|
||||
|
||||
# print information about a golang distribution. To print filename
|
||||
# sha256 and size of the archive that fits to your OS and host:
|
||||
#
|
||||
# go.ver_info "${GO_VERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
||||
#
|
||||
# usage: go.ver_info <go-vers> <kind> <os> <arch> [filename|sha256|size]
|
||||
#
|
||||
# kind: [archive|source|installer]
|
||||
# os: [darwin|freebsd|linux|windows]
|
||||
# arch: [amd64|arm64|386|armv6l|ppc64le|s390x]
|
||||
|
||||
python - "$@" <<EOF
|
||||
import sys, json, requests
|
||||
resp = requests.get("${GO_DL_URL}/?mode=json&include=all")
|
||||
for ver in json.loads(resp.text):
|
||||
if ver['version'] != sys.argv[1]:
|
||||
continue
|
||||
for f in ver['files']:
|
||||
if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
|
||||
continue
|
||||
for x in sys.argv[5:]:
|
||||
print(f[x])
|
||||
sys.exit(0)
|
||||
sys.exit(42)
|
||||
EOF
|
||||
}
|
||||
|
||||
go.os() {
|
||||
local OS
|
||||
case "$(command uname -a)xx" in
|
||||
Linux\ *) OS=linux ;;
|
||||
Darwin\ *) OS=darwin ;;
|
||||
FreeBSD\ *) OS=freebsd ;;
|
||||
CYGWIN* | MSYS* | MINGW*) OS=windows ;;
|
||||
*) die 42 "OS is unknown: $(command uname -a)" ;;
|
||||
esac
|
||||
echo "${OS}"
|
||||
}
|
||||
|
||||
go.arch() {
|
||||
local ARCH
|
||||
case "$(command uname -m)" in
|
||||
"x86_64") ARCH=amd64 ;;
|
||||
"aarch64") ARCH=arm64 ;;
|
||||
"armv6" | "armv7l") ARCH=armv6l ;;
|
||||
"armv8") ARCH=arm64 ;;
|
||||
.*386.*) ARCH=386 ;;
|
||||
ppc64*) ARCH=ppc64le ;;
|
||||
*) die 42 "ARCH is unknown: $(command uname -m)" ;;
|
||||
esac
|
||||
echo "${ARCH}"
|
||||
}
|
||||
|
||||
go.golang() {
|
||||
|
||||
# install golang binary in user's $HOME/local folder:
|
||||
#
|
||||
# go.golang ${GO_VERSION} ${SERVICE_USER}
|
||||
#
|
||||
# usage: go.golang <go-vers> [<username>]
|
||||
|
||||
local version fname sha size user userpr
|
||||
local buf=()
|
||||
|
||||
version="${1:-${GO_VERSION}}"
|
||||
user="${2:-${USERNAME}}"
|
||||
userpr=" ${_Yellow}|${user}|${_creset} "
|
||||
|
||||
rst_title "Install Go in ${user}'s HOME" section
|
||||
|
||||
mapfile -t buf < <(
|
||||
go.ver_info "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
||||
)
|
||||
|
||||
if [ ${#buf[@]} -eq 0 ]; then
|
||||
die 42 "can't find info of golang version: ${version}"
|
||||
fi
|
||||
fname="${buf[0]}"
|
||||
sha="${buf[1]}"
|
||||
size="$(numfmt --to=iec "${buf[2]}")"
|
||||
|
||||
info_msg "Download go binary ${fname} (${size}B)"
|
||||
cache_download "${GO_DL_URL}/${fname}" "${fname}"
|
||||
|
||||
pushd "${CACHE}" &> /dev/null
|
||||
echo "${sha} ${fname}" > "${fname}.sha256"
|
||||
if ! sha256sum -c "${fname}.sha256" >/dev/null; then
|
||||
die 42 "downloaded file ${fname} checksum does not match"
|
||||
else
|
||||
info_msg "${fname} checksum OK"
|
||||
fi
|
||||
popd &> /dev/null
|
||||
|
||||
info_msg "install golang"
|
||||
tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
|
||||
mkdir -p \$HOME/local
|
||||
rm -rf \$HOME/local/go
|
||||
tar -C \$HOME/local -xzf ${CACHE}/${fname}
|
||||
echo "export GOPATH=\$HOME/go-apps" > \$HOME/.go_env
|
||||
echo "export PATH=\$HOME/local/go/bin:\\\$GOPATH/bin:\\\$PATH" >> \$HOME/.go_env
|
||||
EOF
|
||||
info_msg "test golang installation"
|
||||
sudo -i -u "${user}" <<EOF
|
||||
source \$HOME/.go_env
|
||||
command -v go
|
||||
go version
|
||||
EOF
|
||||
}
|
||||
|
||||
go.install() {
|
||||
|
||||
# install go package in user's $HOME/go-apps folder:
|
||||
#
|
||||
# go.install github.com/go-training/helloworld@lates ${SERVICE_USER}
|
||||
#
|
||||
# usage: go.install <package> [<username>]
|
||||
|
||||
local package user userpr
|
||||
|
||||
package="${1}"
|
||||
user="${2:-${USERNAME}}"
|
||||
userpr=" ${_Yellow}|${user}|${_creset} "
|
||||
|
||||
if [ -z "${package}" ]; then
|
||||
die 42 "${FUNCNAME[0]}() - missing argument: <package>"
|
||||
fi
|
||||
tee_stderr 0.1 <<EOF | sudo -i -u "${user}" | prefix_stdout "${userpr}"
|
||||
source \$HOME/.go_env
|
||||
go install -v ${package}
|
||||
EOF
|
||||
}
|
||||
|
||||
go.bash() {
|
||||
|
||||
# start bash interpreter with golang environment sourced
|
||||
#
|
||||
# go.bash ${SERVICE_USER}
|
||||
#
|
||||
# usage: go.bash [<username>]
|
||||
|
||||
local user
|
||||
user="${1:-${USERNAME}}"
|
||||
sudo -i -u "${user}" bash --init-file "~${user}/.go_env"
|
||||
}
|
||||
|
||||
go.version(){
|
||||
local user
|
||||
user="${1:-${USERNAME}}"
|
||||
sudo -i -u "${user}" <<EOF
|
||||
source \$HOME/.go_env
|
||||
go version | cut -d' ' -f 3
|
||||
EOF
|
||||
}
|
||||
226
utils/lib_govm.sh
Executable file
226
utils/lib_govm.sh
Executable file
|
|
@ -0,0 +1,226 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
#
|
||||
# Go versions manager to install and maintain golang [1] binaries & packages.
|
||||
#
|
||||
# [1] https://golang.org/doc/devel/release#policy
|
||||
|
||||
# shellcheck source=utils/lib.sh
|
||||
. /dev/null
|
||||
|
||||
# configure golang environment for go.vm
|
||||
# --------------------------------------
|
||||
|
||||
_GO_DL_URL="https://go.dev/dl"
|
||||
|
||||
GOVERSION="${GOVERSION:-go$(awk '/^go /{print $2}' "${REPO_ROOT}/go.mod")}"
|
||||
GOTOOLCHAIN=local
|
||||
|
||||
GOROOT="${REPO_ROOT}/.govm/${GOVERSION}"
|
||||
GOENV="${GOROOT}/.config/go.env"
|
||||
GOVM_EXE="${GOROOT}/bin/go"
|
||||
|
||||
GOPATH="${REPO_ROOT}/local/${GOVERSION}" # no support for multiple path names!!
|
||||
GOCACHE="${GOPATH}/.cache/go-build"
|
||||
GOMODCACHE="${GOPATH}/pkg/mod"
|
||||
|
||||
# implement go functions
|
||||
# -----------------------
|
||||
|
||||
go.help() {
|
||||
cat <<EOF
|
||||
go: GOROOT=${GOROOT}
|
||||
install : compiles and installs packages
|
||||
EOF
|
||||
}
|
||||
|
||||
go.tool() {
|
||||
# shortcut for "go tool .." in the Go environment
|
||||
go.env.dev
|
||||
"${GOVM_EXE}" tool "$@"
|
||||
}
|
||||
|
||||
go.env.dev() {
|
||||
if [ -z "$_GO_DEVTOOLS_INSTALLED" ]; then
|
||||
build_msg INSTALL "[pkg.go.dev] ./go.mod: developer and CI tools"
|
||||
go.tidy
|
||||
else
|
||||
go.vm.ensure
|
||||
_GO_DEVTOOLS_INSTALLED=1
|
||||
fi
|
||||
}
|
||||
|
||||
go.tidy() {
|
||||
go.vm.ensure
|
||||
"${GOVM_EXE}" mod tidy
|
||||
chmod -R u+w "${GOMODCACHE}"
|
||||
}
|
||||
|
||||
go.clean() {
|
||||
if ! go.vm.is_installed; then
|
||||
build_msg CLEAN "[Go] not installed"
|
||||
return 0
|
||||
fi
|
||||
build_msg CLEAN "[Go] drop folders ${GOROOT} and ${GOPATH}"
|
||||
rm -rf "${GOROOT}" "${GOPATH}"
|
||||
}
|
||||
|
||||
go.install() {
|
||||
go.vm.ensure
|
||||
GOENV="${GOENV}" "${GOVM_EXE}" install "$@"
|
||||
# not sure why, but go installs some files without setting the write access
|
||||
# for the file owner
|
||||
chmod -R u+w "${GOMODCACHE}"
|
||||
}
|
||||
|
||||
go.os() {
|
||||
local OS
|
||||
case "$(command uname -a)xx" in
|
||||
Linux\ *) OS=linux ;;
|
||||
Darwin\ *) OS=darwin ;;
|
||||
FreeBSD\ *) OS=freebsd ;;
|
||||
CYGWIN* | MSYS* | MINGW*) OS=windows ;;
|
||||
*) die 42 "OS is unknown: $(command uname -a)" ;;
|
||||
esac
|
||||
echo "${OS}"
|
||||
}
|
||||
|
||||
go.arch() {
|
||||
local ARCH
|
||||
case "$(command uname -m)" in
|
||||
"x86_64") ARCH=amd64 ;;
|
||||
"aarch64") ARCH=arm64 ;;
|
||||
"armv6" | "armv7l") ARCH=armv6l ;;
|
||||
"armv8") ARCH=arm64 ;;
|
||||
.*386.*) ARCH=386 ;;
|
||||
ppc64*) ARCH=ppc64le ;;
|
||||
*) die 42 "ARCH is unknown: $(command uname -m)" ;;
|
||||
esac
|
||||
echo "${ARCH}"
|
||||
}
|
||||
|
||||
# Go version management (go.vm)
|
||||
# -----------------------------
|
||||
|
||||
go.vm.ensure() {
|
||||
if ! go.vm.is_installed; then
|
||||
# shellcheck disable=SC2119
|
||||
go.vm.install
|
||||
fi
|
||||
}
|
||||
|
||||
go.vm.is_installed() {
|
||||
# is true if "go" command is installed
|
||||
[[ -f "${GOROOT}/bin/go" ]]
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2120
|
||||
go.vm.install() {
|
||||
|
||||
python -m pip install -U requests
|
||||
|
||||
# Go versions manager; to install Go at arbitrary place:
|
||||
#
|
||||
# usage: go.vm.install <version> <dest>
|
||||
|
||||
local version dest fname sha size tmp
|
||||
version="${1:-$GOVERSION}"
|
||||
dest="${2:-$GOROOT}"
|
||||
|
||||
info_msg "Install Go in ${dest}"
|
||||
|
||||
# HINT: the python requirements needed by go.vm.version are taken from the
|
||||
# developer environment. If it is not yet installed, install it now ..
|
||||
pyenv.install
|
||||
|
||||
# fetch go version ..
|
||||
local buf=()
|
||||
mapfile -t buf < <(
|
||||
go.vm.version "${version}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
||||
)
|
||||
if [ ${#buf[@]} -eq 0 ]; then
|
||||
die 42 "can't find info of golang version: ${version}"
|
||||
fi
|
||||
fname="${buf[0]}"
|
||||
sha="${buf[1]}"
|
||||
size="$(numfmt --to=iec "${buf[2]}")"
|
||||
|
||||
info_msg "Download go binary ${fname} (${size}B)"
|
||||
cache_download "${_GO_DL_URL}/${fname}" "${fname}"
|
||||
|
||||
pushd "${CACHE}" &>/dev/null
|
||||
echo "${sha} ${fname}" >"${fname}.sha256"
|
||||
if ! sha256sum -c "${fname}.sha256" >/dev/null; then
|
||||
die 42 "downloaded file ${fname} checksum does not match"
|
||||
else
|
||||
info_msg "${fname} checksum OK"
|
||||
fi
|
||||
popd &>/dev/null
|
||||
|
||||
info_msg "install golang"
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
tar -C "${tmp}" -xzf "${CACHE}/${fname}"
|
||||
rm -rf "${dest}"
|
||||
mkdir -p "$(dirname "${dest}")"
|
||||
mv "${tmp}/go" "${dest}"
|
||||
|
||||
mkdir -p "$(dirname "$GOENV")"
|
||||
export GOENV
|
||||
|
||||
"${GOVM_EXE}" telemetry off
|
||||
"${GOVM_EXE}" env -w \
|
||||
GOBIN="$GOBIN" \
|
||||
GOTOOLCHAIN="$GOTOOLCHAIN" \
|
||||
GOCACHE="$GOCACHE" \
|
||||
GOPATH="$GOPATH" \
|
||||
GOMODCACHE="$GOMODCACHE"
|
||||
|
||||
mkdir -p "${GOMODCACHE}"
|
||||
}
|
||||
|
||||
go.vm.list() {
|
||||
|
||||
# Go versions manager; list Go versions (stable)
|
||||
|
||||
python <<EOF
|
||||
import sys, json, requests
|
||||
resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
|
||||
for ver in json.loads(resp.text):
|
||||
if not ver['stable']:
|
||||
continue
|
||||
for f in ver['files']:
|
||||
if f['kind'] != 'archive' or not f['size'] or not f['sha256'] or len(f['os']) < 2:
|
||||
continue
|
||||
print(" %(version)-10s|%(os)-8s|%(arch)-8s|%(filename)-30s|%(size)-10s|%(sha256)s" % f)
|
||||
EOF
|
||||
}
|
||||
|
||||
go.vm.version() {
|
||||
|
||||
# Print information about a Go distribution. To print filename sha256 and
|
||||
# size of the archive that fits to your OS and host:
|
||||
#
|
||||
# go.ver_info "${GOVERSION}" archive "$(go.os)" "$(go.arch)" filename sha256 size
|
||||
#
|
||||
# usage: go.vm.version <go-vers> <kind> <os> <arch> [filename|sha256|size]
|
||||
#
|
||||
# kind: [archive|source|installer]
|
||||
# os: [darwin|freebsd|linux|windows]
|
||||
# arch: [amd64|arm64|386|armv6l|ppc64le|s390x]
|
||||
|
||||
python - "$@" <<EOF
|
||||
import sys, json, requests
|
||||
resp = requests.get("${_GO_DL_URL}/?mode=json&include=all")
|
||||
for ver in json.loads(resp.text):
|
||||
if ver['version'] != sys.argv[1]:
|
||||
continue
|
||||
for f in ver['files']:
|
||||
if (f['kind'] != sys.argv[2] or f['os'] != sys.argv[3] or f['arch'] != sys.argv[4]):
|
||||
continue
|
||||
for x in sys.argv[5:]:
|
||||
print(f[x])
|
||||
sys.exit(0)
|
||||
sys.exit(42)
|
||||
EOF
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue