From 48456caeb3cc701cc0d0e15091a42c5b5cb540d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Fri, 9 May 2025 18:46:20 +0000 Subject: [PATCH 01/21] chore: docker + github-actions dependabot (#4754) * chore: docker dependabot * Add github actions too --- .github/dependabot.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4128911f3..a106397f3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,3 +20,21 @@ updates: target-branch: "master" commit-message: prefix: "[upd] web-client (simple):" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "friday" + target-branch: "master" + commit-message: + prefix: "[upd] docker:" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "friday" + target-branch: "master" + commit-message: + prefix: "[upd] github-actions:" From e982b9f73205dec9da2c5c1fc757bb40c605c899 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Fri, 9 May 2025 23:05:20 +0200 Subject: [PATCH 02/21] [fix] documentation should run on push/pr Instead of executing the workflow after integration.yml completes correctly, let's run this workflow parallel to integration.yml restoring the original behaviour. --- .github/workflows/documentation.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 6d9f2cac5..629d9a272 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -4,11 +4,10 @@ name: Documentation # yamllint disable-line rule:truthy on: workflow_dispatch: - workflow_run: - workflows: - - Integration - types: - - completed + push: + branches: + - master + pull_request: branches: - master @@ -24,7 +23,6 @@ env: jobs: release: - if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' name: Release runs-on: ubuntu-24.04-arm permissions: @@ -56,7 +54,8 @@ jobs: - name: Build documentation run: make V=1 docs.clean docs.html - - name: Release + - if: github.ref_name == 'master' + name: Release uses: JamesIves/github-pages-deploy-action@v4 with: folder: "dist/docs" From 8e2e7774d7aeb13eac29c4dc16d65a2051ca250d Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sat, 10 May 2025 13:08:20 +0200 Subject: [PATCH 03/21] [mod] new l10n.yml workflow (#4734) l10n.yml will run after integration.yml finishes successfully (will defer anything depending on integration.yml until heavy loads like container building are moved to separate workflows) and in master branch. * After every integration.yml workflow completes successfully, only the `update` job runs. * Dispatch and Crontab triggers only the `pr` job. Style changes, cleanup and improved integration with CI by leveraging the use of shared cache between all workflows (not functional until all workflows have been refactored). --- .github/workflows/integration.yml | 42 ------- .github/workflows/l10n.yml | 136 ++++++++++++++++++++++ .github/workflows/translations-update.yml | 59 ---------- 3 files changed, 136 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/l10n.yml delete mode 100644 .github/workflows/translations-update.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9ca96bf69..eb4f458d7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -47,48 +47,6 @@ jobs: - name: Build themes run: make themes.all - babel: - name: Update translations branch - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - needs: - - python - - themes - permissions: - contents: write # for make V=1 weblate.push.translations - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Update transations - id: update - run: | - make V=1 weblate.push.translations - dockers: name: Docker if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml new file mode 100644 index 000000000..4220c5c65 --- /dev/null +++ b/.github/workflows/l10n.yml @@ -0,0 +1,136 @@ +--- +name: Translation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + schedule: + - cron: "05 07 * * 5" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + update: + if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + name: Update + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.push.translations" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Update translations + run: make V=1 weblate.push.translations + + pr: + if: | + github.repository_owner == 'searxng' + && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') + name: Pull Request + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.translations.commit" + contents: write + # For action "peter-evans/create-pull-request" + pull-requests: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Merge and push translation updates + run: make V=1 weblate.translations.commit + + - name: Create PR + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[l10n] update translations from Weblate" + commit-message: "[l10n] update translations from Weblate" + branch: "translations_update" + delete-branch: "true" + draft: "false" + signoff: "false" + labels: | + translation + + - name: Display information + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml deleted file mode 100644 index 85e141e7f..000000000 --- a/.github/workflows/translations-update.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Update translations" -on: # yamllint disable-line rule:truthy - schedule: - - cron: "05 07 * * 5" - workflow_dispatch: - -jobs: - babel: - name: "create PR for additions from weblate" - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Merge and push transation updates - run: | - make V=1 weblate.translations.commit - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - commit-message: '[l10n] update translations from Weblate' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: translations_update - delete-branch: true - draft: false - title: '[l10n] update translations from Weblate' - body: | - update translations from Weblate - labels: | - translation From 1b787ed35e9c51e335c42faee1f76695780ba4cb Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sat, 10 May 2025 13:59:31 +0200 Subject: [PATCH 04/21] [mod] refactor integration.yml (#4763) Style changes, cleanup and improved integration with CI by leveraging the use of shared cache between all workflows. --- .github/workflows/integration.yml | 102 ++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 27 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index eb4f458d7..abdaf0c18 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,58 +1,106 @@ +--- name: Integration -on: # yamllint disable-line rule:truthy +# yamllint disable-line rule:truthy +on: push: - branches: ["master"] + branches: + - master pull_request: - branches: ["master"] + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false permissions: contents: read +env: + PYTHON_VERSION: "3.13" + jobs: - python: + test: name: Python ${{ matrix.python-version }} runs-on: ubuntu-24.04 strategy: matrix: - os: [ubuntu-24.04] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages - - name: Set up Python + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' + python-version: "${{ matrix.python-version }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ matrix.python-version }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ matrix.python-version }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + - name: Run tests run: make V=1 ci.test - themes: - name: Themes - runs-on: ubuntu-24.04 + theme: + name: Theme + runs-on: ubuntu-24.04-arm steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Ubuntu packages - run: sudo ./utils/searxng.sh install buildhost - - name: Set up Python + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' - - name: Build themes + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version-file: "./.nvmrc" + + - name: Setup cache Node.js + uses: actions/cache@v4 + with: + key: "nodejs-${{ runner.arch }}-${{ hashFiles('./.nvmrc', './package.json') }}" + path: "./client/simple/node_modules/" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Build run: make themes.all dockers: name: Docker if: github.ref == 'refs/heads/master' needs: - - python - - themes + - test + - theme env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} runs-on: ubuntu-24.04 From d16854e67a5bf2f640aabf119c9b50f5a1a3f24f Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 11 May 2025 18:12:51 +0200 Subject: [PATCH 05/21] [mod] rework container deployment (#4764) container.yml will run after integration.yml COMPLETES successfully and in master branch. Style changes, cleanup and improved integration with CI by leveraging the use of shared cache between all workflows. * Podman is now supported to build the container images (Docker also received a refactor, merging both build and buildx) * Container images are being built by Buildah instead of Docker BuildKit. * Container images are tested before release. * Splitting "modern" (amd64 & arm64) and "legacy" (armv7) arches on different Dockerfiles allowing future optimizations. --- .github/workflows/container.yml | 183 ++++++++++ .github/workflows/integration.yml | 46 --- Makefile | 10 +- container/Dockerfile | 100 ++++++ .../docker-entrypoint.sh | 4 +- Dockerfile => container/legacy/Dockerfile | 13 +- {dockerfiles => container}/uwsgi.ini | 0 docs/admin/installation-docker.rst | 11 +- manage | 91 +---- searx/version.py | 6 + utils/lib_sxng_container.sh | 319 ++++++++++++++++++ 11 files changed, 628 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/container.yml create mode 100644 container/Dockerfile rename {dockerfiles => container}/docker-entrypoint.sh (97%) rename Dockerfile => container/legacy/Dockerfile (90%) rename {dockerfiles => container}/uwsgi.ini (100%) create mode 100644 utils/lib_sxng_container.sh diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 000000000..d232a0737 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,183 @@ +--- +name: Container + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + # Organization GHCR + packages: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + build: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + outputs: + version_string: ${{ steps.build.outputs.version_string }} + version_tag: ${{ steps.build.outputs.version_tag }} + docker_tag: ${{ steps.build.outputs.docker_tag }} + git_url: ${{ steps.build.outputs.git_url }} + git_branch: ${{ steps.build.outputs.git_branch }} + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + # yamllint disable-line rule:line-length + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile ./container/legacy/Dockerfile') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + id: build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make podman.build + + test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + GIT_URL: "${{ needs.build.outputs.git_url }}" + run: make container.test + + release: + if: github.repository_owner == 'searxng' && github.ref_name == 'master' + name: Release + runs-on: ubuntu-24.04-arm + needs: + - build + - test + + steps: + - if: env.DOCKERHUB_USERNAME != '' + name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: "docker.io" + username: "${{ env.DOCKERHUB_USERNAME }}" + password: "${{ secrets.DOCKERHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Release + env: + GIT_URL: "${{ needs.build.outputs.git_url }}" + DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" + run: make container.push diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index abdaf0c18..b40ae26ab 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -94,49 +94,3 @@ jobs: - name: Build run: make themes.all - - dockers: - name: Docker - if: github.ref == 'refs/heads/master' - needs: - - test - - theme - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - runs-on: ubuntu-24.04 - steps: - - name: Checkout - if: env.DOCKERHUB_USERNAME != null - uses: actions/checkout@v4 - with: - # make sure "make docker.push" can get the git history - fetch-depth: '0' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Set up QEMU - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - if: env.DOCKERHUB_USERNAME != null - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - if: env.DOCKERHUB_USERNAME != null - run: make -e GIT_URL=$(git remote get-url origin) docker.buildx diff --git a/Makefile b/Makefile index c1c067149..15e43be08 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ ci.test: test.yamllint test.black test.types.ci test.pylint test.unit test.robo test: test.yamllint test.black test.types.dev test.pylint test.unit test.robot test.rst test.shell test.shell: $(Q)shellcheck -x -s dash \ - dockerfiles/docker-entrypoint.sh + container/docker-entrypoint.sh $(Q)shellcheck -x -s bash \ utils/brand.sh \ $(MTOOLS) \ @@ -77,7 +77,9 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean -MANAGE += docker.build docker.push docker.buildx +MANAGE += podman.build +MANAGE += docker.build docker.buildx +MANAGE += container.build container.test container.push MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -95,8 +97,8 @@ $(MANAGE): # short hands of selected targets -PHONY += docs docker themes +PHONY += docs container themes docs: docs.html -docker: docker.build +container: container.build themes: themes.all diff --git a/container/Dockerfile b/container/Dockerfile new file mode 100644 index 000000000..b0530dfec --- /dev/null +++ b/container/Dockerfile @@ -0,0 +1,100 @@ +FROM docker.io/library/python:3.13-slim AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + brotli \ + # uwsgi + libpcre3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/local/searxng/ + +COPY ./requirements.txt ./requirements.txt + +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ + && . ./venv/bin/activate \ + && pip install -r requirements.txt \ + && pip install "uwsgi~=2.0" + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 + +RUN python -m compileall -q searx \ + && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ + && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \ + && find /usr/local/searxng/searx/static \ + \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ + -type f -exec gzip -9 -k {} + -exec brotli --best {} + + +ARG SEARXNG_UID=977 +ARG SEARXNG_GID=977 + +RUN grep -m1 root /etc/group > /tmp/.searxng.group \ + && grep -m1 root /etc/passwd > /tmp/.searxng.passwd \ + && echo "searxng:x:$SEARXNG_GID:" >> /tmp/.searxng.group \ + && echo "searxng:x:$SEARXNG_UID:$SEARXNG_GID:searxng:/usr/local/searxng:/bin/bash" >> /tmp/.searxng.passwd + +FROM docker.io/library/python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # healthcheck + wget \ + # uwsgi + libpcre3 \ + libxml2 \ + mailcap \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --from=builder /tmp/.searxng.passwd /etc/passwd +COPY --chown=root:root --from=builder /tmp/.searxng.group /etc/group + +ARG LABEL_DATE="0001-01-01T00:00:00Z" +ARG GIT_URL="unspecified" +ARG SEARXNG_GIT_VERSION="unspecified" +ARG LABEL_VCS_REF="unspecified" +ARG LABEL_VCS_URL="unspecified" + +WORKDIR /usr/local/searxng/ + +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./container/ ./container/ + +LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ + org.opencontainers.image.created=$LABEL_DATE \ + org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ + org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision=$LABEL_VCS_REF \ + org.opencontainers.image.source=$LABEL_VCS_URL \ + org.opencontainers.image.title="searxng" \ + org.opencontainers.image.url=$LABEL_VCS_URL \ + org.opencontainers.image.version=$SEARXNG_GIT_VERSION + +ENV CONFIG_PATH=/etc/searxng \ + DATA_PATH=/var/cache/searxng + +ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ + INSTANCE_NAME=searxng \ + AUTOCOMPLETE="" \ + BASE_URL="" \ + BIND_ADDRESS=[::]:8080 \ + MORTY_KEY="" \ + MORTY_URL="" \ + SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ + UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ + UWSGI_WORKERS=%k \ + UWSGI_THREADS=4 + +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"] diff --git a/dockerfiles/docker-entrypoint.sh b/container/docker-entrypoint.sh similarity index 97% rename from dockerfiles/docker-entrypoint.sh rename to container/docker-entrypoint.sh index 3668fb589..72d020dcf 100755 --- a/dockerfiles/docker-entrypoint.sh +++ b/container/docker-entrypoint.sh @@ -140,14 +140,14 @@ if [ "$SEARX_CONF" -eq "1" ]; then cat << EOF > /etc/searx/deprecated_volume_read_me.txt This Docker image uses the volume /etc/searxng Update your configuration: -* remove uwsgi.ini (or very carefully update your existing uwsgi.ini using https://github.com/searxng/searxng/blob/master/dockerfiles/uwsgi.ini ) +* remove uwsgi.ini (or very carefully update your existing uwsgi.ini using https://github.com/searxng/searxng/blob/master/container/uwsgi.ini ) * mount /etc/searxng instead of /etc/searx EOF fi # end of searx compatibility # make sure there are uwsgi settings -update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/dockerfiles/uwsgi.ini" "patch_uwsgi_settings" +update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/container/uwsgi.ini" "patch_uwsgi_settings" # make sure there are searxng settings update_conf "${FORCE_CONF_UPDATE}" "${SEARXNG_SETTINGS_PATH}" "/usr/local/searxng/searx/settings.yml" "patch_searxng_settings" diff --git a/Dockerfile b/container/legacy/Dockerfile similarity index 90% rename from Dockerfile rename to container/legacy/Dockerfile index 9aeb28214..5436ea5da 100644 --- a/Dockerfile +++ b/container/legacy/Dockerfile @@ -1,3 +1,5 @@ +# For armv7 architecture + FROM docker.io/library/python:3.13-slim AS builder RUN apt-get update \ @@ -16,8 +18,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -# Readd on #4707 "--mount=type=cache,id=pip,target=/root/.cache/pip" -RUN python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" @@ -29,7 +30,7 @@ ARG TIMESTAMP_UWSGI=0 RUN python -m compileall -q searx \ && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ - && touch -c --date=@$TIMESTAMP_UWSGI ./dockerfiles/uwsgi.ini \ + && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \ && find /usr/local/searxng/searx/static \ \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ -type f -exec gzip -9 -k {} + -exec brotli --best {} + @@ -69,7 +70,7 @@ WORKDIR /usr/local/searxng/ COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ -COPY --chown=searxng:searxng ./dockerfiles/ ./dockerfiles/ +COPY --chown=searxng:searxng ./container/ ./container/ LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ org.opencontainers.image.created=$LABEL_DATE \ @@ -90,8 +91,6 @@ ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ AUTOCOMPLETE="" \ BASE_URL="" \ BIND_ADDRESS=[::]:8080 \ - MORTY_KEY="" \ - MORTY_URL="" \ SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ UWSGI_WORKERS=%k \ @@ -104,4 +103,4 @@ EXPOSE 8080 HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 -ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] +ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"] diff --git a/dockerfiles/uwsgi.ini b/container/uwsgi.ini similarity index 100% rename from dockerfiles/uwsgi.ini rename to container/uwsgi.ini diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst index 09471891b..06b3fe465 100644 --- a/docs/admin/installation-docker.rst +++ b/docs/admin/installation-docker.rst @@ -145,13 +145,6 @@ shell inside container - `How to make bash scripts work in dash `_ - `Checking for Bashisms `_ -Like in many other distributions, Alpine's `/bin/sh -`__ is :man:`dash`. Dash is meant to be -`POSIX-compliant `__. -Compared to debian, in the Alpine image :man:`bash` is not installed. The -:origin:`dockerfiles/docker-entrypoint.sh` script is checked *against dash* -(``make tests.shell``). - To open a shell inside the container: .. code:: sh @@ -188,10 +181,10 @@ Command line `__. In the :origin:`Dockerfile` the ENTRYPOINT_ is defined as -:origin:`dockerfiles/docker-entrypoint.sh` +:origin:`container/docker-entrypoint.sh` .. code:: sh docker run --rm -it searxng/searxng -h -.. program-output:: ../dockerfiles/docker-entrypoint.sh -h +.. program-output:: ../container/docker-entrypoint.sh -h diff --git a/manage b/manage index 61bc68b74..ee2a29281 100755 --- a/manage +++ b/manage @@ -11,6 +11,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" +# shellcheck source=utils/lib_sxng_container.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_container.sh" + # shellcheck source=utils/lib_sxng_data.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" @@ -77,9 +80,6 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -docker.: - build : build docker image - push : build and push docker image gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -101,6 +101,7 @@ EOF go.help node.help weblate.help + container.help data.help test.help themes.help @@ -136,90 +137,6 @@ webapp.run() { SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } -docker.push() { - docker.build push -} - -docker.buildx() { - docker.build buildx -} - -# shellcheck disable=SC2119 -docker.build() { - pyenv.install - - local SEARXNG_GIT_VERSION - local VERSION_GITCOMMIT - local GITHUB_USER - local SEARXNG_IMAGE_NAME - local BUILD - - build_msg DOCKER build - # run installation in a subprocess and activate pyenv - - # See https://www.shellcheck.net/wiki/SC1001 and others .. - # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e - pyenv.activate - - # Check if it is a git repository - if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" - fi - - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" - fi - - # This is a git repository - git update-index -q --refresh - python -m searx.version freeze - eval "$(python -m searx.version)" - - # Get the last git commit id - VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" - - # define the docker image name - GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') - SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" - - BUILD="build" - if [ "$1" = "buildx" ]; then - # buildx includes the push option - CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" - BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" - shift - fi - build_msg DOCKER "Build command: ${BUILD}" - - # build Docker image - build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" - # shellcheck disable=SC2086 - docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . - - if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi - ) - dump_return $? -} - # shellcheck disable=SC2119 gecko.driver() { pyenv.install diff --git a/searx/version.py b/searx/version.py index d2013808b..565cc7e7a 100644 --- a/searx/version.py +++ b/searx/version.py @@ -41,6 +41,12 @@ def subprocess_run(args, **kwargs): def get_git_url_and_branch(): + # handle GHA directly + if "GITHUB_REPOSITORY" in os.environ and "GITHUB_REF_NAME" in os.environ: + git_url = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" + git_branch = os.environ["GITHUB_REF_NAME"] + return git_url, git_branch + try: ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}") except subprocess.CalledProcessError: diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh new file mode 100644 index 000000000..b3f84594f --- /dev/null +++ b/utils/lib_sxng_container.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +container.help() { + cat </dev/null; then + die 1 "Git is not installed" + fi + + # Check if podman or docker is installed + if [ "$1" = "docker" ]; then + if command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Docker is not installed" + fi + elif [ "$1" = "podman" ]; then + if command -v podman &>/dev/null; then + container_engine="podman" + else + die 1 "Podman is not installed" + fi + else + # If no explicit engine is passed, prioritize podman over docker + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Podman/Docker is not installed" + fi + fi + info_msg "Selected engine: $container_engine" + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + dockerfile="legacy/Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; $parch" + exit 1 + ;; + esac + info_msg "Selected platform: $platform" + + pyenv.install + + ( + set -e + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin &>/dev/null; then + die 1 "There is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + info_msg "Set \$VERSION_STRING: $VERSION_STRING" + info_msg "Set \$VERSION_TAG: $VERSION_TAG" + info_msg "Set \$DOCKER_TAG: $DOCKER_TAG" + info_msg "Set \$GIT_URL: $GIT_URL" + info_msg "Set \$GIT_BRANCH: $GIT_BRANCH" + + if [ "$container_engine" = "podman" ]; then + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" + else + params_build_builder="build --platform=$platform --target=builder" + params_build="build --platform=$platform --squash" + fi + + if [ "$GITHUB_ACTIONS" = "true" ]; then + params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + params_build+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + + # Tags + params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + else + # Tags + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest" + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG" + fi + + # shellcheck disable=SC2086 + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./container/uwsgi.ini)" \ + --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \ + --file="./container/$dockerfile" \ + . + build_msg CONTAINER "Image \"builder\" built" + + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --file="./container/$dockerfile" \ + . + build_msg CONTAINER "Image built" + + if [ "$GITHUB_ACTIONS" = "true" ]; then + "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + # Output to GHA + { + echo "version_string=$VERSION_STRING" + echo "version_tag=$VERSION_TAG" + echo "docker_tag=$DOCKER_TAG" + echo "git_url=$GIT_URL" + echo "git_branch=$GIT_BRANCH" + } >>"$GITHUB_OUTPUT" + fi + ) + dump_return $? +} + +container.test() { + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; $parch" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + ( + set -e + + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + name="$CONTAINER_IMAGE_NAME-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + +container.push() { + # Architectures on manifest + local release_archs=("amd64" "arm64" "armv7") + + local archs=() + local variants=() + local platforms=() + + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + for arch in "${release_archs[@]}"; do + case $arch in + "X64" | "x86_64" | "amd64") + archs+=("amd64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARM64" | "aarch64" | "arm64") + archs+=("arm64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARMV7" | "armv7" | "armhf" | "arm") + archs+=("arm") + variants+=("v7") + platforms+=("linux/${archs[-1]}/${variants[-1]}") + ;; + *) + err_msg "Unsupported architecture; $arch" + exit 1 + ;; + esac + done + + ( + set -e + + # Pull archs + for i in "${!archs[@]}"; do + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + + # Manifest tags + release_tags=("latest") + release_tags+=("$DOCKER_TAG") + + # Create manifests + for tag in "${release_tags[@]}"; do + if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then + podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + fi + + # Add archs to manifest + for i in "${!archs[@]}"; do + podman manifest add \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + done + + podman image list + + # Push manifests + for tag in "${release_tags[@]}"; do + build_msg CONTAINER "Pushing manifest with tag: $tag" + + podman manifest push \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "docker://docker.io/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + done + ) + dump_return $? +} + +# Alias +podman.build() { + container.build podman +} + +# Alias +docker.build() { + container.build docker +} + +# Alias +docker.buildx() { + container.build docker +} From da3c640cef71032bced6f4e8baf1665801cdc4b9 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sun, 11 May 2025 18:39:25 +0200 Subject: [PATCH 06/21] revert using null keyword env.DOCKERHUB_USERNAME shouldn't be an empty string as it's defined and set (I think, I can't see this). Even if wasn't defined, GitHub Org/Repo wide envs/secrets should return an empty string (?) --- .github/workflows/container.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index d232a0737..ee1093acb 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -153,13 +153,13 @@ jobs: - test steps: - - if: env.DOCKERHUB_USERNAME != '' + - if: env.DOCKERHUB_USERNAME != null name: Checkout uses: actions/checkout@v4 with: persist-credentials: "false" - - if: env.DOCKERHUB_USERNAME != '' + - if: env.DOCKERHUB_USERNAME != null name: Login to GHCR uses: docker/login-action@v3 with: @@ -167,7 +167,7 @@ jobs: username: "${{ github.repository_owner }}" password: "${{ secrets.GITHUB_TOKEN }}" - - if: env.DOCKERHUB_USERNAME != '' + - if: env.DOCKERHUB_USERNAME != null name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -175,7 +175,7 @@ jobs: username: "${{ env.DOCKERHUB_USERNAME }}" password: "${{ secrets.DOCKERHUB_TOKEN }}" - - if: env.DOCKERHUB_USERNAME != '' + - if: env.DOCKERHUB_USERNAME != null name: Release env: GIT_URL: "${{ needs.build.outputs.git_url }}" From 750a7b4d6fba37f5d0d6bf42b99a865dc7e035e8 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 May 2025 08:27:16 +0200 Subject: [PATCH 07/21] [doc] add chapter on the purpose of (git) commits (#4729) Chapter on the purpose of (git) commits The commits and their messages are elementary for the traceability of changes and are unfortunately still too often given too little attention. It therefore seems necessary to dedicate a chapter to this topic in the context of development. Signed-off-by: Markus Heiser --- docs/dev/commits.rst | 108 ++++++++++++++++++++++++++++++++ docs/dev/contribution_guide.rst | 30 +++------ docs/dev/index.rst | 1 + 3 files changed, 116 insertions(+), 23 deletions(-) create mode 100644 docs/dev/commits.rst diff --git a/docs/dev/commits.rst b/docs/dev/commits.rst new file mode 100644 index 000000000..e9b10fa2e --- /dev/null +++ b/docs/dev/commits.rst @@ -0,0 +1,108 @@ +.. _create commit: + +=============================== +Git Commits & Change Management +=============================== + +.. sidebar:: Create good commits! + + - `Conventional Commits`_ + - `Structural split of changes`_ + - `Git Commit Good Practice`_ + +A commit and its commit message are among the most important information +available to a developer for bug fixing and further development. A commit is a +change and changes have a context (a change request). + +In a SCM system (git), the change history is derived from the commit history. A +commit message is therefore part of the documentation for change management and +thus elementary for the traceability of changes. + +**What a commit is not**: *A commit to an SCM system is not used to save files!* + +A commit should always have a context and the commit message describes what is +to be changed in that context, just as a function description should describe +what the intention and the goal of the function is, a commit message should +describe what the intention and the goal of that commit is. + +The commit messages form the history and are the first and therefore most +important information a developer has when he has to research when and why a +change had to be made and how it was made (what the goal was). + +Like any text, a commit message should be written for the reader and not from +the perspective of the author. + +When scrolling through the history, the first thing one see is the title of the +commit message. Therefore the title should describe the change as briefly and +precisely as possible ... followed by a blank line and then a somewhat detailed +description of the change. + +---- + +The follwing rules should be in mind, when creating a commit: + +- **Commit history should be read like a history book.** +- **Commit messages are for the reader not for the author of the commit.** +- **A commit is the atomic code-modification of a change in change management.** +- **Think about which descriptions from your PR might belong in the commit message.** +- **The maximum line length in a commit message is 80 characters.** + +---- + +Choose meaningful commit messages: + + .. code:: + + [type] optional scope: description + + [body] + + [optional trailers] + +``[type]``: + Commits MUST be prefixed with a type .. ``feat``, ``fix``, ``refactor``, + ``mod``, ``upd``, ``doc``, ``l10n``, ``build`` .. + +``[body]`` + `Information in commit messages`_ + +``[optional trailers]``: + - `Signed-off-by`_: certify that the committer has the rights to submit the + work under the project’s license. That the developer has this right is a + prerequisite for a merge. If the `Signed-off-by`_ is not set in the + commit, the contributor enters his `Developer's Certificate of Origin` at + the latest when creating a PR! + - Closes: Link to the bug report or the bug number (e.g. ``Closes: #10``) + - `Co-authored-by`_: email address of the co-author + - Reported-by: email address (if there is no bug report) + - Suggested-by: email address (if there is no bug report) + +---- + +To give examples at hand, here are a few commits. Follow the links to see the +full commit messages: + +:patch:`44d941c93` + ``[fix] mojeek web engine: don't add empty fmt argument for web searches`` + +:patch:`feb15e387` + ``[fix] brave.news engine: response is HTML and no longer JSON`` + +:patch:`bdfe1c2a1` + ``[mod] engines: migration of the individual cache solutions to EngineCache`` + + +.. _Conventional Commits: + https://www.conventionalcommits.org/ +.. _Structural split of changes: + https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes +.. _Git Commit Good Practice: + https://wiki.openstack.org/wiki/GitCommitMessages +.. _Information in commit messages: + https://wiki.openstack.org/wiki/GitCommitMessages#Information_in_commit_messages +.. _`Developer's Certificate of Origin`: + https://developercertificate.org/ +.. _Signed-off-by: + https://git-scm.com/docs/git-commit#Documentation/git-commit.txt-code--signoffcode +.. _Co-authored-by: + https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/creating-a-commit-with-multiple-authors diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 61dd28c05..026cfbe73 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -30,15 +30,15 @@ SearXNG was born out of the need for a **privacy-respecting** search tool which can be extended easily to maximize both its search and its privacy protecting capabilities. -Some widely used search engine features may work differently, -may be turned off by default, or may not be implemented at all in SearXNG +Some widely used search engine features may work differently, +may be turned off by default, or may not be implemented at all in SearXNG **as a consequence of a privacy-by-design approach**. Following this approach, features reducing the privacy preserving aspects of SearXNG should be switched off by default or should not be implemented at all. There are plenty of search engines already providing such features. If a feature reduces -SearXNG's efficacy in protecting a user's privacy, the user must be informed about -the effect of choosing to enable it. Features that protect privacy but differ from the +SearXNG's efficacy in protecting a user's privacy, the user must be informed about +the effect of choosing to enable it. Features that protect privacy but differ from the expectations of the user should also be carefully explained to them. Also, if you think that something works weird with SearXNG, it might be because @@ -56,20 +56,12 @@ Code ==== .. _PEP8: https://www.python.org/dev/peps/pep-0008/ -.. _Conventional Commits: https://www.conventionalcommits.org/ -.. _Git Commit Good Practice: https://wiki.openstack.org/wiki/GitCommitMessages .. _Structural split of changes: https://wiki.openstack.org/wiki/GitCommitMessages#Structural_split_of_changes -.. _gitmoji: https://gitmoji.carloscuesta.me/ -.. _Semantic PR: https://github.com/zeke/semantic-pull-requests .. sidebar:: Create good commits! - - `Structural split of changes`_ - - `Conventional Commits`_ - - `Git Commit Good Practice`_ - - some like to use: gitmoji_ - - not yet active: `Semantic PR`_ + - :ref:`create commit` In order to submit a patch, please follow the steps below: @@ -88,15 +80,7 @@ In order to submit a patch, please follow the steps below: - Add yourself to the :origin:`AUTHORS.rst` file. -- Choose meaningful commit messages, read `Conventional Commits`_ - - .. code:: - - [optional scope]: - - [optional body] - - [optional footer(s)] +- Choose meaningful commit messages, see :ref:`create commit` - Create a pull request. @@ -161,7 +145,7 @@ changed (:ref:`make docs.clean`). Live builds are implemented by sphinx-autobuild_. Use environment ``$(SPHINXOPTS)`` to pass arguments to the sphinx-autobuild_ command. You can -pass any argument except for the ``--host`` option (which is always set to ``0.0.0.0``). +pass any argument except for the ``--host`` option (which is always set to ``0.0.0.0``). E.g., to find and use a free port, use: .. code:: sh diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 01a16ba80..6d601d166 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -6,6 +6,7 @@ Developer documentation :maxdepth: 2 quickstart + commits rtm_asdf contribution_guide extended_types From 346d7fe0192ea180552adbd2877dc13b3dd4050e Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 12 May 2025 16:42:43 +0200 Subject: [PATCH 08/21] [mod] CI: dependabot group updates by minor & patch and major updates (#4773) By default, Dependabot opens a new pull request to update each dependency and we have a very large number of Dependabot pull requests to review and merge, which can quickly become difficult to manage. The intention of this patch to dependabot is to have: - one PR for all minor & patch level updates - one PR for every dependency with a major update [1] https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/optimizing-pr-creation-version-updates [2] https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#groups Signed-off-by: Markus Heiser --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a106397f3..46e341362 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,6 +10,12 @@ updates: target-branch: "master" commit-message: prefix: "[upd] pypi:" + groups: + minor: + applies-to: version-updates + update-types: + - "minor" + - "patch" - package-ecosystem: "npm" directory: "/client/simple" @@ -20,6 +26,12 @@ updates: target-branch: "master" commit-message: prefix: "[upd] web-client (simple):" + groups: + minor: + applies-to: version-updates + update-types: + - "minor" + - "patch" - package-ecosystem: "docker" directory: "/" From 64a5b6920f1fd386d5ec8bce575b2ea0c82b2572 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Mon, 12 May 2025 10:36:04 +0200 Subject: [PATCH 09/21] [mod] lib_sxng_container.sh use required_commands() helper Suggested-by: @return42 https://github.com/searxng/searxng/pull/4764#discussion_r2083564489 --- utils/lib_sxng_container.sh | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh index b3f84594f..1df9ef358 100644 --- a/utils/lib_sxng_container.sh +++ b/utils/lib_sxng_container.sh @@ -19,24 +19,14 @@ container.build() { local variant local platform - # Check if git is installed - if ! command -v git &>/dev/null; then - die 1 "Git is not installed" - fi + required_commands git # Check if podman or docker is installed - if [ "$1" = "docker" ]; then - if command -v docker &>/dev/null; then - container_engine="docker" - else - die 1 "Docker is not installed" - fi - elif [ "$1" = "podman" ]; then - if command -v podman &>/dev/null; then - container_engine="podman" - else - die 1 "Podman is not installed" + if [ "$1" = "podman" ] || [ "$1" = "docker" ]; then + if ! command -v "$1" &>/dev/null; then + die 42 "$1 is not installed" fi + container_engine="$1" else # If no explicit engine is passed, prioritize podman over docker if command -v podman &>/dev/null; then @@ -44,7 +34,7 @@ container.build() { elif command -v docker &>/dev/null; then container_engine="docker" else - die 1 "Podman/Docker is not installed" + die 42 "no compatible container engine is installed (podman or docker)" fi fi info_msg "Selected engine: $container_engine" @@ -168,10 +158,7 @@ container.test() { die 1 "This command is intended to be run in GitHub Actions" fi - # Check if podman is installed - if ! command -v podman &>/dev/null; then - die 1 "podman is not installed" - fi + required_commands podman # Setup arch specific case $parch in @@ -234,10 +221,7 @@ container.push() { die 1 "This command is intended to be run in GitHub Actions" fi - # Check if podman is installed - if ! command -v podman &>/dev/null; then - die 1 "podman is not installed" - fi + required_commands podman for arch in "${release_archs[@]}"; do case $arch in From 945b30a1c459e6b437a3f379e638c2002a81fcab Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Mon, 12 May 2025 10:41:49 +0200 Subject: [PATCH 10/21] [mod] lib_sxng_container.sh: replace echo commands with heredoc Suggested-by: @return42 https://github.com/searxng/searxng/pull/4764#discussion_r2083571202 --- utils/lib_sxng_container.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh index 1df9ef358..f87b64e2b 100644 --- a/utils/lib_sxng_container.sh +++ b/utils/lib_sxng_container.sh @@ -136,13 +136,13 @@ container.build() { "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" # Output to GHA - { - echo "version_string=$VERSION_STRING" - echo "version_tag=$VERSION_TAG" - echo "docker_tag=$DOCKER_TAG" - echo "git_url=$GIT_URL" - echo "git_branch=$GIT_BRANCH" - } >>"$GITHUB_OUTPUT" + cat <>"$GITHUB_OUTPUT" +version_string=$VERSION_STRING +version_tag=$VERSION_TAG +docker_tag=$DOCKER_TAG +git_url=$GIT_URL +git_branch=$GIT_BRANCH +EOF fi ) dump_return $? From 5d99373bc65c7087ee743a1fe44897bad6065338 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Mon, 12 May 2025 23:43:47 +0200 Subject: [PATCH 11/21] [fix] DOCKERHUB_USERNAME env reference (#4778) When making the container rework, I unknowingly deleted the section where an env with the same name as the secret was defined on the job scope, making it look like it was originally defined as an organization env. Since we can't validate the secrets in a condition directly, it's better to let docker/login-action take care of failing the entire job if the credentials are invalid. Reported in: https://github.com/searxng/searxng/issues/4777 --- .github/workflows/container.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index ee1093acb..ef92a0662 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -153,30 +153,26 @@ jobs: - test steps: - - if: env.DOCKERHUB_USERNAME != null - name: Checkout + - name: Checkout uses: actions/checkout@v4 with: persist-credentials: "false" - - if: env.DOCKERHUB_USERNAME != null - name: Login to GHCR + - name: Login to GHCR uses: docker/login-action@v3 with: registry: "ghcr.io" username: "${{ github.repository_owner }}" password: "${{ secrets.GITHUB_TOKEN }}" - - if: env.DOCKERHUB_USERNAME != null - name: Login to Docker Hub + - name: Login to Docker Hub uses: docker/login-action@v3 with: registry: "docker.io" - username: "${{ env.DOCKERHUB_USERNAME }}" + username: "${{ secrets.DOCKERHUB_USERNAME }}" password: "${{ secrets.DOCKERHUB_TOKEN }}" - - if: env.DOCKERHUB_USERNAME != null - name: Release + - name: Release env: GIT_URL: "${{ needs.build.outputs.git_url }}" DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" From 90068660196d898896219d1df7a088348c5d3d14 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Tue, 13 May 2025 10:18:28 +0200 Subject: [PATCH 12/21] [fix] engine archlinux: avoid Anubis challenge by User-Agent "SearXNG" (#4779) Of the archlinux wikis only wiki.archlinux.org has a has Anubis challenge. About Anubis[1]: > Anubis decides to present a challenge using this logic: > > - User-Agent contains "Mozilla" > ... > This should ensure that git clients, RSS readers, and other low-harm clients > can get through without issue .. [1] https://github.com/TecharoHQ/anubis/blob/6c0ff3f4d5e7a66ea85e5873e6ee4c28692091d7/docs/docs/design/how-anubis-works.mdx#challenge-presentation Suggested-by: @unixfox https://github.com/searxng/searxng/issues/4646#issuecomment-2855322406 Closes: https://github.com/searxng/searxng/issues/4646 Signed-off-by: Markus Heiser --- searx/engines/archlinux.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py index 820b31799..63de157dc 100644 --- a/searx/engines/archlinux.py +++ b/searx/engines/archlinux.py @@ -51,6 +51,9 @@ def request(query, params): if netloc == main_wiki: eng_lang: str = traits.get_language(sxng_lang, 'English') # type: ignore query += ' (' + eng_lang + ')' + # wiki.archlinux.org is protected by anubis + # - https://github.com/searxng/searxng/issues/4646#issuecomment-2817848019 + params['headers']['User-Agent'] = "SearXNG" elif netloc == 'wiki.archlinuxcn.org': base_url = 'https://' + netloc + '/wzh/index.php?' From 2cfd3fc44b16e7f3c6010c0ac1004efb6dacf382 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Sat, 5 Apr 2025 10:59:07 +0200 Subject: [PATCH 13/21] [enh] tidy: clean old morty, filtron, searx references Everyone should have already switched from legacy methods --- Makefile | 5 +- container/docker-entrypoint.sh | 43 +----- docs/admin/arch_public.dot | 2 +- docs/admin/update-searxng.rst | 45 ------- docs/dev/lxcdev.rst | 5 +- docs/own-instance.rst | 3 +- docs/utils/searxng.sh.rst | 2 +- searx/settings.yml | 12 -- searx/settings_defaults.py | 5 - searx/templates/simple/macros.html | 4 +- .../simple/result_templates/code.html | 2 +- .../simple/result_templates/default.html | 2 +- .../simple/result_templates/map.html | 2 +- .../simple/result_templates/paper.html | 2 +- .../simple/result_templates/products.html | 2 +- .../simple/result_templates/torrent.html | 2 +- .../simple/result_templates/videos.html | 2 +- searx/webapp.py | 25 ---- tests/unit/settings/user_settings_simple.yml | 3 - utils/filtron.sh | 125 ------------------ utils/lib.sh | 4 +- utils/morty.sh | 124 ----------------- utils/searx.sh | 88 ------------ utils/searxng.sh | 9 -- .../lib/systemd/system/filtron.service | 29 ---- .../lib/systemd/system/morty.service | 29 ---- 26 files changed, 18 insertions(+), 558 deletions(-) delete mode 100755 utils/filtron.sh delete mode 100755 utils/morty.sh delete mode 100755 utils/searx.sh delete mode 100644 utils/templates/lib/systemd/system/filtron.service delete mode 100644 utils/templates/lib/systemd/system/morty.service diff --git a/Makefile b/Makefile index 15e43be08..917d3aeb4 100644 --- a/Makefile +++ b/Makefile @@ -65,10 +65,7 @@ test.shell: utils/lib_redis.sh \ utils/searxng.sh \ utils/lxc.sh \ - utils/lxc-searxng.env \ - utils/searx.sh \ - utils/filtron.sh \ - utils/morty.sh + utils/lxc-searxng.env $(Q)$(MTOOLS) build_msg TEST "$@ OK" diff --git a/container/docker-entrypoint.sh b/container/docker-entrypoint.sh index 72d020dcf..ee14b2f05 100755 --- a/container/docker-entrypoint.sh +++ b/container/docker-entrypoint.sh @@ -12,8 +12,7 @@ Environment variables: INSTANCE_NAME settings.yml : general.instance_name AUTOCOMPLETE settings.yml : search.autocomplete BASE_URL settings.yml : server.base_url - MORTY_URL settings.yml : result_proxy.url - MORTY_KEY settings.yml : result_proxy.key + Volume: /etc/searxng the docker entry point copies settings.yml and uwsgi.ini in this directory (see the -f command line option)" @@ -70,20 +69,6 @@ patch_searxng_settings() { -e "s/autocomplete: \"\"/autocomplete: \"${AUTOCOMPLETE}\"/g" \ -e "s/ultrasecretkey/$(head -c 24 /dev/urandom | base64 | tr -dc 'a-zA-Z0-9')/g" \ "${CONF}" - - # Morty configuration - - if [ -n "${MORTY_KEY}" ] && [ -n "${MORTY_URL}" ]; then - sed -i -e "s/image_proxy: false/image_proxy: true/g" \ - "${CONF}" - cat >> "${CONF}" <<-EOF - -# Morty configuration -result_proxy: - url: ${MORTY_URL} - key: !!binary "${MORTY_KEY}" -EOF - fi } update_conf() { @@ -122,30 +107,6 @@ update_conf() { fi } -# searx compatibility: copy /etc/searx/* to /etc/searxng/* -SEARX_CONF=0 -if [ -f "/etc/searx/settings.yml" ]; then - if [ ! -f "${SEARXNG_SETTINGS_PATH}" ]; then - printf '⚠️ /etc/searx/settings.yml is copied to /etc/searxng\n' - cp "/etc/searx/settings.yml" "${SEARXNG_SETTINGS_PATH}" - fi - SEARX_CONF=1 -fi -if [ -f "/etc/searx/uwsgi.ini" ]; then - printf '⚠️ /etc/searx/uwsgi.ini is ignored. Use the volume /etc/searxng\n' - SEARX_CONF=1 -fi -if [ "$SEARX_CONF" -eq "1" ]; then - printf '⚠️ The deprecated volume /etc/searx is mounted. Please update your configuration to use /etc/searxng ⚠️\n' - cat << EOF > /etc/searx/deprecated_volume_read_me.txt -This Docker image uses the volume /etc/searxng -Update your configuration: -* remove uwsgi.ini (or very carefully update your existing uwsgi.ini using https://github.com/searxng/searxng/blob/master/container/uwsgi.ini ) -* mount /etc/searxng instead of /etc/searx -EOF -fi -# end of searx compatibility - # make sure there are uwsgi settings update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/container/uwsgi.ini" "patch_uwsgi_settings" @@ -158,8 +119,6 @@ if [ $DRY_RUN -eq 1 ]; then exit fi -unset MORTY_KEY - printf 'Listen on %s\n' "${BIND_ADDRESS}" # Start uwsgi diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index 526fb53da..49b03d157 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -6,7 +6,7 @@ digraph G { browser [label="browser", shape=tab, fillcolor=aliceblue]; rp [label="reverse proxy"]; static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray]; - uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"] + uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searxng.sh.html"] redis [label="redis DB", shape=cylinder]; searxng1 [label="SearXNG #1", fontcolor=blue3]; searxng2 [label="SearXNG #2", fontcolor=blue3]; diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index b9d15c3f7..16715f00d 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -58,9 +58,6 @@ and then, to name just a few: - Bot protection has been switched from filtron to SearXNG's :ref:`limiter `, this requires a :ref:`Redis ` database. -- The image proxy morty is no longer needed, it has been replaced by the - :ref:`image proxy ` from SearXNG. - - To save bandwidth :ref:`cache busting ` has been implemented. To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi setup`. @@ -71,12 +68,6 @@ examples show, this is not always enough, sometimes services have to be set up or reconfigured and sometimes services that are no longer needed should be uninstalled. -.. hint:: - - First of all: SearXNG is installed by the script :ref:`searxng.sh`. If you - have old filtron, morty or searx setup you should consider complete - uninstall/reinstall. - Here you will find a list of changes that affect the infrastructure. Please check to what extent it is necessary to update your installations: @@ -85,39 +76,6 @@ check to what extent it is necessary to update your installations: file manually. -remove obsolete services ------------------------- - -If your searx instance was installed *"Step by step"* or by the *"Installation -scripts"*, you need to undo the installation procedure completely. If you have -morty & filtron installed, it is recommended to uninstall these services also. -In case of scripts, to uninstall use the scripts from the origin you installed -searx from or try:: - - $ sudo -H ./utils/filtron.sh remove all - $ sudo -H ./utils/morty.sh remove all - $ sudo -H ./utils/searx.sh remove all - -.. hint:: - - If you are migrate from searx take into account that the ``.config.sh`` is no - longer used. - -If you upgrade from searx or from before :pull:`1332` has been merged and you -have filtron and/or morty installed, don't forget to remove HTTP sites. - -Apache:: - - $ sudo -H ./utils/filtron.sh apache remove - $ sudo -H ./utils/morty.sh apache remove - -nginx:: - - $ sudo -H ./utils/filtron.sh nginx remove - $ sudo -H ./utils/morty.sh nginx remove - - - Check after Installation ------------------------ @@ -130,9 +88,6 @@ to see if there are some left overs. In this example there exists a *old* SearXNG checks -------------- ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/ - INFO: [OK] (old) account 'searx' does not exists - INFO: [OK] (old) account 'filtron' does not exists - INFO: [OK] (old) account 'morty' does not exists ... INFO searx.redisdb : connecting to Redis db=0 path='/usr/local/searxng-redis/run/redis.sock' INFO searx.redisdb : connected to Redis diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index 79716ae57..9edd9f672 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -319,13 +319,13 @@ To *inspect* the SearXNG instance (already described above): .. code:: bash - $ ./utils/searx.sh inspect service + $ ./utils/searxng.sh inspect service .. group-tab:: desktop (HOST) .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searx.sh inspect service + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh inspect service Run :ref:`makefile`, e.g. to test inside the container: @@ -435,4 +435,3 @@ use: [searxng-archlinux] SEARXNG_URL : http:///n.n.n.140/searxng [searxng-archlinux] SEARXNG_PORT : 8888 [searxng-archlinux] SEARXNG_BIND_ADDRESS : 127.0.0.1 - diff --git a/docs/own-instance.rst b/docs/own-instance.rst index 7d79f26c2..8c0009aff 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -39,8 +39,7 @@ Removing private data means not sending cookies to external search engines and generating a random browser profile for every request. Thus, it does not matter if a public or private instance handles the request, because it is anonymized in both cases. The IP address used will be the IP of the instance, but SearXNG can also be -configured to use proxy or Tor. `Result proxy -`__ is supported, too. +configured to use proxy or Tor. SearXNG does not serve ads or tracking content, unlike most search services. Therefore, private data is not forwarded to third parties who might monetize it. Besides diff --git a/docs/utils/searxng.sh.rst b/docs/utils/searxng.sh.rst index bedc1ba4c..d45d7014f 100644 --- a/docs/utils/searxng.sh.rst +++ b/docs/utils/searxng.sh.rst @@ -28,7 +28,7 @@ In most cases you will install SearXNG simply by running the command: .. code:: bash - sudo -H ./utils/searx.sh install all + sudo -H ./utils/searxng.sh install all The installation is described in chapter :ref:`installation basic`. diff --git a/searx/settings.yml b/searx/settings.yml index d756e9b1c..7e0455701 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -169,18 +169,6 @@ ui: # - image_proxy # - query_in_title -# searx supports result proxification using an external service: -# https://github.com/asciimoo/morty uncomment below section if you have running -# morty proxy the key is base64 encoded (keep the !!binary notation) -# Note: since commit af77ec3, morty accepts a base64 encoded key. -# -# result_proxy: -# url: http://127.0.0.1:3000/ -# # the key is a base64 encoded string, the YAML !!binary prefix is optional -# key: !!binary "your_morty_proxy_key" -# # [true|false] enable the "proxy" button next to each result -# proxify_results: true - # communication with search engines # outgoing: diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index b91657ff6..4cee7e345 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -230,11 +230,6 @@ SCHEMA = { 'extra_proxy_timeout': SettingsValue(int, 0), 'networks': {}, }, - 'result_proxy': { - 'url': SettingsValue((None, str), None), - 'key': SettingsBytesValue((None, bytes), None), - 'proxify_results': SettingsValue(bool, False), - }, 'plugins': SettingsValue(dict, {}), 'checker': { 'off_when_debug': SettingsValue(bool, True, None), diff --git a/searx/templates/simple/macros.html b/searx/templates/simple/macros.html index 6010a5a3d..df8469d72 100644 --- a/searx/templates/simple/macros.html +++ b/searx/templates/simple/macros.html @@ -44,10 +44,10 @@ {%- endmacro -%} -{%- macro result_sub_footer(result, proxify) -%} +{%- macro result_sub_footer(result) -%}
{% for engine in result.engines %}{{ engine }}{% endfor %} - {{ icon_small('ellipsis-vertical') + result_link(cache_url + result.url, _('cached'), "cache_link") }}‎ {% if proxify and proxify_results %} {{ result_link(proxify(result.url), _('proxied'), "proxyfied_link") }} {% endif %} + {{ icon_small('ellipsis-vertical') + result_link(cache_url + result.url, _('cached'), "cache_link") }}
{{- '' -}}
{{- '' -}} {%- endmacro -%} diff --git a/searx/templates/simple/result_templates/code.html b/searx/templates/simple/result_templates/code.html index 7d2c8ff79..49326aed5 100644 --- a/searx/templates/simple/result_templates/code.html +++ b/searx/templates/simple/result_templates/code.html @@ -28,5 +28,5 @@ {{- result.codelines|code_highlighter(result.code_language)|safe -}} -{{- result_sub_footer(result, proxify) -}} +{{- result_sub_footer(result) -}} {{- result_footer(result) -}} diff --git a/searx/templates/simple/result_templates/default.html b/searx/templates/simple/result_templates/default.html index 263c19a36..8a6329248 100644 --- a/searx/templates/simple/result_templates/default.html +++ b/searx/templates/simple/result_templates/default.html @@ -14,7 +14,7 @@ {{ _('This site did not provide any description.')|safe }}

{% endif -%} -{{- result_sub_footer(result, proxify) -}} +{{- result_sub_footer(result) -}} {% if result.iframe_src -%}