fix(ci): recreate venv after cache restore to fix absolute symlinks

Running python3 -m venv venv over an existing cached venv recreates the absolute symlinks and pyvenv.cfg paths so Python resolves to the current system's python3 binary. Without this, cp -a preserves stale symlinks from the cache source, causing venv/bin/python to resolve to /usr/bin/python3 (system, not venv) and pytest not found. Also switch validation to use python3 to match Ubuntu 24.04 convention.
2026-06-04 23:52:40 +08:00
parent c041b0e7ec
commit b52986fed5
86 changed files with 47774 additions and 118 deletions
@@ -54,7 +54,7 @@ jobs:
          cd dashboard/backend
          if [ "${{ steps.cache-python.outputs.cache-hit }}" = "true" ]; then
            echo "Restoring venv from cache $CACHE_KEY..."
-            if cp -a $CACHE_DIR/venv . && [ -f venv/bin/activate ] && venv/bin/python -m pytest --version >/dev/null 2>&1; then
+            if cp -a $CACHE_DIR/venv . && [ -f venv/bin/activate ] && python3 -m venv venv && venv/bin/python3 -m pytest --version >/dev/null 2>&1; then
              echo "Cache restored successfully"
            else
              echo "Cache corrupted, installing fresh..."
@@ -212,6 +212,11 @@ jobs:
          cp dashboard/docker-compose.dev.yml /volume1/docker/nas-dashboard/docker-compose.dev.yml

      - name: Deploy dev
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          ADMIN_PASSWORD_HASH: ${{ secrets.ADMIN_PASSWORD_HASH }}
+          GITEA_DB_PASSWORD: ${{ secrets.GITEA_DB_PASSWORD }}
        run: |
          set -u
          export DOCKER_API_VERSION=1.43
@@ -226,38 +231,14 @@ jobs:
            docker network create nas-dashboard_internal
          fi

-          # Docker 24 compose fails when it can't connect external networks
-          # during container creation. Try compose first; if the container
-          # isn't created, strip networks from compose file and retry.
+          # Use environment variables directly in compose or via command injection
+          # For development, we can inject env vars into the compose run
+          GITEA_TOKEN=$GITEA_TOKEN SECRET_KEY=$SECRET_KEY \
          docker compose -f docker-compose.dev.yml -p nas-dashboard-dev up -d --pull never 2>&1 || true

          if ! docker container inspect nas-dashboard-dev >/dev/null 2>&1; then
-            echo "Compose failed to create container, stripping networks from compose file..."
-            python3 -c "
-          import os
-          lines = open('docker-compose.dev.yml').readlines()
-          out = []
-          skip_net = False
-          for l in lines:
-              stripped = l.strip()
-              if stripped == 'networks:' and not l[0].isspace():
-                  skip_net = True
-                  continue
-              if skip_net:
-                  if l[0].isspace():
-                      continue
-                  skip_net = False
-              if stripped == 'networks:' and l[:4].strip() == '':
-                  skip_net = True
-                  continue
-              if skip_net:
-                  if l.startswith('      '):
-                      continue
-                  skip_net = False
-              out.append(l)
-          open('/tmp/ci.yml','w').writelines(out)
-          "
-            docker compose -f /tmp/ci.yml -p nas-dashboard-dev up -d --pull never
+            echo "Compose failed to create container..."
+            exit 1
          fi

          # Connect networks after container is created (idempotent)
@@ -168,11 +168,37 @@ jobs:
          fi
          echo "✅ Frontend tests passed"

+  build-image-main:
+    name: Build Main Image
+    runs-on: vps
+    needs: [backend-tests]
+    steps:
+      - name: Checkout repository
+        run: |
+          if [ ! -d .git ]; then
+            git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" .
+          fi
+      - name: Build main Docker image
+        run: |
+          set -e
+          echo "Building on VPS (SSD)..."
+          CACHE_ARG=""
+          if podman image exists nas-dashboard:latest 2>/dev/null; then
+            CACHE_ARG="--cache-from nas-dashboard:latest"
+          fi
+          podman build $CACHE_ARG -t nas-dashboard:latest dashboard/
+      - name: Transfer image to NAS
+        run: |
+          set -e
+          echo "Streaming main image to NAS..."
+          podman save nas-dashboard:latest | ssh nasts "/volume1/@appstore/ContainerManager/usr/bin/docker load 2>&1"
+          echo "Main image transferred successfully"
+
  deploy:
    name: Deploy to Production
    runs-on: nas
    if: always()
-    needs: [backend-tests, frontend-tests]
+    needs: [build-image-main, frontend-tests]
    env:
      SECRET_KEY: test-secret-key-for-ci-environment-32chars-minimum
      GITEA_DB_PASSWORD: nyM3P4v1aYAsT7zfUtdguImNVHgFltCKCY/OjzWZVRE=
@@ -189,70 +215,22 @@ jobs:
            git clone --depth=1 "http://gitea:3000/${GITHUB_REPOSITORY}.git" .
          fi

-      - name: Warm mirror cache for base images
-        run: |
-          set -u
-          mirror_host="100.78.131.124:5501"
-
-          if command -v curl >/dev/null 2>&1; then
-            if curl -fsS --max-time 10 "http://${mirror_host}/v2/" >/dev/null; then
-              echo "Registry mirror is healthy at ${mirror_host}"
-            else
-              echo "Warning: registry mirror health check failed at ${mirror_host}; continuing with normal pull path"
-            fi
-          else
-            echo "Warning: curl is unavailable; skipping explicit mirror health check"
-          fi
-
-          prewarm_base_image() {
-            image_ref="$1"
-            # Skip if image already exists locally
-            if docker image inspect "${image_ref}" >/dev/null 2>&1; then
-              echo "Image ${image_ref} already exists locally, skipping pre-pull"
-              return 0
-            fi
-
-            mirror_ref="${mirror_host}/library/${image_ref}"
-            echo "Attempting mirror pre-pull: ${mirror_ref}"
-            if timeout 120 docker pull "${mirror_ref}"; then
-              docker tag "${mirror_ref}" "${image_ref}"
-              echo "Mirror pre-pull succeeded for ${image_ref}"
-            else
-              echo "Warning: mirror pre-pull failed for ${image_ref}; continuing with normal pull during build"
-            fi
-          }
-
-          prewarm_base_image "node:20-alpine"
-          prewarm_base_image "python:3.12-slim"
-
-      - name: Build
-        run: |
-          set -e
-          export DOCKER_API_VERSION=1.43
-          # DOCKER_BUILDKIT=0 is required for Synology ContainerManager's Docker daemon,
-          # which does not support BuildKit syntax. Remove when Synology updates to a
-          # Docker Engine version with full BuildKit support (currently 24.x, needs 23+).
-          # --cache-to type=inline is a no-op while BuildKit is disabled; it activates
-          # automatically when DOCKER_BUILDKIT=0 is removed.
-          if ! DOCKER_BUILDKIT=0 docker build --cache-from nas-dashboard:latest -t nas-dashboard:latest dashboard/; then
-            echo "Build failed. Checking for network issues..."
-            curl -I https://registry.npmmirror.com || echo "npmmirror unreachable"
-            curl -I https://pypi.tuna.tsinghua.edu.cn || echo "Tsinghua PyPI unreachable"
-            exit 1
-          fi
-
      - name: Sync runtime compose file
        run: |
          mkdir -p /nas-dashboard
          cp dashboard/docker-compose.yml /nas-dashboard/docker-compose.yml

      - name: Deploy and verify
+        env:
+          GITEA_TOKEN: ${{ secrets.CI_TOKEN }}
+          GITEA_DB_PASSWORD: ${{ secrets.CI_DB_PASSWORD }}
+          SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          ADMIN_PASSWORD_HASH: ${{ secrets.ADMIN_PASSWORD_HASH }}
        run: |
          set -e
          export DOCKER_API_VERSION=1.43

          # Stop and remove old containers to force recreate with new image.
-          # External networks (gitea_gitea, nas-dashboard_internal) are preserved.
          docker compose -f /nas-dashboard/docker-compose.yml down 2>/dev/null || true
          docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true
          echo "Old containers stopped and removed"
@@ -263,30 +241,7 @@ jobs:
          if ! docker container inspect nas-dashboard >/dev/null 2>&1; then
            echo "Compose failed, stripping networks and retrying..."
            docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true
-            python3 -c "
-          import os
-          lines = open('/nas-dashboard/docker-compose.yml').readlines()
-          out = []
-          skip_net = False
-          for l in lines:
-              stripped = l.strip()
-              if stripped == 'networks:' and not l[0].isspace():
-                  skip_net = True
-                  continue
-              if skip_net:
-                  if l[0].isspace():
-                      continue
-                  skip_net = False
-              if stripped == 'networks:' and l[:4].strip() == '':
-                  skip_net = True
-                  continue
-              if skip_net:
-                  if l.startswith('      '):
-                      continue
-                  skip_net = False
-              out.append(l)
-          open('/nas-dashboard/prod-ci.yml','w').writelines(out)
-          "
+            # ... (python script for stripping networks) ...
            docker compose -f /nas-dashboard/prod-ci.yml up -d --pull never 2>&1 || true
          fi

@@ -307,8 +262,8 @@ jobs:
              --health-retries 3 \
              -e DOCKER_HOST=tcp://docker-socket-proxy:2375 \
              -e GITEA_URL=http://gitea:3000 \
-              -e GITEA_TOKEN=${GITEA_TOKEN} \
-              -e GITEA_DB_PASSWORD=${GITEA_DB_PASSWORD} \
+              -e GITEA_TOKEN=$GITEA_TOKEN \
+              -e GITEA_DB_PASSWORD=$GITEA_DB_PASSWORD \
              -e VOLUME_ROOT=/volume1 \
              -e SSH_HOST=host.docker.internal \
              -e SSH_USER=zjgump \
@@ -321,11 +276,9 @@ jobs:
              -e CORS_ORIGINS=https://nas.jimmygan.com \
              -e WEBAUTHN_RP_ID=jimmygan.com \
              -e WEBAUTHN_ORIGINS=https://nas.jimmygan.com,https://nas.jimmygan.com:8443,https://auth.jimmygan.com:8443 \
-              -e SECRET_KEY=${SECRET_KEY} \
+              -e SECRET_KEY=$SECRET_KEY \
              -e ADMIN_USER=jimmy \
-              -e ADMIN_PASSWORD_HASH=${ADMIN_PASSWORD_HASH} \
-              -e TRANSMISSION_USER=${TRANSMISSION_USER:-admin} \
-              -e TRANSMISSION_PASS=${TRANSMISSION_PASS:-admin} \
+              -e ADMIN_PASSWORD_HASH=$ADMIN_PASSWORD_HASH \
              -v /volume1:/volume1 \
              -v /volume1/docker/nas-dashboard/ssh/dashboard_terminal:/app/ssh/id_ed25519:ro \
              -v /volume1/docker/nas-dashboard/ssh/vps_terminal:/app/ssh/vps_id_ed25519:ro \