nas-tools/.gitea/workflows/deploy.yml

name: Deploy Dashboard (Main)
on:
  push:
    branches: [main]
    paths:
      - 'dashboard/**'
      - '.gitea/workflows/deploy.yml'
  workflow_dispatch:

concurrency:
  group: deploy-dashboard-main
  cancel-in-progress: true

jobs:
  backend-tests:
    name: Backend Tests
    runs-on: vps
    env:
      SECRET_KEY: test-secret-key-for-ci-environment-32chars-minimum

    steps:
      - name: Checkout repository
        run: |
          if [ ! -d .git ]; then
            git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" .
          fi

      - name: Setup Python
        run: |
          python3 --version
          python3 -c "import sys; assert sys.version_info[:2] == (3, 12), f'Expected Python 3.12, got {sys.version}'; print('Python 3.12 confirmed')"
          pip3 --version

      - name: Cache Python dependencies
        id: cache-python
        run: |
          CACHE_KEY="python-$(cat dashboard/backend/requirements.txt dashboard/backend/requirements-dev.txt | md5sum | cut -d' ' -f1)"
          CACHE_DIR="/tmp/pytest-cache/$CACHE_KEY"
          PIP_CACHE_DIR="/tmp/pip-cache"
          echo "CACHE_DIR=$CACHE_DIR" >> $GITHUB_ENV
          echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV
          echo "PIP_CACHE_DIR=$PIP_CACHE_DIR" >> $GITHUB_ENV
          mkdir -p "$PIP_CACHE_DIR"
          if [ -d "$CACHE_DIR" ]; then
            echo "Cache hit for $CACHE_KEY"
            echo "cache-hit=true" >> $GITHUB_OUTPUT
          else
            echo "Cache miss for $CACHE_KEY"
            echo "cache-hit=false" >> $GITHUB_OUTPUT
            mkdir -p "$CACHE_DIR"
          fi

      - name: Install dependencies
        run: |
          cd dashboard/backend
          if [ "${{ steps.cache-python.outputs.cache-hit }}" = "true" ]; then
            echo "Restoring venv from cache $CACHE_KEY..."
            if cp -a $CACHE_DIR/venv . && [ -f venv/bin/activate ]; then
              echo "Cache restored successfully"
            else
              echo "Cache corrupted, installing fresh..."
              rm -rf venv
              python3 -m venv venv
              . venv/bin/activate
              timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements.txt || \
                timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements.txt || \
                pip install -r requirements.txt
              timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements-dev.txt || \
                timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements-dev.txt || \
                pip install -r requirements-dev.txt
              timeout 600 pip install --cache-dir=$PIP_CACHE_DIR pytest-timeout pytest-cov || pip install pytest-timeout pytest-cov
            fi
          else
            echo "Installing fresh dependencies..."
            python3 -m venv venv
            . venv/bin/activate
            timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements.txt || \
              timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements.txt || \
              pip install -r requirements.txt
            timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements-dev.txt || \
              timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements-dev.txt || \
              pip install -r requirements-dev.txt
            timeout 600 pip install --cache-dir=$PIP_CACHE_DIR pytest-timeout pytest-cov || pip install pytest-timeout pytest-cov
            echo "Saving venv to cache $CACHE_KEY..."
            cp -a venv $CACHE_DIR/ || echo "Warning: cache save failed"
          fi

      - name: Run tests with coverage
        run: |
          cd dashboard/backend
          . venv/bin/activate
          pytest tests/ -v --timeout=60 \
            --cov=. --cov-report=xml --cov-report=term \
            --junit-xml=test-results.xml \
            --cov-fail-under=49

          if [ $? -ne 0 ]; then
            echo "❌ Backend tests failed!"
            exit 1
          fi
          echo "✅ Backend tests passed"

  frontend-tests:
    name: Frontend Tests
    runs-on: vps

    steps:
      - name: Checkout repository
        run: |
          if [ ! -d .git ]; then
            git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" .
          fi

      - name: Setup Node.js
        run: |
          node --version
          echo "Node $(node --version) detected"
          npm --version

      - name: Cache Node dependencies
        id: cache-node
        run: |
          CACHE_KEY="node-$(md5sum dashboard/frontend/package-lock.json | cut -d' ' -f1)"
          CACHE_DIR="/tmp/npm-cache/$CACHE_KEY"
          echo "CACHE_DIR=$CACHE_DIR" >> $GITHUB_ENV
          echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV
          if [ -d "$CACHE_DIR" ]; then
            echo "Cache hit for $CACHE_KEY"
            echo "cache-hit=true" >> $GITHUB_OUTPUT
          else
            echo "Cache miss for $CACHE_KEY"
            echo "cache-hit=false" >> $GITHUB_OUTPUT
            mkdir -p "$CACHE_DIR"
          fi

      - name: Install dependencies
        env:
          NODE_OPTIONS: "--max-old-space-size=2048"
        run: |
          cd dashboard/frontend
          npm config set registry https://registry.npmmirror.com || true
          if [ "${{ steps.cache-node.outputs.cache-hit }}" = "true" ]; then
            echo "Restoring from cache $CACHE_KEY..."
            if cp -a $CACHE_DIR/node_modules . && [ -d node_modules ]; then
              echo "Cache restored successfully"
            else
              echo "Cache corrupted, installing fresh..."
              rm -rf node_modules
              npm ci || { echo "mirror failed, trying default registry..."; npm config set registry https://registry.npmjs.org; npm ci; }
            fi
          else
            echo "Installing fresh dependencies..."
            npm ci || { echo "mirror failed, trying default registry..."; npm config set registry https://registry.npmjs.org; npm ci; }
            echo "Saving to cache $CACHE_KEY..."
            cp -a node_modules $CACHE_DIR/ || echo "Warning: cache save failed"
          fi

      - name: Run tests
        env:
          NODE_OPTIONS: "--max-old-space-size=2048"
        run: |
          cd dashboard/frontend
          npm run test:coverage -- --reporter=verbose --run --pool=forks --poolOptions.forks.maxForks=2

          if [ $? -ne 0 ]; then
            echo "❌ Frontend tests failed!"
            exit 1
          fi
          echo "✅ Frontend tests passed"

  deploy:
    name: Deploy to Production
    runs-on: nas
    if: always()
    needs: [backend-tests, frontend-tests]
    env:
      SECRET_KEY: test-secret-key-for-ci-environment-32chars-minimum
      GITEA_DB_PASSWORD: nyM3P4v1aYAsT7zfUtdguImNVHgFltCKCY/OjzWZVRE=
    container:
      network: gitea_gitea
      options: --add-host=host.docker.internal:host-gateway --dns 192.168.31.1 --dns 8.8.8.8
      volumes:
        - /var/packages/ContainerManager/target/usr/bin/docker:/usr/bin/docker
        - /volume1/docker/nas-dashboard:/nas-dashboard
    steps:
      - name: Checkout repository
        run: |
          if [ ! -d .git ]; then
            git clone --depth=1 "http://gitea:3000/${GITHUB_REPOSITORY}.git" .
          fi

      - name: Warm mirror cache for base images
        run: |
          set -u
          mirror_host="100.78.131.124:5501"

          if command -v curl >/dev/null 2>&1; then
            if curl -fsS --max-time 10 "http://${mirror_host}/v2/" >/dev/null; then
              echo "Registry mirror is healthy at ${mirror_host}"
            else
              echo "Warning: registry mirror health check failed at ${mirror_host}; continuing with normal pull path"
            fi
          else
            echo "Warning: curl is unavailable; skipping explicit mirror health check"
          fi

          prewarm_base_image() {
            image_ref="$1"
            # Skip if image already exists locally
            if docker image inspect "${image_ref}" >/dev/null 2>&1; then
              echo "Image ${image_ref} already exists locally, skipping pre-pull"
              return 0
            fi

            mirror_ref="${mirror_host}/library/${image_ref}"
            echo "Attempting mirror pre-pull: ${mirror_ref}"
            if timeout 120 docker pull "${mirror_ref}"; then
              docker tag "${mirror_ref}" "${image_ref}"
              echo "Mirror pre-pull succeeded for ${image_ref}"
            else
              echo "Warning: mirror pre-pull failed for ${image_ref}; continuing with normal pull during build"
            fi
          }

          prewarm_base_image "node:20-alpine"
          prewarm_base_image "python:3.12-slim"

      - name: Build
        run: |
          set -e
          export DOCKER_API_VERSION=1.43
          # DOCKER_BUILDKIT=0 is required for Synology ContainerManager's Docker daemon,
          # which does not support BuildKit syntax. Remove when Synology updates to a
          # Docker Engine version with full BuildKit support (currently 24.x, needs 23+).
          # --cache-to type=inline is a no-op while BuildKit is disabled; it activates
          # automatically when DOCKER_BUILDKIT=0 is removed.
          if ! DOCKER_BUILDKIT=0 docker build --cache-from nas-dashboard:latest -t nas-dashboard:latest dashboard/; then
            echo "Build failed. Checking for network issues..."
            curl -I https://registry.npmmirror.com || echo "npmmirror unreachable"
            curl -I https://pypi.tuna.tsinghua.edu.cn || echo "Tsinghua PyPI unreachable"
            exit 1
          fi

      - name: Sync runtime compose file
        run: cp dashboard/docker-compose.yml /nas-dashboard/docker-compose.yml

      - name: Deploy and verify
        run: |
          set -e
          export DOCKER_API_VERSION=1.43

          # Stop and remove old containers to force recreate with new image.
          # External networks (gitea_gitea, nas-dashboard_internal) are preserved.
          docker compose -f /nas-dashboard/docker-compose.yml down 2>/dev/null || true
          docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true
          echo "Old containers stopped and removed"

          # --pull never prevents overwriting the image we just built
          docker compose -f /nas-dashboard/docker-compose.yml up -d --pull never 2>&1 || true

          if ! docker container inspect nas-dashboard >/dev/null 2>&1; then
            echo "Compose failed, stripping networks and retrying..."
            docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true
            python3 -c "
          import os
          lines = open('/nas-dashboard/docker-compose.yml').readlines()
          out = []
          skip_net = False
          for l in lines:
              stripped = l.strip()
              if stripped == 'networks:' and not l[0].isspace():
                  skip_net = True
                  continue
              if skip_net:
                  if l[0].isspace():
                      continue
                  skip_net = False
              if stripped == 'networks:' and l[:4].strip() == '':
                  skip_net = True
                  continue
              if skip_net:
                  if l.startswith('      '):
                      continue
                  skip_net = False
              out.append(l)
          open('/nas-dashboard/prod-ci.yml','w').writelines(out)
          "
            docker compose -f /nas-dashboard/prod-ci.yml up -d --pull never 2>&1 || true
          fi

          if ! docker container inspect nas-dashboard >/dev/null 2>&1; then
            echo "Compose still failed, deploying with docker run..."
            docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true
            docker run -d --name docker-socket-proxy \
              --restart unless-stopped \
              -e CONTAINERS=1 -e IMAGES=1 -e INFO=1 -e POST=1 -e NETWORKS=1 \
              -v /var/run/docker.sock:/var/run/docker.sock:ro \
              tecnativa/docker-socket-proxy
            docker run -d --name nas-dashboard \
              --restart unless-stopped \
              -p 127.0.0.1:4000:4000 \
              --health-cmd 'python -c "import urllib.request; urllib.request.urlopen(\"http://localhost:4000/api/health\")"' \
              --health-interval 30s \
              --health-timeout 5s \
              --health-retries 3 \
              -e DOCKER_HOST=tcp://docker-socket-proxy:2375 \
              -e GITEA_URL=http://gitea:3000 \
              -e GITEA_TOKEN=${GITEA_TOKEN} \
              -e GITEA_DB_PASSWORD=${GITEA_DB_PASSWORD} \
              -e VOLUME_ROOT=/volume1 \
              -e SSH_HOST=host.docker.internal \
              -e SSH_USER=zjgump \
              -e VPS_SSH_HOST=158.101.140.85 \
              -e VPS_SSH_USER=jimmyg \
              -e CADDY_VPS_SSH_HOST=161.33.182.109 \
              -e CADDY_VPS_SSH_USER=ubuntu \
              -e MAC_SSH_HOST=192.168.31.22 \
              -e MAC_SSH_USER=jimmyg \
              -e CORS_ORIGINS=https://nas.jimmygan.com \
              -e WEBAUTHN_RP_ID=jimmygan.com \
              -e WEBAUTHN_ORIGINS=https://nas.jimmygan.com,https://nas.jimmygan.com:8443,https://auth.jimmygan.com:8443 \
              -e SECRET_KEY=${SECRET_KEY} \
              -e ADMIN_USER=jimmy \
              -e ADMIN_PASSWORD_HASH=${ADMIN_PASSWORD_HASH} \
              -e TRANSMISSION_USER=${TRANSMISSION_USER:-admin} \
              -e TRANSMISSION_PASS=${TRANSMISSION_PASS:-admin} \
              -v /volume1:/volume1 \
              -v /volume1/docker/nas-dashboard/ssh/dashboard_terminal:/app/ssh/id_ed25519:ro \
              -v /volume1/docker/nas-dashboard/ssh/vps_terminal:/app/ssh/vps_id_ed25519:ro \
              -v /volume1/docker/nas-dashboard/ssh/mac_terminal:/app/ssh/mac_id_ed25519:ro \
              -v /volume1/docker/nas-dashboard/ssh/known_hosts:/app/ssh/known_hosts:ro \
              -v /volume1/docker/chat-summarizer/data:/app/data/chat-summarizer:ro \
              -v /volume1/docker/info-engine/data:/app/data/info-engine:ro \
              --add-host=host.docker.internal:host-gateway \
              nas-dashboard:latest
          fi

          # Create the internal network if compose didn't create it
          docker network create internal 2>/dev/null || true

          # Connect docker-socket-proxy to networks
          docker network connect internal docker-socket-proxy 2>/dev/null || true
          docker network connect nas-dashboard_internal docker-socket-proxy 2>/dev/null || true

          # Connect dashboard to networks
          docker network connect internal nas-dashboard 2>/dev/null || true
          docker network connect nas-dashboard_internal nas-dashboard 2>/dev/null || true
          docker network connect gitea_gitea nas-dashboard 2>/dev/null || true

          # Verify network connections
          echo "=== Network connections ==="
          docker inspect nas-dashboard --format '{{json .NetworkSettings.Networks}}' | python3 -c 'import json,sys;nets=json.load(sys.stdin);[print(f"  Connected to {n}") for n in nets];assert "nas-dashboard_internal" in nets, "Missing nas-dashboard_internal";print("All network connections established")'

          # Wait for container to be healthy
          echo "Waiting for container to be healthy..."
          for i in {1..120}; do
            STATUS=$(docker inspect --format='{{.State.Health.Status}}' nas-dashboard 2>/dev/null || echo "starting")
            if [ "$STATUS" = "healthy" ]; then
              echo "✅ Container is healthy"
              break
            fi
            echo "Waiting... ($i/120) Status: $STATUS"
            sleep 2
          done
          STATUS=$(docker inspect --format='{{.State.Health.Status}}' nas-dashboard 2>/dev/null || echo "unknown")
          if [ "$STATUS" != "healthy" ]; then
            echo "❌ Container failed to become healthy: $STATUS"
            docker logs nas-dashboard --tail 50
            exit 1
          fi

          # Run smoke tests via docker exec
          echo "=== Running smoke tests ==="
          docker exec nas-dashboard python3 -c "import urllib.request,json; d=json.loads(urllib.request.urlopen('http://localhost:4000/api/health',timeout=5).read()); assert d['status']=='ok', f'unexpected: {d}'" && echo "✅ Health check passed" || { echo "❌ Health check failed"; docker logs nas-dashboard --tail 50; exit 1; }
          docker exec nas-dashboard python3 -c "import urllib.request; html=urllib.request.urlopen('http://localhost:4000/',timeout=5).read().decode(); assert 'NAS Dashboard' in html, f'missing NAS Dashboard'" && echo "✅ Frontend loads" || { echo "❌ Frontend failed to load"; exit 1; }
          echo ""
          echo "=== All deployment checks passed! ==="