name: Deploy Dashboard (Main) on: push: branches: [main] paths: - 'dashboard/**' - '.gitea/workflows/deploy.yml' workflow_dispatch: concurrency: group: deploy-dashboard-main cancel-in-progress: true jobs: backend-tests: name: Backend Tests runs-on: vps env: SECRET_KEY: test-secret-key-for-ci-environment-32chars-minimum steps: - name: Checkout repository run: | if [ ! -d .git ]; then git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" . fi - name: Setup Python run: | python3 --version python3 -c "import sys; assert sys.version_info[:2] == (3, 12), f'Expected Python 3.12, got {sys.version}'; print('Python 3.12 confirmed')" pip3 --version - name: Cache Python dependencies id: cache-python run: | CACHE_KEY="python-$(cat dashboard/backend/requirements.txt dashboard/backend/requirements-dev.txt | md5sum | cut -d' ' -f1)" CACHE_DIR="/tmp/pytest-cache/$CACHE_KEY" PIP_CACHE_DIR="/tmp/pip-cache" echo "CACHE_DIR=$CACHE_DIR" >> $GITHUB_ENV echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV echo "PIP_CACHE_DIR=$PIP_CACHE_DIR" >> $GITHUB_ENV mkdir -p "$PIP_CACHE_DIR" if [ -d "$CACHE_DIR" ]; then echo "Cache hit for $CACHE_KEY" echo "cache-hit=true" >> $GITHUB_OUTPUT else echo "Cache miss for $CACHE_KEY" echo "cache-hit=false" >> $GITHUB_OUTPUT mkdir -p "$CACHE_DIR" fi - name: Install dependencies run: | cd dashboard/backend if [ "${{ steps.cache-python.outputs.cache-hit }}" = "true" ]; then echo "Restoring venv from cache $CACHE_KEY..." if cp -a $CACHE_DIR/venv . && [ -f venv/bin/activate ] && venv/bin/python -m pytest --version >/dev/null 2>&1; then echo "Cache restored successfully" else echo "Cache corrupted, installing fresh..." rm -rf venv python3 -m venv venv . venv/bin/activate timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements.txt || \ timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements.txt || \ pip install -r requirements.txt timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements-dev.txt || \ timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements-dev.txt || \ pip install -r requirements-dev.txt timeout 600 pip install --cache-dir=$PIP_CACHE_DIR pytest-timeout pytest-cov || pip install pytest-timeout pytest-cov fi else echo "Installing fresh dependencies..." python3 -m venv venv . venv/bin/activate timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements.txt || \ timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements.txt || \ pip install -r requirements.txt timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --retries 3 --timeout 30 -r requirements-dev.txt || \ timeout 600 pip install --cache-dir=$PIP_CACHE_DIR --index-url https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host pypi.tuna.tsinghua.edu.cn -r requirements-dev.txt || \ pip install -r requirements-dev.txt timeout 600 pip install --cache-dir=$PIP_CACHE_DIR pytest-timeout pytest-cov || pip install pytest-timeout pytest-cov echo "Saving venv to cache $CACHE_KEY..." cp -a venv $CACHE_DIR/ || echo "Warning: cache save failed" fi - name: Run tests with coverage run: | cd dashboard/backend . venv/bin/activate python -m pytest tests/ -v --timeout=60 \ --cov=. --cov-report=xml --cov-report=term \ --junit-xml=test-results.xml \ --cov-fail-under=49 if [ $? -ne 0 ]; then echo "❌ Backend tests failed!" exit 1 fi echo "✅ Backend tests passed" frontend-tests: name: Frontend Tests runs-on: vps steps: - name: Checkout repository run: | if [ ! -d .git ]; then git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" . fi - name: Setup Node.js run: | node --version echo "Node $(node --version) detected" npm --version - name: Cache Node dependencies id: cache-node run: | CACHE_KEY="node-$(md5sum dashboard/frontend/package-lock.json | cut -d' ' -f1)" CACHE_DIR="/tmp/npm-cache/$CACHE_KEY" echo "CACHE_DIR=$CACHE_DIR" >> $GITHUB_ENV echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV if [ -d "$CACHE_DIR" ]; then echo "Cache hit for $CACHE_KEY" echo "cache-hit=true" >> $GITHUB_OUTPUT else echo "Cache miss for $CACHE_KEY" echo "cache-hit=false" >> $GITHUB_OUTPUT mkdir -p "$CACHE_DIR" fi - name: Install dependencies env: NODE_OPTIONS: "--max-old-space-size=2048" run: | cd dashboard/frontend npm config set registry https://registry.npmmirror.com || true if [ "${{ steps.cache-node.outputs.cache-hit }}" = "true" ]; then echo "Restoring from cache $CACHE_KEY..." if cp -a $CACHE_DIR/node_modules . && [ -d node_modules ]; then echo "Cache restored successfully" else echo "Cache corrupted, installing fresh..." rm -rf node_modules npm ci || { echo "mirror failed, trying default registry..."; npm config set registry https://registry.npmjs.org; npm ci; } fi else echo "Installing fresh dependencies..." npm ci || { echo "mirror failed, trying default registry..."; npm config set registry https://registry.npmjs.org; npm ci; } echo "Saving to cache $CACHE_KEY..." cp -a node_modules $CACHE_DIR/ || echo "Warning: cache save failed" fi - name: Run tests env: NODE_OPTIONS: "--max-old-space-size=2048" run: | cd dashboard/frontend npm run test:coverage -- --reporter=verbose --run --pool=forks --poolOptions.forks.maxForks=2 if [ $? -ne 0 ]; then echo "❌ Frontend tests failed!" exit 1 fi echo "✅ Frontend tests passed" build-image-main: name: Build Main Image runs-on: vps needs: [backend-tests] steps: - name: Checkout repository run: | if [ ! -d .git ]; then git clone --depth=1 "http://100.78.131.124:3300/${GITHUB_REPOSITORY}.git" . fi - name: Build main Docker image run: | set -e echo "Building on VPS (SSD)..." CACHE_ARG="" if podman image exists nas-dashboard:latest 2>/dev/null; then CACHE_ARG="--cache-from nas-dashboard:latest" fi podman build $CACHE_ARG -t nas-dashboard:latest dashboard/ - name: Transfer image to NAS run: | set -e echo "Streaming main image to NAS..." podman save nas-dashboard:latest | ssh nasts "/volume1/@appstore/ContainerManager/usr/bin/docker load 2>&1" echo "Main image transferred successfully" deploy: name: Deploy to Production runs-on: nas if: always() needs: [build-image-main, frontend-tests] env: SECRET_KEY: test-secret-key-for-ci-environment-32chars-minimum GITEA_DB_PASSWORD: nyM3P4v1aYAsT7zfUtdguImNVHgFltCKCY/OjzWZVRE= container: network: gitea_gitea options: --add-host=host.docker.internal:host-gateway --dns 192.168.31.1 --dns 8.8.8.8 volumes: - /var/packages/ContainerManager/target/usr/bin/docker:/usr/bin/docker - /volume1/docker/nas-dashboard:/nas-dashboard steps: - name: Checkout repository run: | if [ ! -d .git ]; then git clone --depth=1 "http://gitea:3000/${GITHUB_REPOSITORY}.git" . fi - name: Sync runtime compose file run: | mkdir -p /nas-dashboard cp dashboard/docker-compose.yml /nas-dashboard/docker-compose.yml - name: Deploy and verify env: GITEA_TOKEN: ${{ secrets.CI_TOKEN }} GITEA_DB_PASSWORD: ${{ secrets.CI_DB_PASSWORD }} SECRET_KEY: ${{ secrets.SECRET_KEY }} ADMIN_PASSWORD_HASH: ${{ secrets.ADMIN_PASSWORD_HASH }} run: | set -e export DOCKER_API_VERSION=1.43 # Stop and remove old containers to force recreate with new image. docker compose -f /nas-dashboard/docker-compose.yml down 2>/dev/null || true docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true echo "Old containers stopped and removed" # --pull never prevents overwriting the image we just built docker compose -f /nas-dashboard/docker-compose.yml up -d --pull never 2>&1 || true if ! docker container inspect nas-dashboard >/dev/null 2>&1; then echo "Compose failed, stripping networks and retrying..." docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true # ... (python script for stripping networks) ... docker compose -f /nas-dashboard/prod-ci.yml up -d --pull never 2>&1 || true fi if ! docker container inspect nas-dashboard >/dev/null 2>&1; then echo "Compose still failed, deploying with docker run..." docker rm -f docker-socket-proxy nas-dashboard 2>/dev/null || true docker run -d --name docker-socket-proxy \ --restart unless-stopped \ -e CONTAINERS=1 -e IMAGES=1 -e INFO=1 -e POST=1 -e NETWORKS=1 \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ tecnativa/docker-socket-proxy docker run -d --name nas-dashboard \ --restart unless-stopped \ -p 127.0.0.1:4000:4000 \ --health-cmd 'python -c "import urllib.request; urllib.request.urlopen(\"http://localhost:4000/api/health\")"' \ --health-interval 30s \ --health-timeout 5s \ --health-retries 3 \ -e DOCKER_HOST=tcp://docker-socket-proxy:2375 \ -e GITEA_URL=http://gitea:3000 \ -e GITEA_TOKEN=$GITEA_TOKEN \ -e GITEA_DB_PASSWORD=$GITEA_DB_PASSWORD \ -e VOLUME_ROOT=/volume1 \ -e SSH_HOST=host.docker.internal \ -e SSH_USER=zjgump \ -e VPS_SSH_HOST=158.101.140.85 \ -e VPS_SSH_USER=jimmyg \ -e CADDY_VPS_SSH_HOST=161.33.182.109 \ -e CADDY_VPS_SSH_USER=ubuntu \ -e MAC_SSH_HOST=192.168.31.22 \ -e MAC_SSH_USER=jimmyg \ -e CORS_ORIGINS=https://nas.jimmygan.com \ -e WEBAUTHN_RP_ID=jimmygan.com \ -e WEBAUTHN_ORIGINS=https://nas.jimmygan.com,https://nas.jimmygan.com:8443,https://auth.jimmygan.com:8443 \ -e SECRET_KEY=$SECRET_KEY \ -e ADMIN_USER=jimmy \ -e ADMIN_PASSWORD_HASH=$ADMIN_PASSWORD_HASH \ -v /volume1:/volume1 \ -v /volume1/docker/nas-dashboard/ssh/dashboard_terminal:/app/ssh/id_ed25519:ro \ -v /volume1/docker/nas-dashboard/ssh/vps_terminal:/app/ssh/vps_id_ed25519:ro \ -v /volume1/docker/nas-dashboard/ssh/mac_terminal:/app/ssh/mac_id_ed25519:ro \ -v /volume1/docker/nas-dashboard/ssh/known_hosts:/app/ssh/known_hosts:ro \ -v /volume1/docker/chat-summarizer/data:/app/data/chat-summarizer:ro \ -v /volume1/docker/info-engine/data:/app/data/info-engine:ro \ --add-host=host.docker.internal:host-gateway \ nas-dashboard:latest fi # Create the internal network if compose didn't create it docker network create internal 2>/dev/null || true # Connect docker-socket-proxy to networks docker network connect internal docker-socket-proxy 2>/dev/null || true docker network connect nas-dashboard_internal docker-socket-proxy 2>/dev/null || true # Connect dashboard to networks docker network connect internal nas-dashboard 2>/dev/null || true docker network connect nas-dashboard_internal nas-dashboard 2>/dev/null || true docker network connect gitea_gitea nas-dashboard 2>/dev/null || true # Verify network connections echo "=== Network connections ===" docker inspect nas-dashboard --format '{{json .NetworkSettings.Networks}}' | python3 -c 'import json,sys;nets=json.load(sys.stdin);[print(f" Connected to {n}") for n in nets];assert "nas-dashboard_internal" in nets, "Missing nas-dashboard_internal";print("All network connections established")' # Wait for container to be healthy echo "Waiting for container to be healthy..." for i in {1..120}; do STATUS=$(docker inspect --format='{{.State.Health.Status}}' nas-dashboard 2>/dev/null || echo "starting") if [ "$STATUS" = "healthy" ]; then echo "✅ Container is healthy" break fi echo "Waiting... ($i/120) Status: $STATUS" sleep 2 done STATUS=$(docker inspect --format='{{.State.Health.Status}}' nas-dashboard 2>/dev/null || echo "unknown") if [ "$STATUS" != "healthy" ]; then echo "❌ Container failed to become healthy: $STATUS" docker logs nas-dashboard --tail 50 exit 1 fi # Run smoke tests via docker exec echo "=== Running smoke tests ===" docker exec nas-dashboard python3 -c "import urllib.request,json; d=json.loads(urllib.request.urlopen('http://localhost:4000/api/health',timeout=5).read()); assert d['status']=='ok', f'unexpected: {d}'" && echo "✅ Health check passed" || { echo "❌ Health check failed"; docker logs nas-dashboard --tail 50; exit 1; } docker exec nas-dashboard python3 -c "import urllib.request; html=urllib.request.urlopen('http://localhost:4000/',timeout=5).read().decode(); assert 'NAS Dashboard' in html, f'missing NAS Dashboard'" && echo "✅ Frontend loads" || { echo "❌ Frontend failed to load"; exit 1; } echo "" echo "=== All deployment checks passed! ==="