From 2d6dbf9f8161534264e71dbffcbb18d7737625a3 Mon Sep 17 00:00:00 2001 From: Hanashi Date: Thu, 6 Aug 2026 01:58:41 -0400 Subject: [PATCH] pod reset recovery: new SSH port, consolidated downloads to 1T volume, retire tmpfs overflow The network volume was resized to 1TB (which reset the pod and wiped the container disk). All weights now live persistently on /workspace; the /dev/shm overflow scheme is no longer needed. Co-Authored-By: Claude Fable 5 --- ...ttings.local.json.tmp.3530818.d2f908315140 | 78 +++++++++++++++++++ scripts/local/port-forward.sh | 2 +- scripts/pod/download-weights-tmpfs.sh | 41 ---------- scripts/pod/download-weights.sh | 1 + scripts/pod/extra_model_paths.yaml | 10 +-- 5 files changed, 82 insertions(+), 50 deletions(-) create mode 100644 .claude/settings.local.json.tmp.3530818.d2f908315140 delete mode 100644 scripts/pod/download-weights-tmpfs.sh diff --git a/.claude/settings.local.json.tmp.3530818.d2f908315140 b/.claude/settings.local.json.tmp.3530818.d2f908315140 new file mode 100644 index 0000000..512a469 --- /dev/null +++ b/.claude/settings.local.json.tmp.3530818.d2f908315140 @@ -0,0 +1,78 @@ +{ + "permissions": { + "allow": [ + "Bash(curl -sS -m 5 -o /dev/null -w \"HTTP %{http_code}\\\\n\" http://localhost:8188)", + "Bash(ssh *)", + "Bash(ssh-keygen -y -P \"\" -f ~/.ssh/id_ed25519_echelon)", + "Bash(ssh-add -l)", + "Bash(env)", + "Bash(echo \"exit: $?\")", + "Read(//run/user/1000/keyring/**)", + "Read(//tmp/**)", + "Bash(SSH_AUTH_SOCK=/run/user/1000/gnupg/S.gpg-agent.ssh ssh-add -l)", + "Bash(SSH_AUTH_SOCK=/run/user/1000/keyring/.ssh ssh-add -l)", + "Bash(echo \"exit2: $?\")", + "Bash(export SSH_AUTH_SOCK=/run/user/1000/keyring/.ssh)", + "Bash(curl -s \"https://huggingface.co/api/models/Comfy-Org/MiniMax-H3/tree/main?recursive=true\")", + "Bash(python3 -c ' *)", + "WebFetch(domain:github.com)", + "WebSearch", + "WebFetch(domain:huggingface.co)", + "Bash(curl -s \"https://huggingface.co/api/models/Comfy-Org/MiniMax-H3/tree/main/diffusion_models\")", + "WebFetch(domain:docs.comfy.org)", + "WebFetch(domain:blog.comfy.org)", + "WebFetch(domain:raw.githubusercontent.com)", + "WebFetch(domain:recipes.vllm.ai)", + "Bash(chmod +x scripts/pod/*.sh scripts/local/*.sh)", + "Bash(curl -sL https://raw.githubusercontent.com/Comfy-Org/ComfyUI/master/comfy_extras/nodes_minimax_h3.py -o nodes_minimax_h3.py)", + "WebFetch(domain:docs.sglang.io)", + "Bash(scp -P 14278 scripts/pod/download-weights.sh scripts/pod/start-comfyui.sh root@216.243.220.136:/workspace/minimax-h3/)", + "Bash(scp -P 14278 scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)", + "WebFetch(domain:comfyui-wiki.com)", + "WebFetch(domain:docs.runpod.io)", + "Bash(curl -s \"https://api.github.com/repos/Comfy-Org/workflow_templates/git/trees/main?recursive=1\")", + "WebFetch(domain:kingy.ai)", + "Bash(bash /home/bt/dev/storyteller/cloud-worker/scripts/local/port-forward.sh)", + "Bash(curl -s -o /dev/null -w \"%{http_code}\" http://127.0.0.1:8188/)", + "Bash(python3 -)", + "Bash(curl -s http://127.0.0.1:8188/object_info)", + "Bash(curl -s \"https://api.github.com/search/issues?q=repo:Comfy-Org/ComfyUI+minimax+h3&per_page=30\")", + "Bash(curl -s http://127.0.0.1:8188/object_info/UNETLoader)", + "Bash(curl -s http://127.0.0.1:8188/object_info/VAELoader)", + "Bash(curl -s http://127.0.0.1:8188/object_info/CLIPLoader)", + "Bash(curl -s http://127.0.0.1:8188/object_info/CreateVideo)", + "Bash(python3 -m json.tool)", + "Bash(curl -s http://127.0.0.1:8188/object_info/SaveVideo)", + "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(json.dumps\\(d['SaveVideo']['input'],indent=1\\)[:800]\\)\")", + "Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/make-test-images.py root@216.243.220.136:/workspace/minimax-h3/)", + "Bash(timeout 550 python3 bench/minimax_bench.py --task t2v --width 608 --height 352 --seconds 5 --steps 20 --label smoke_t2v 2>&1)", + "Bash(timeout 550 python3 bench/minimax_bench.py --task i2v --width 608 --height 352 --seconds 5 --label smoke_i2v --warm)", + "Bash(timeout 550 python3 bench/minimax_bench.py --task ref2v --width 608 --height 352 --seconds 5 --label smoke_ref2v)", + "Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/download-weights-tmpfs.sh root@216.243.220.136:/workspace/minimax-h3/)", + "Bash(scp -q -O -P 14278 /home/bt/dev/storyteller/cloud-worker/scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)", + "Bash(ffmpeg -v error -i /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/smoke_t2v.mp4 -vf 'select=eq\\(n\\\\,60\\)' -frames:v 1 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/t2v_frame.png -y)", + "Bash(ffmpeg -v error -i /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/smoke_ref2v.mp4 -vf 'select=eq\\(n\\\\,60\\)' -frames:v 1 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/scratchpad/ref2v_frame.png -y)", + "Bash(python3 -c \"import ast; ast.parse\\(open\\('bench/minimax_bench.py'\\).read\\(\\)\\); print\\('syntax ok'\\)\")", + "Bash(python3 *)", + "Bash(git add *)", + "Bash(git commit -q -m 'MiniMax H3 on RunPod: install/download scripts, ComfyUI wiring, benchmark harness, research docs *)", + "Bash(tail -6 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/tasks/betgtau6s.output)", + "Bash(git commit -q -m 'WEIGHTS.md: final download status — all 9 files present, none in progress *)", + "Bash(git commit -q -m 'docs: pad markdown table columns for raw-text readability *)", + "Bash(tail -4 /tmp/claude-1000/-home-bt-dev-storyteller-cloud-worker/309f1cfc-b7d8-47ef-a9b1-357cee94033d/tasks/betgtau6s.output)", + "Bash(git commit -q -m 'bench: multi-reference ref2v support \\(1-8 ref images, match/max sizing\\) *)", + "Bash(curl -s http://127.0.0.1:8188/queue)", + "Bash(curl -s -o /dev/null -w \"local tunnel: %{http_code}\\\\n\" --max-time 5 http://127.0.0.1:8188/)", + "Bash(ls -la ~/.ssh/)", + "Bash(git rm *)", + "Bash(scp -q -O -P 17561 scripts/pod/download-weights.sh scripts/pod/start-comfyui.sh scripts/pod/make-test-images.py root@216.243.220.136:/workspace/minimax-h3/)", + "Bash(scp -q -O -P 17561 scripts/pod/extra_model_paths.yaml root@216.243.220.136:/ComfyUI/extra_model_paths.yaml)", + "Bash(curl -s -o /dev/null -w \"tunnel to ComfyUI: %{http_code}\\\\n\" http://127.0.0.1:8188/)", + "Bash(curl -s -o /dev/null -w \"tunnel: %{http_code}\\\\n\" --max-time 8 http://127.0.0.1:8188/)", + "Bash(curl -s --max-time 8 http://127.0.0.1:8188/object_info/UNETLoader)", + "Bash(pkill -f \"ssh -N -L 8188\")", + "Bash(kill 3568392)", + "Bash(git commit -q -m 'pod reset recovery: new SSH port, consolidated downloads to 1T volume, retire tmpfs overflow *)" + ] + } +} diff --git a/scripts/local/port-forward.sh b/scripts/local/port-forward.sh index 8be71b1..31125b6 100755 --- a/scripts/local/port-forward.sh +++ b/scripts/local/port-forward.sh @@ -3,7 +3,7 @@ # Auto-reconnects if the tunnel drops. Ctrl-C to stop. # Requires the echelon key loaded in your ssh agent. POD_HOST=216.243.220.136 -POD_PORT=14278 +POD_PORT=17561 while true; do ssh -N -L 8188:127.0.0.1:8188 \ -o ServerAliveInterval=30 -o ServerAliveCountMax=3 \ diff --git a/scripts/pod/download-weights-tmpfs.sh b/scripts/pod/download-weights-tmpfs.sh deleted file mode 100644 index ba5d7a4..0000000 --- a/scripts/pod/download-weights-tmpfs.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Overflow download for the weight files that don't fit under the /workspace -# network-volume quota (~100GB on the current pod). Puts them on tmpfs -# (/dev/shm, 176GB on the B200 pod) — EPHEMERAL: lost on pod restart, so this -# is for benchmarking sessions. Re-run after any pod restart. -# -# tmux new -s downloads-shm -d 'bash /workspace/minimax-h3/download-weights-tmpfs.sh' -# -# ComfyUI sees this path via the minimax_shm section of extra_model_paths.yaml. -set -uo pipefail - -REPO="Comfy-Org/MiniMax-H3" -DEST="/dev/shm/minimax-h3-models" -LOG="/workspace/minimax-h3/download.log" - -mkdir -p "$DEST/diffusion_models" -export HF_HUB_ENABLE_HF_TRANSFER=1 - -log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; } - -dl() { - local file="$1" - if [ -f "$DEST/$file" ]; then - log "SKIP $file (already present in shm)" - return 0 - fi - log "START $file -> tmpfs" - if hf download "$REPO" "$file" --local-dir "$DEST" >>"$LOG" 2>&1; then - log "DONE $file ($(du -h "$DEST/$file" | cut -f1))" - else - log "FAILED $file" - return 1 - fi -} - -log "=== tmpfs overflow download starting ===" -dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors -dl diffusion_models/minimax_h3_ref2va_bf16.safetensors -dl diffusion_models/minimax_h3_fl2va_bf16.safetensors -rm -rf "$DEST/.cache" -log "=== tmpfs overflow downloads complete ===" diff --git a/scripts/pod/download-weights.sh b/scripts/pod/download-weights.sh index cc8d81c..9624948 100755 --- a/scripts/pod/download-weights.sh +++ b/scripts/pod/download-weights.sh @@ -55,4 +55,5 @@ dl diffusion_models/minimax_h3_fl2va_int8_convrot.safetensors dl diffusion_models/minimax_h3_ref2va_bf16.safetensors dl diffusion_models/minimax_h3_fl2va_bf16.safetensors +rm -rf "$DEST/.cache" log "=== All downloads complete ===" diff --git a/scripts/pod/extra_model_paths.yaml b/scripts/pod/extra_model_paths.yaml index e6f6bb7..7109314 100644 --- a/scripts/pod/extra_model_paths.yaml +++ b/scripts/pod/extra_model_paths.yaml @@ -1,13 +1,7 @@ -# ComfyUI extra model paths — points at the RunPod network volume where all -# MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml +# ComfyUI extra model paths — points at the RunPod network volume (1 TB) where +# all MiniMax H3 weights live. Install to /ComfyUI/extra_model_paths.yaml minimax_workspace: base_path: /workspace/minimax-h3/models diffusion_models: diffusion_models text_encoders: text_encoders vae: vae - -# tmpfs overflow for files that exceed the network-volume quota (~100GB). -# EPHEMERAL — refill with download-weights-tmpfs.sh after a pod restart. -minimax_shm: - base_path: /dev/shm/minimax-h3-models - diffusion_models: diffusion_models