#!/usr/bin/env bash
# v8: THE QUALITY RUN. Full 1494-image raw dual-fisheye at native 2880, our COLMAP
# 4.1 sparse (which beats the Denis reference), operator masks with a 6% radial trim,
# NO pinhole reframe, NO subsample. 30k iterations, MRNF (LFS default strategy).
set -uo pipefail
BASE="https://gitea.cyango.com/api/packages/gustavo.domingues/generic/lfs-build/v1"
LOG=/var/log/lfsquality.log
exec > >(tee -a "$LOG") 2>&1
mark() { printf '%s\n' "$2" | curl -s -X PUT -H "Authorization: token ${GITEA_TOKEN}" \
  -H "Content-Type: application/octet-stream" --data-binary @- "$BASE/$1" >/dev/null; echo "[mark] $1"; }
fail() { tail -c 8000 "$LOG" | curl -s -X PUT -H "Authorization: token ${GITEA_TOKEN}" \
  -H "Content-Type: application/octet-stream" --data-binary @- "$BASE/V8FAILED-$1.txt" >/dev/null
  echo "[FAILED] $1"; sleep 1800; exit 1; }

heartbeat() { while true; do
  { echo "== $(date -u) elapsed=${SECONDS}s"; echo "== load:$(cat /proc/loadavg)";
    echo "== gpu:$(nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total --format=csv,noheader 2>&1 | head -1)";
    echo "== log tail:"; tail -c 2500 "$LOG"; } > /tmp/live.txt 2>&1
  curl -s -X DELETE -H "Authorization: token ${GITEA_TOKEN}" "$BASE/LIVE.log" >/dev/null 2>&1
  curl -s -X PUT -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/octet-stream" \
    --data-binary @/tmp/live.txt "$BASE/LIVE.log" >/dev/null 2>&1
  sleep 60; done; }
heartbeat & HB=$!; trap 'kill $HB 2>/dev/null' EXIT

BIN=/opt/lfs/build/LichtFeld-Studio
[ -x "$BIN" ] || fail no-binary

if [ ! -f /data/full/.ready ]; then
  rm -rf /data/full; mkdir -p /data/full
  curl -sL -H "Authorization: token ${GITEA_TOKEN}" \
    "https://gitea.cyango.com/api/packages/gustavo.domingues/generic/gsplat-frames/lfs-full/lfs_dfrig2880_c41.zip" \
    -o /tmp/full.zip || fail fetch
  unzip -q -o /tmp/full.zip -d /data/full || fail unzip
  rm -f /tmp/full.zip; touch /data/full/.ready
fi
mark "v8-00-data.txt" "images=$(find /data/full/images -name '*.jpg' | wc -l) masks=$(find /data/full/masks -name '*.png' | wc -l) sparse=$(ls /data/full/sparse/0 | tr '\n' ' ') df=$(df -h / | tail -1)"

# --- archive runtime bundle BEFORE training, so the build survives pod loss ---
if ! curl -sf -H "Authorization: token ${GITEA_TOKEN}" -o /dev/null "$BASE/lfs_runtime.tgz"; then
  tar -C /opt/lfs/build -czf /tmp/rt.tgz \
    --exclude='vcpkg_installed' --exclude='CMakeFiles' --exclude='*.o' \
    --exclude='_deps' --exclude='.cmake' --exclude='Testing' . 2>/dev/null
  SZ=$(stat -c%s /tmp/rt.tgz 2>/dev/null || echo 0)
  if [ "$SZ" -gt 1000000 ] && [ "$SZ" -lt 1900000000 ]; then
    curl -s -X PUT -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/octet-stream" \
      --data-binary @/tmp/rt.tgz "$BASE/lfs_runtime.tgz" >/dev/null
    mark "v8-00b-archived.txt" "runtime bundle $((SZ/1000000))MB uploaded"
  else
    mark "v8-00b-archived.txt" "SKIPPED size=$((SZ/1000000))MB"
  fi
  rm -f /tmp/rt.tgz
fi

mkdir -p /out8
S=$SECONDS
# --mask-mode ignore = pure loss exclusion, i.e. parity with our fixed gsplat masked
# loss (white=keep, no inversion needed). --max-width 0 disables the 3840 cap so our
# native 2880 pixels are never downscaled.
"$BIN" --headless --train \
  -d /data/full -o /out8 \
  -i 30000 --strategy mrnf \
  --mask-mode ignore --max-width 0 2>&1 | tail -80
RC=${PIPESTATUS[0]}
mark "v8-01-train.txt" "rc=$RC secs=$((SECONDS-S)) out=$(ls /out8 2>/dev/null | tr '\n' ' ')"
[ "$RC" = "0" ] || fail train

for P in /out8/*.ply; do
  NAME=$(basename "$P")
  curl -s -X PUT -H "Authorization: token ${GITEA_TOKEN}" -H "Content-Type: application/octet-stream" \
    --data-binary @"$P" "$BASE/quality_$NAME" >/dev/null && echo "uploaded quality_$NAME"
done
mark "v8-99-DONE.txt" "train=$((SECONDS-S))s plys=$(ls /out8/*.ply 2>/dev/null | wc -l) sizes=$(du -h /out8/*.ply 2>/dev/null | tr '\n' ' ')"
# Idle until told to reload (PUT GO-NEXT.txt to trigger); self-clears the flag.
# RunPod has no exec-in-pod API and this pod has no volume, so a container reload
# is the only safe way to hand it a new script.
for _ in $(seq 1 1440); do
  if curl -sf -H "Authorization: token ${GITEA_TOKEN}" -o /dev/null "$BASE/GO-NEXT.txt"; then
    curl -s -X DELETE -H "Authorization: token ${GITEA_TOKEN}" "$BASE/GO-NEXT.txt" >/dev/null 2>&1
    echo "GO-NEXT received; exiting so the container reloads"; exit 0
  fi
  sleep 10
done