Nightscout to Nightscout migration script

nightscout_migrate.sh

#!/usr/bin/env bash
set -euo pipefail

############################################
# Nightscout -> Nightscout full migrator
# - Paginated per endpoint
# - Stateful (resume-safe)
# - URL-encoded find[date][$gte] (curl-safe)
# - Endpoint-specific batch sizes to avoid 413
############################################

# -------- CONFIG --------
SRC_BASE="https://source_server.com"
SRC_API_SECRET_PLAIN="CHANGE_ME_OLD_SECRET"

DEST_BASE="https://destination_server.com"
DEST_API_SECRET_PLAIN="CHANGE_ME_NEW_SECRET"

STATE_DIR="./ns-sync-state"   # per-endpoint since_ms lives here
TMP_DIR="./ns-sync-tmp"

# Add/remove endpoints as needed
ENDPOINTS=("entries" "treatments" "devicestatus" "profile" "food" "activity")

# Set to 0 for full-history migration.
# Leave as "" to use existing state (resume) or default fallback.
START_SINCE_MS="0"

# Safety to avoid infinite loops if max date doesn't advance
MAX_STUCK_LOOPS=10

# -------- HELPERS --------
need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1"; exit 1; }; }

sha1_hex() { printf "%s" "$1" | openssl sha1 | awk '{print $2}'; }

api_secret_header() {
  local plain="$1"
  echo "api-secret: $(sha1_hex "$plain")"
}

# Batch sizes (tune if you like)
batch_for_endpoint() {
  case "$1" in
    entries) echo 2500 ;;       # keep high for speed
    treatments) echo 1000 ;;
    devicestatus) echo 500 ;;
    profile) echo 25 ;;         # keep small to avoid 413
    food) echo 250 ;;
    activity) echo 250 ;;
    *) echo 500 ;;
  esac
}

load_since() {
  local ep="$1"
  local f="${STATE_DIR}/${ep}.since_ms"
  if [[ -f "$f" ]]; then
    cat "$f"
  else
    # first ever run for this endpoint
    if [[ -n "${START_SINCE_MS:-}" ]]; then
      echo "$START_SINCE_MS"
    else
      # default: last 30 days
      python3 - <<'PY'
import time
print(int((time.time() - 30*24*3600) * 1000))
PY
    fi
  fi
}

save_since() {
  local ep="$1"
  local ms="$2"
  echo "$ms" > "${STATE_DIR}/${ep}.since_ms"
}

# URL-encoded: find[date][$gte]  -> find%5Bdate%5D%5B%24gte%5D
src_url_since() {
  local ep="$1" since="$2" count="$3"
  echo "${SRC_BASE}/api/v1/${ep}.json?count=${count}&find%5Bdate%5D%5B%24gte%5D=${since}"
}

fetch_batch() {
  local ep="$1" since="$2" count="$3" out="$4"
  curl -fsS \
    -H "$(api_secret_header "$SRC_API_SECRET_PLAIN")" \
    "$(src_url_since "$ep" "$since" "$count")" > "$out"
}

json_len() {
  python3 - "$1" <<'PY'
import json, sys
with open(sys.argv[1], "r", encoding="utf-8") as f:
    data = json.load(f)
print(len(data) if isinstance(data, list) else 0)
PY
}

max_date() {
  python3 - "$1" <<'PY'
import json, sys
with open(sys.argv[1], "r", encoding="utf-8") as f:
    data = json.load(f)
mx = None
if isinstance(data, list):
  for item in data:
    d = item.get("date")
    if isinstance(d, (int, float)):
      d = int(d)
      mx = d if mx is None else max(mx, d)
print(mx if mx is not None else "")
PY
}

push_batch() {
  local ep="$1" file="$2"
  curl -fsS -X POST \
    -H "$(api_secret_header "$DEST_API_SECRET_PLAIN")" \
    -H "Content-Type: application/json" \
    --data-binary @"$file" \
    "${DEST_BASE}/api/v1/${ep}.json" >/dev/null
}

# -------- MAIN --------
need curl
need openssl
need python3

mkdir -p "$STATE_DIR" "$TMP_DIR"

echo "Nightscout migration"
echo "  SRC : $SRC_BASE"
echo "  DEST: $DEST_BASE"
echo "  Endpoints: ${ENDPOINTS[*]}"
echo "  State: $STATE_DIR"
echo ""

for ep in "${ENDPOINTS[@]}"; do
  echo "============================"
  echo "Endpoint: $ep"
  echo "============================"

  since="$(load_since "$ep")"
  count="$(batch_for_endpoint "$ep")"
  echo "Starting since_ms: $since"
  echo "Batch size: $count"

  stuck=0

  while true; do
    out="${TMP_DIR}/${ep}.json"
    err="${TMP_DIR}/${ep}.err"

    if ! fetch_batch "$ep" "$since" "$count" "$out" 2>"$err"; then
      # If endpoint not available on source, skip
      if grep -Eqi "404|410" "$err"; then
        echo "Source endpoint not available (404/410). Skipping: $ep"
        break
      fi
      echo "Fetch failed for $ep:"
      sed -e 's/^/  /' "$err" | tail -n 30
      exit 1
    fi

    n="$(json_len "$out")"
    if [[ "$n" -eq 0 ]]; then
      echo "No more data for $ep."
      break
    fi

    echo "Pushing $n records..."
    if ! push_batch "$ep" "$out" 2>"$err"; then
      # Common: 413 payload too large
      if grep -q "413" "$err"; then
        echo "Got 413 from destination for $ep. Reduce batch size in batch_for_endpoint()."
      fi
      echo "Push failed for $ep:"
      sed -e 's/^/  /' "$err" | tail -n 30
      exit 1
    fi

    mx="$(max_date "$out" || true)"
    if [[ -z "${mx:-}" ]]; then
      # Some endpoints may not include date consistently; avoid looping forever
      echo "Warning: no max date found in batch for $ep. Stopping this endpoint to avoid looping."
      break
    fi

    next=$((mx + 1))

    if [[ "$next" -le "$since" ]]; then
      stuck=$((stuck + 1))
      if [[ "$stuck" -ge "$MAX_STUCK_LOOPS" ]]; then
        echo "Stuck advancing since_ms for $ep (since=$since, next=$next). Stopping to avoid infinite loop."
        break
      fi
    else
      stuck=0
    fi

    since="$next"
    save_since "$ep" "$since"
    echo "Advanced since_ms -> $since"
  done

  echo ""
done

echo "Migration complete."
echo "State saved in: $STATE_DIR"