Preferences -> supermoth scraper

command.sh

python player_saves_to_sql.py path/to/player_saves/ $(hostname) > ./preferences.tsv

# Asks input for password. Not sure the best way to do this but you ops fellows will figure it out.
mysql --local-infile=1 -h HOST -u USERNAME -p supermoth < load.sql

load.sql

LOAD DATA LOCAL INFILE 'preferences.tsv'
REPLACE INTO TABLE preferences
FIELDS TERMINATED BY '\t' ESCAPED BY ''
LINES TERMINATED BY '\n'
(ckey, machine, preference_key, value, character_index);

player_saves_to_psql.py

#!/usr/bin/env python3
"""
Converts player_saves preferences.json files into a TSV file
suitable for MariaDB LOAD DATA INFILE into a preferences table.

DDL:
  ckey VARCHAR(32)
  machine VARCHAR(16)
  preference_key VARCHAR(100)
  value JSON
  character INT NULLABLE
"""

import argparse
import json
import os
import re
import sys
from typing import IO, Any, Optional

CHARACTER_RE = re.compile(r"^character(\d+)$")

row_count = 0


def write_row(out: IO[str], ckey: str, machine: str, preference_key: str, value: Any, character: Optional[int]) -> None:
    global row_count
    ckey = ckey.replace("\t", " ")
    preference_key = preference_key.replace("\t", " ")
    json_value = json.dumps(value, ensure_ascii=False, separators=(",", ":")).replace("\t", " ")
    char_str = "-1" if character is None else str(character)
    out.write(f"{ckey}\t{machine}\t{preference_key}\t{json_value}\t{char_str}\n")
    row_count += 1


def process_preferences(ckey: str, data: dict[str, Any], machine: str, out: IO[str]) -> None:
    for key, value in data.items():
        m = CHARACTER_RE.match(key)
        if m:
            char_index = int(m.group(1))
            if not isinstance(value, dict):
                continue
            for pref_key, pref_value in value.items():
                write_row(out, ckey, machine, pref_key, pref_value, char_index)
        else:
            write_row(out, ckey, machine, key, value, None)


def main() -> None:
    global row_count
    parser = argparse.ArgumentParser(description="Convert player_saves preferences.json files to a TSV for MariaDB LOAD DATA INFILE.")
    parser.add_argument("player_saves_dir", help="Path to the player_saves directory")
    parser.add_argument("machine", help="Machine identifier (max 16 chars)")
    args = parser.parse_args()

    player_saves_dir = os.path.abspath(args.player_saves_dir)
    machine = args.machine[:16]
    error_count = 0

    for entry in sorted(os.listdir(player_saves_dir)):
        entry_path = os.path.join(player_saves_dir, entry)
        if not os.path.isdir(entry_path):
            continue

        for ckey in sorted(os.listdir(entry_path)):
            ckey_path = os.path.join(entry_path, ckey)
            if not os.path.isdir(ckey_path):
                continue

            prefs_path = os.path.join(ckey_path, "preferences.json")
            if not os.path.isfile(prefs_path):
                continue

            try:
                with open(prefs_path, "r", encoding="utf-8") as f:
                    data = json.load(f)
            except (json.JSONDecodeError, OSError) as e:
                print(f"WARN: skipping {prefs_path}: {e}", file=sys.stderr)
                error_count += 1
                continue

            process_preferences(ckey, data, machine, sys.stdout)

    print(f"Done. Wrote {row_count} rows.", file=sys.stderr)
    if error_count:
        print(f"Skipped {error_count} files due to errors.", file=sys.stderr)


if __name__ == "__main__":
    main()