script for https://github.com/JuliaRegistries/General/issues/16777

pkgserver_watchdog.jl

using Pkg
using Dates

### BEGIN Setup ###
const pkg_servers = [
    "https://cn-east.pkg.julialang.org",
    "https://kr.pkg.julialang.org",
    "https://us-east.pkg.julialang.org",
    "https://kr.storage.julialang.org",
    "https://us-east.storage.julialang.org",
]
const registry_dir = abspath("General")
const registry_uuid = "23338594-aafe-5451-b93e-139f81909106"
const watch_interval = 60 # seconds

const git_recordfile = joinpath("log", "general.git.csv")
const pkgserver_recordfiles = map(pkg_servers) do server
    joinpath("log", "general." * split(server, "https://")[2] * ".csv")
end
mkpath("log")

### END Setup ###

@info "Watchdog Setup" server=string(pkg_servers) registry=registry_dir logdir="log"

### BEGIN Helpers ###

# https://discourse.julialang.org/t/collecting-all-output-from-shell-commands/15592
function execute(cmd::Cmd)
  out = Pipe()
  err = Pipe()

  process = run(pipeline(ignorestatus(cmd), stdout=out, stderr=err))
  close(out.in)
  close(err.in)
  stdout = @async readchomp(out)
  stderr = @async readchomp(err)
  (
    stdout = readchomp(out),
    stderr = readchomp(err),
    code = process.exitcode
  )
end

function git_commit_datetime(repo_path)
    cmd_out = execute(`git -C $(repo_path) log -1 --format=%cd --date=iso-local`)
    if cmd_out[3] != 0
        # this is expected to always work
        error(cmd_out[2])
    end
    original_time = strip(join(split(cmd_out[1], ('-', '+'))[1:end-1], '-')) # strip UTC suffix
    DateTime(original_time, "yyyy-mm-dd HH:MM:SS")
end

function git_pull(repo_path)
    cmd_out = execute(`git -C $(repo_path) pull`)
    cmd_out[3] != 0 && @warn cmd_out[2]
    nothing
end

function git_hash(repo_path)
    readchomp(`git -C $repo_path rev-parse 'HEAD^{tree}'`)
end

function get_latest_registry_hash(registry_uuid, pkg_server; max_try=3)
    registries = nothing
    try_count = 0

    while try_count != max_try
        cmd_out = execute(`curl -L $(pkg_server)"/registries"`)
        if cmd_out[3] == 0
            registries = cmd_out[1]
            break
        end

        sleep(30)
        try_count += 1
    end
    isnothing(registries) && error("failed to fetch registries from $pkg_server")

    registry_path = first(filter(split(registries, "\n")) do line
        occursin(registry_uuid, line)
    end)
    return split(registry_path, "/")[end]
end
### END Helpers ###

function main()
    isdir(registry_dir) || run(`git clone https://github.com/JuliaRegistries/General.git $registry_dir`)

    registry_records = Dict{String, DateTime}() # this memory grows infinitely, but it's negligible even if we run this for weeks
    last_commit_hash = git_hash(registry_dir)
    last_registry_hashes = get_latest_registry_hash.(Ref(registry_uuid), pkg_servers)

    while true
        git_pull(registry_dir)
        latest_commit_hash = git_hash(registry_dir)

        if last_commit_hash != latest_commit_hash
            last_commit_hash = latest_commit_hash
            commit_datetime = git_commit_datetime(registry_dir)

            println()
            @info "$(now())\tget new commit from registry" hash=last_commit_hash commit_datetime=commit_datetime
            if !isfile(git_recordfile)
                open(git_recordfile, "w") do io
                    println(io, "registry_hash, commit_time, pull_time")
                end
            end
            open(git_recordfile, "a") do io
                println(io, latest_commit_hash, ", ", commit_datetime, ", ", now())
            end

            registry_records[git_hash(registry_dir)] = git_commit_datetime(registry_dir)
        end

        items = collect(enumerate(zip(pkg_servers, pkgserver_recordfiles)))
        Threads.@threads for (i, (pkg_server, logfile)) in items
            last_hash = last_registry_hashes[i]
            # @info server=pkg_server
            latest_registry_hash = last_hash
            try
                latest_registry_hash = get_latest_registry_hash(registry_uuid, pkg_server)
            catch err
                @warn err server=pkg_server
            end
            if last_hash != latest_registry_hash && latest_registry_hash in keys(registry_records)
                commit_time = registry_records[latest_registry_hash]
                available_time = now()
                wait_time = (available_time - commit_time).value / 60_000

                println()
                @info "$(now())\tget new registry from pkg server" hash=latest_registry_hash server=pkg_server
                if !isfile(logfile)
                    open(logfile, "w") do io
                        println(io, "registry_hash, available_time, wait_time(min)")
                    end
                end
                open(logfile, "a") do io
                    println(io, latest_registry_hash, ", ", now(), ", ", wait_time)
                end

                last_registry_hashes[i] = latest_registry_hash
            end
        end

    #     @info "hashes" github=latest_commit_hash pkg_server=latest_registry_hash
        print(".")
        sleep(watch_interval)
    end
end

main()