check the artifact usage of BinaryBuilderBase

check_bb_size.py

#! /usr/bin/env python

import os
import requests
import toml
from tqdm import tqdm


# BinaryBuilderBase:
# UUID: 7f725544-6523-48cd-82d1-3fa08ff4056e
# Versions: https://raw.githubusercontent.com/JuliaRegistries/General/master/B/BinaryBuilderBase/Versions.toml
def get_versions():
    url = "https://raw.githubusercontent.com/JuliaRegistries/General/master/B/BinaryBuilderBase/Versions.toml"
    r = requests.get(url)
    versions = toml.loads(r.text)
    return versions


def get_artifacts(uuid, version_sha):
    url = f"https://mirrors.bfsu.edu.cn/julia/package/{uuid}/{version_sha}"
    r = requests.get(url)
    if r.status_code != 200:
        raise RuntimeError(f"failed to download {url}")
    os.path.exists('out') or os.mkdir('out')
    with open(f"out/BinaryBuilderBase-{version_sha}.tar.gz", 'wb') as f:
        f.write(r.content)
    try:
        os.system(
            f"tar -xzf out/BinaryBuilderBase-{version_sha}.tar.gz -C out")
        info = toml.load(f"out/Artifacts.toml")
        return [x[0]["git-tree-sha1"] for x in info.values()]
    finally:
        os.system(f"rm -rf out/*")


uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
versions = get_versions()
artifacts = [
    get_artifacts(uuid, info["git-tree-sha1"]) for info in versions.values()
]

total_size = {}
failed = []
for xs in artifacts:
    for x in tqdm(xs):
        if x in total_size:
            continue
        url = f"https://mirrors.bfsu.edu.cn/julia/artifact/{x}"
        r = requests.head(url)
        if r.status_code == 200:
            total_size[x] = int(r.headers['content-length'])
        else:
            failed.append(x)

sz = sum(total_size.values())/1024/1024/1024 # GB
print("total size used by BinaryBuilderBase:", sz) # 312.3008377617225
print("failed curl count:", len(failed)) # 112