Simple script to maintain a local copy of your GitHub repos

gitmirror.py

#!/usr/bin/env python3

import requests
from requests.auth import HTTPBasicAuth
import click
from tqdm import tqdm
from pathlib import Path
import os.path
import subprocess
from glob import glob

USERNAME = <GH USERNAME>
TOKEN = <TOKEN HERE>
MIRROR_ROOT = Path(<DESTINATION PATH>)


def get_my_repos():
    page = 0
    while True:
        r = requests.get(
            "https://api.github.com/user/repos",
            headers={
                "Accept": "application/vnd.github.v3+json",
            },
            params={"type": "all", "per_page": 100, "page": page},
            auth=HTTPBasicAuth(USERNAME, TOKEN),
        )
        if not r.ok:
            r.raise_for_status()
        if len(r.json()) == 0:
            break
        for repo in r.json():
            yield repo["ssh_url"]
        page += 1


def remove_prefix(text, prefix):
    return text[text.startswith(prefix) and len(prefix) :]


def get_fs_path(url: str):
    domain, path = remove_prefix(url, "git@").split(":")
    return Path(MIRROR_ROOT, domain, *path.split("/"))


def mirror_repo(repo_url):
    fs_path = get_fs_path(repo_url)
    fs_path.mkdir(parents=True, exist_ok=True)
    subprocess.run(["git", "clone", "--mirror", repo_url, fs_path])


def get_mirrors():
    return glob(f"{MIRROR_ROOT}/*/*/*.git")


@click.group()
def cli():
    pass


@cli.command()
def clone():
    click.echo("Cloning repos")
    repos = list(tqdm(get_my_repos()))
    for repo_url in tqdm(repos):
        fs_path = get_fs_path(repo_url)
        if os.path.exists(fs_path):
            continue
        mirror_repo(repo_url)


@cli.command()
def update():
    click.echo("Updating existing mirrors")
    for mirror_path in tqdm(get_mirrors()):
        print(mirror_path)
        subprocess.run(["git", "remote", "update"], cwd=mirror_path)
        subprocess.run(["git", "lfs", "fetch", "--all"], cwd=mirror_path)


if __name__ == "__main__":
    cli()