Download all gists of a user

get_gists.py

import sys
from subprocess import call
import json
import os
import requests
import argparse
import json

from pathlib import Path
from subprocess import run

def download_gists(gists: list):
    # Todo add rate-limit https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#rate-limiting
    
    # Don't want to exceed rate limits
    # if authenticated you can make 1.4 requests per second
    # if not you can make 1 request per minute
    wait_time=1.4*4
    
    for gist in gists:
        time.sleep(wait_time)
        
        print("downloading gist")
        # call( [ "cd", f"{gist_folder}" ] )
        call([ "git", "clone", gist["git_pull_url"], f"{gist_folder}/{gist['id']}" ])
        
        if args.verbose:
            print(f"gist git_pull_url: {gist['git_pull_url']}")
        #fi

        #  name of the first file in the gist
        new_folder_name = f"{gist_folder}/" + sorted(list(gist.get("files", {}).keys()))[0].split(".")[0]
        
        if args.verbose:
            print(f"new_folder_name: {new_folder_name}")
        #fi
        
        # rename folders
        if args.rename == True:
            os.rename( f"{gist_folder}/{gist['id']}", new_folder_name )
        #fi

        description_file = os.path.join(new_folder_name, "description.txt")
        if args.verbose:
            print(f"description_file: {description_file}")
        #fi
        with open(description_file, "w") as f:
            f.write(f"{gist['description']}\n")
#enddef

def visit_pages(user: str):
    
    next_page = True
    i = 1
    print(f"visiting gist page {i}")
    while next_page:
        url = f"https://api.github.com/users/{user}/gists?page={i}"
        r = requests.get(url)

        if not len(r.json()):
            next_page = False
        else:
            i += 1
    #end-while

    # dump json for debugging if flag is present
    dump_json()
    download_gists(r.json())
#endef

def visit_1_page(user: str):
    url = f"https://api.github.com/users/{user}/gists?page=1"
    r = requests.get(url)
    if args.verbose == True:
        print(r.json())
    # fi

    # dump json for debugging if flag is present
    dump_json()
    # fi
    
    download_gists(r.json())
# enddef

def dump_json():
    if args.json_dump == True:
        f = open("gists.json", "w")
        f.write(json.dumps(r.json()))
        f.close()
#enddef

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="command-line parser")
    parser.add_argument("-u","--user", help="your username (required)")
    parser.add_argument("-V", "--verbose", action="store_true", help="print verbose output for debugging")
    parser.add_argument("-r", "--rename", action="store_true", default=True, help="rename to more readable folder names instead of keeping gist id")
    parser.add_argument("-json", "--json-dump", action="store_true", help="dump json file for debugging")
    parser.add_argument("-t", "--test", action="store_true", help="test by downloading only 1 page of gists")
    parser.add_argument("-o", "--output", help="output folder name, if none is provided default name is <user>-gists")
# user = "test"
# user = sys.argv[1]

    args = parser.parse_args()
    print(f"user: {args.user}")
    print(f"verbose: {args.verbose}")
    print(f"rename: {args.rename}")
    print(f"dump json for debugging: {args.json_dump}")
    print(f"target folder: {args.output}")

    if args.output == None:
        gist_folder = f"{args.user}-gists"
    else:
        gist_folder = args.output
    #fi

    #cmd_str = f"mkdir {gist_folder}" # && cd {gist_folder}"
    # print(cmd_str)
    Path(gist_folder).mkdir(exist_ok=True)

    if args.test == True:
        visit_1_page(args.user)
    else:
        visit_pages(args.user)

Run this with: wget https://gist.githubusercontent.com/tnn4/45e88f6068af629681f73916baf12993/raw/23aa21ea157238369820270b03ddb9b4a89bc918/get_gists.py, then
python get_gists.py -u <your_username> where your_username is your Github user name.