Download All Apple OSS Tarballs from Github

get_apple_oss.sh

#!/bin/zsh

: '
You need a personal access token for GitHub to avoid hitting the rate limit. Refer to the docs:
https://docs.github.com/en/rest/guides/getting-started-with-the-rest-api
https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
'

APPLE_OSS_DIR="all_apple_oss_archives"
APPLE_OSS_REPO_FILE="all_apple_oss_repo_names.txt"
APPLE_OSS_ARCHIVE_ALL_FILE="all_apple_oss_archives.txt"
GH_USERNAME="put_github_username_here"
GH_TOKEN="put_github_personal_token_here"

mkdir $APPLE_OSS_DIR
cd $APPLE_OSS_DIR
#rm APPLE_OSS_REPO_FILE

: ' Here we get all the repo names by iterating over the apple-oss-distributions user. We can get 100 repo name / page and we iterate over 6 pages (Apple has about 508 repos). The "name" field in JSON contains the repo name, we save it in a file
'
for i in {1..6}
do
  curl -u $GH_USERNAME:$GH_TOKEN "https://api.github.com/users/apple-oss-distributions/repos?page=$i&per_page=100" | grep \"name\" | grep -v Other | grep -v License | cut -d ":" -f 2 | cut -d "\"" -f 2 >> $APPLE_OSS_REPO_FILE
done

: ' Here we get all the tags per repo, the tags contain the ZIP and TAR archives, we save the TAR archive links to a file, and then download them
'
while read repo; do
  echo "$repo"
  for i in {1..5}; do
    curl -u $GH_USERNAME:$GH_TOKEN "https://api.github.com/repos/apple-oss-distributions/$repo/tags?page=$i&per_page=100" | grep tarball | cut -d ":" -f 2,3 | cut -d "\"" -f 2 >> "$repo.txt"
  done
  mkdir "$repo"
  cat "$repo.txt" >> $APPLE_OSS_ARCHIVE_ALL_FILE
  while read archive; do
    wget --no-clobber "$archive" -P "$repo"
  done <"$repo.txt"
  rm "$repo.txt"
done <$APPLE_OSS_REPO_FILE

#rm APPLE_OSS_REPO_FILE