github-org-cloner.py

"""This module provides tools to clone github repos. The GithubOrgCloner
can clone an entire org worth of codebases and create a local corpus of projects.
"""

import git
import shutil
import traceback
import urllib.parse
from argparse import ArgumentParser
from functools import partial
from getpass import getpass
from github import Github
from os import environ, makedirs, path
from subprocess import call


class GithubOrgCloner:
    def get_parser(self):
        """ Create parser for command line arguments """
        parser = ArgumentParser(
            usage=u'python archex/github.py -u\'\n\t\t\tUsername and password will be prompted.',
            description='Clone all your Github repositories.')
        parser.add_argument('-u', '--user', help='Your github username')
        parser.add_argument('-p', '--password', help=u'Github password')
        parser.add_argument('-t', '--token', help=u'Github OAuth token')
        parser.add_argument(
            '-o', '--org', help=u'Organisation/team. User used by default.')
        parser.add_argument(
            '-d', '--dest', help=u'Destination directory. Created if doesn\'t exist. [curr_dir]')
        parser.add_argument('--nopull', action='store_true',
                            help=u'Don\'t pull if repository exists. [false]')
        parser.add_argument('--shallow', action='store_true',
                            help=u'Perform shallow clone. [false]')
        parser.add_argument('--ssh', action='store_true',
                            help=u'Use ssh+git urls for checkout. [false]')
        return parser

    def get_github_client(self, args):
        """ Create github agent to auth """
        if args.token:
            g = Github(args.token)
        else:
            user = args.user
            password = args.password
            if not user:
                user = input(u'Username: ')
            if not password:
                password = getpass('Password: ')
            if not args.dest:
                args.dest = input(u'Destination: ')
            g = Github(user, password)
        return g

    def clone_org(self):
        """ Clone all repos """
        print("Cloning all repos from an org.")
        parser = self.get_parser()
        args = parser.parse_args()
        g = self.get_github_client(args)
        user = g.get_user().login
        # (BadCredentialsException, TwoFactorException, RateLimitExceededException)

        join = path.join
        if args.dest:
            if not path.exists(args.dest):
                makedirs(args.dest)
                print(u'mkdir -p "{}"'.format(args.dest))
            join = partial(path.join, args.dest)

        get_repos = g.get_organization(
            args.org).get_repos if args.org else g.get_user().get_repos
        for repo in get_repos():
            if not path.exists(join(repo.name)):
                clone_url = repo.clone_url
                if args.ssh:
                    clone_url = repo.ssh_url
                if args.shallow:
                    print(
                        u'Shallow cloning "{repo.full_name}"'.format(repo=repo))
                    call([u'git', u'clone', '--depth=1',
                          clone_url, join(repo.name)])
                else:
                    print(
                        u'Cloning "{repo.full_name}"'.format(repo=repo))
                    call([u'git', u'clone', clone_url, join(repo.name)])
            elif not args.nopull:
                print(u'Updating "{repo.name}"'.format(repo=repo))
                call([u'git', u'pull'], env=dict(
                    environ, GIT_DIR=join(repo.name, '.git').encode('utf8')))
            else:
                print(
                    u'Already cloned, skipping...\t"{repo.name}"'.format(repo=repo))
        print(u'FIN')


if __name__ == '__main__':
    GithubOrgCloner().clone_org()