remote_file_processor.py

import requests
import hashlib
import subprocess
import time

url = 'http://cachefly.cachefly.net/100mb.test'
backup_file = 'backup.out'
md5sum_file = 'backup.out.md5'
gpg_file = 'backup.out.gpg'

gpg_recipient = '[email protected]'
keyring_file = ''

chunk_size = 16*1024

class MD5Filter(object):
    def __init__(self, out_filename):
        self._hash = hashlib.md5()
        self._hash_filename = out_filename

    def processor(self, chunk):
        self._hash.update(chunk)

    def hash(self):
        return self._hash.hexdigest()

    def save(self):
        with open(self._hash_filename, 'w') as fh:
            fh.write(self.hash())

class GPGFilter(object):
    def __init__(self, out_filename, recipient, keyring):
        self._recipeint = recipient
        self._keyring = keyring
        self._gpg_file = out_filename
        
        self._gpg_process = self._create_gpg_process()

    def processor(self, chunk):
        self._gpg_process.stdin.write(chunk)

    def save(self):
        self._gpg_process.stdin.close()

    def _create_gpg_process(self):
        gpg_cmd = ['gpg2',
                   '--trust-model', 'always',
                   '-q', '--batch', '--yes',
                   '--compress-algo', 'none',
                   '--encrypt',
                   '--recipient', self._recipeint]
        gpg = subprocess.Popen(gpg_cmd, stdin=subprocess.PIPE, 
                               stdout=open(self._gpg_file, 'wb'))

        return gpg

class ChunkedStream(object):
    def __init__(self, url, chunk_size):
        self._url = url
        self._chunk_size = chunk_size

        response = requests.get(url, stream=True)
        self._stream = response.iter_content(chunk_size=chunk_size)

        self._current_chunk = None

    def __iter__(self):
        for chunk in self._stream:
            self._current_chunk = chunk
            yield self

    def process(self, processer_func):
        if self._current_chunk is not None:
            processer_func(self._current_chunk)

        return self

start_time = time.time()

md5_filter = MD5Filter(md5sum_file)
gpg_filter = GPGFilter(gpg_file, gpg_recipient, keyring_file)

chunked_stream = ChunkedStream(url, chunk_size)
for chunk in chunked_stream:
    chunk.process(md5_filter.processor).process(gpg_filter.processor)

md5_filter.save()
gpg_filter.save()

end_time = time.time()

print("Time taken: %s seconds" % str(end_time - start_time))
print("MD5 checksum: %s" % md5_filter.hash())