Created
August 12, 2020 10:18
-
-
Save zyr17/d6bb2ca1effda37cf1fa6ac97985c9e6 to your computer and use it in GitHub Desktop.
split & transfer big files for unstable network and broken hard drives
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import sys | |
import random | |
import time | |
os.system('~/remount.sh') | |
filename = sys.argv[1] | |
fsize = os.path.getsize(filename) | |
blocksize = 100 * 1024 * 1024 | |
nowpos = 0 | |
count = 0 | |
if len(sys.argv) > 3: | |
count = int(sys.argv[3]) | |
nowpos = count * blocksize | |
def randstr(length = 32): | |
base = "0123456789qazwsxedcrfvtgbyhnujmikolp" | |
s = '/tmp/' | |
for i in range(length): | |
s += base[random.randint(0, len(base) - 1)] | |
return s | |
''' | |
file = open(filename, 'rb') | |
folder = randstr() | |
os.mkdir(folder) | |
while nowpos < fsize: | |
pname = '%s/%s.%06d' % (folder, filename, count) | |
file.seek(nowpos, 0) | |
with open(pname, 'wb') as f: | |
f.write(file.read(blocksize)) | |
count += 1 | |
nowpos += blocksize | |
''' | |
target = sys.argv[2] | |
tmpfile = randstr() | |
fails = [] | |
while nowpos < fsize: | |
pname = '%s/%s.%06d' % (target, filename.split('/')[-1], count) | |
wait_expand = 1 | |
starttime = 0 | |
while True: | |
try: | |
starttime = time.time() | |
with open(filename, 'rb') as file: | |
file.seek(nowpos, 0) | |
with open(tmpfile, 'wb') as f: | |
f.write(file.read(blocksize)) | |
wait_expand = 1 | |
break | |
except: | |
if wait_expand > 5: | |
fname = filename.split('/')[-1] + '.' + '%06d' % (count) | |
print('Fail too many times! skip it.', fname) | |
fails.append(fname) | |
break | |
print('open file error! wait %ds and remount' % (30 * wait_expand)) | |
time.sleep(30 * wait_expand) | |
wait_expand += 1 | |
os.system('~/remount.sh') | |
if os.path.getsize(tmpfile) != 0: | |
cmd = 'rsync -P --rsh=ssh %s %s' % (tmpfile, pname) | |
print(pname, time.time() - starttime) | |
os.system(cmd) | |
count += 1 | |
nowpos += blocksize | |
time.sleep(2) | |
os.remove(tmpfile) | |
if len(fails) > 0: | |
fails.append('') | |
with open('/tmp/transferfail.log', 'a') as f: | |
f.write('\n'.join(fails)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment