Skip to content

Instantly share code, notes, and snippets.

@zyr17
Created August 12, 2020 10:18
Show Gist options
  • Save zyr17/d6bb2ca1effda37cf1fa6ac97985c9e6 to your computer and use it in GitHub Desktop.
Save zyr17/d6bb2ca1effda37cf1fa6ac97985c9e6 to your computer and use it in GitHub Desktop.
split & transfer big files for unstable network and broken hard drives
import os
import sys
import random
import time
os.system('~/remount.sh')
filename = sys.argv[1]
fsize = os.path.getsize(filename)
blocksize = 100 * 1024 * 1024
nowpos = 0
count = 0
if len(sys.argv) > 3:
count = int(sys.argv[3])
nowpos = count * blocksize
def randstr(length = 32):
base = "0123456789qazwsxedcrfvtgbyhnujmikolp"
s = '/tmp/'
for i in range(length):
s += base[random.randint(0, len(base) - 1)]
return s
'''
file = open(filename, 'rb')
folder = randstr()
os.mkdir(folder)
while nowpos < fsize:
pname = '%s/%s.%06d' % (folder, filename, count)
file.seek(nowpos, 0)
with open(pname, 'wb') as f:
f.write(file.read(blocksize))
count += 1
nowpos += blocksize
'''
target = sys.argv[2]
tmpfile = randstr()
fails = []
while nowpos < fsize:
pname = '%s/%s.%06d' % (target, filename.split('/')[-1], count)
wait_expand = 1
starttime = 0
while True:
try:
starttime = time.time()
with open(filename, 'rb') as file:
file.seek(nowpos, 0)
with open(tmpfile, 'wb') as f:
f.write(file.read(blocksize))
wait_expand = 1
break
except:
if wait_expand > 5:
fname = filename.split('/')[-1] + '.' + '%06d' % (count)
print('Fail too many times! skip it.', fname)
fails.append(fname)
break
print('open file error! wait %ds and remount' % (30 * wait_expand))
time.sleep(30 * wait_expand)
wait_expand += 1
os.system('~/remount.sh')
if os.path.getsize(tmpfile) != 0:
cmd = 'rsync -P --rsh=ssh %s %s' % (tmpfile, pname)
print(pname, time.time() - starttime)
os.system(cmd)
count += 1
nowpos += blocksize
time.sleep(2)
os.remove(tmpfile)
if len(fails) > 0:
fails.append('')
with open('/tmp/transferfail.log', 'a') as f:
f.write('\n'.join(fails))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment