Created
March 5, 2017 10:24
-
-
Save Millnert/8f938162867bb6e752ba2f6f57c44223 to your computer and use it in GitHub Desktop.
python-rados writes to ec-pool
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def put_object(ioctx, diskpath, objprefix): | |
""" Puts an object from disk, using (objprefix + filename) as key. | |
Naive code which reads entire object to memory before writing. | |
""" | |
bufsize = 2**20 ## 1 MiB | |
key = objprefix + os.path.basename(diskpath) | |
offset = 0 | |
sha = hashlib.sha512() | |
fhd = open(diskpath, 'rb') | |
while True: | |
datachunk = fhd.read(bufsize) | |
if datachunk != "": | |
sha.update(datachunk) | |
print("key: %s, chunk: %d, offset: %d" % (key, len(datachunk), offset)) | |
completion = ioctx.aio_write(key, datachunk, offset) | |
completion.wait_for_safe_and_cb() | |
offset += bufsize ## usually not true on the last of several | |
## chunked reads, but doesn't matter then | |
else: | |
break | |
fhd.close() | |
ioctx.set_xattr(key, 'sha512sum', sha.hexdigest()) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def put_object(ioctx, diskpath, objprefix): | |
""" Puts an object from disk, using (objprefix + filename) as key. | |
Naive code which reads entire object to memory before writing. | |
""" | |
bufsize = 2**20 ## 1 MiB | |
key = objprefix + os.path.basename(diskpath) | |
offset = 0 | |
sha = hashlib.sha512() | |
fhd = open(diskpath, 'rb') | |
while True: | |
datachunk = fhd.read(bufsize) | |
if datachunk != "": | |
sha.update(datachunk) | |
print("key: %s, chunk: %d, offset: %d" % (key, len(datachunk), offset)) | |
if offset == 0: | |
completion = ioctx.aio_write(key, datachunk, offset) | |
else: | |
completion = ioctx.aio_append(key, datachunk) | |
completion.wait_for_safe_and_cb() | |
offset += bufsize ## usually not true on the last of several | |
## chunked reads, but doesn't matter then | |
else: | |
break | |
fhd.close() | |
ioctx.set_xattr(key, 'sha512sum', sha.hexdigest()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment