Last active
November 6, 2022 09:43
-
-
Save shuson/67b8bfcda372b59ac1710d4f403ef750 to your computer and use it in GitHub Desktop.
handle huge file reading in python
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from functools import partial, wraps | |
from typing import TextIO, Callable | |
def chunked_file(fp, block_size): | |
for chunk in iter(partial(fp.read, block_size), ''): | |
yield chunk | |
def transformer(data): | |
//do transformation for input data | |
output = data | |
return output | |
def read_file(file_to_read, file_to_write): | |
with open(file_to_read) as f_read: | |
for chunk in chunked_file(f_read): | |
//do transformation | |
output = transformer(chunk) | |
write_file(file_to_write, output) | |
def write_file(file_path, data): | |
with open(file_path, "w+") as f_write: | |
f_write.write(data) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment