to_lines.py

from StringIO import StringIO
from hypothesis import given
from hypothesis.strategies import lists, binary


def to_lines(f, _block_size=32<<10):
    """file.__iter__ but in python and ought to be slower

    >>> list(to_lines(StringIO("foo\\nbar\\nbaz\\n"), _block_size=2))
    ['foo\\n', 'bar\\n', 'baz\\n']
    >>> list(to_lines(StringIO(""))) == list(StringIO(""))
    True
    """
    buffer = f.read(_block_size)
    cursor = 0
    while buffer:
        first_nl = buffer.find(b"\n", cursor)
        if first_nl == -1:
            more = f.read(_block_size)
            if not more:
                remainder = buffer[cursor:]
                if remainder:
                    yield remainder
                return
            else:
                buffer = buffer[cursor:] + more
                cursor = 0
        else:
            yield buffer[cursor:first_nl + 1]
            cursor = first_nl + 1


@given(lists(binary()))
def test_to_lines_acts_like_file_iter(lines):
    s = b"\n".join(lines)
    assert list(StringIO(s)) == list(to_lines(StringIO(s)))