foo.py

#!/usr/bin/env python3

import json
import time

start = time.time()

L = []

i = 0
with open('in.json') as f:
    for line in f:
        L.append(json.loads(line))
        i += 1
        if i % 100000 == 0:
            print(i)

print('read', time.time() - start)
L.sort(key=lambda x: x['id'])
print('sort', time.time() - start)

i = 0
with open('out.json', 'w') as f:
    for d in L:
        f.write(json.dumps(d, sort_keys=True) + '\n')

        i += 1
        if i % 100000 == 0:
            print(i)

print('write', time.time() - start)

not my experience dealing with large new line delimited JSON files. what's inside in.json? the bottleneck has always been with the json module...

It's part of a db table dump - just part of the largest line-delimited json I had lying around -- I was curious what a python script could do re https://genius.engineering/faster-and-simpler-with-the-command-line-deep-comparing-two-5gb-json-files-3x-faster-by-ditching-the-code/