Trim embedding decimals from pandas

trim_embedding_decimals.py

import pandas as pd
import json
import gzip

# Load data
df = pd.read_json("your_file.json.gz")

# Round embeddings to 4 decimal places
df["embeddings"] = df["embeddings"].apply(lambda emb: [round(e, 4) for e in emb])

# Save to a gzipped JSON file
with gzip.open("your_file__4_dec.json.gz", "wt", encoding="utf-8") as f:
    json.dump(df.to_dict(orient="records"), f)