A python script to clean up google cloud logs to just the important stuff

parse-logs.py

import json
import sys
import os

def filter_json_array(input_file, output_file=None):
    # Set default output file if not provided
    if output_file is None:
        input_dir = os.path.dirname(input_file) or '.'
        output_file = os.path.join(input_dir, "processed_logs.json")
    
    # Read input JSON
    with open(input_file, 'r') as f:
        data = json.load(f)
    
    # Ensure data is an array
    if not isinstance(data, list):
        raise ValueError("Input JSON is not an array")
    
    # Filter objects
    filtered_data = []
    for item in data:
        filtered_item = {}
        # Only include specified fields if they exist
        for field in ['severity', 'jsonPayload', 'textPayload', 'timestamp']:
            if field in item:
                filtered_item[field] = item[field]
        filtered_data.append(filtered_item)
    
    # Write output
    with open(output_file, 'w') as f:
        json.dump(filtered_data, f, indent=2)
    
    print(f"Processed {len(data)} objects. Output written to {output_file}")

if __name__ == "__main__":
    if len(sys.argv) < 2 or len(sys.argv) > 3:
        print("Usage: python filter_json.py <input_file> [output_file]")
        sys.exit(1)
    
    input_file = sys.argv[1]
    output_file = sys.argv[2] if len(sys.argv) == 3 else None
    
    try:
        filter_json_array(input_file, output_file)
    except Exception as e:
        print(f"Error: {e}")
        sys.exit(1)

Usage: python filter_json.py <input_file> [output_file]