Save aws logs to postgresql

save_aws_logs_to_postgresql.py

import subprocess
import IPython
import psycopg2


def main(log_group):
    def process_log_entry(timestamp, log_group, log_entry):
        cursor.execute(
            "INSERT INTO cloudwatch_logs (timestamp, log_group, log_entry) VALUES (%s, %s, %s)",
            (timestamp, log_group, log_entry)
        )
        conn.commit()

    conn = psycopg2.connect(dsn="postgresql://localhost:5432/vncsna")
    cursor = conn.cursor()
    cursor.execute(
        """
        CREATE TABLE IF NOT EXISTS cloudwatch_logs (
            timestamp TIMESTAMP,
            log_group TEXT,
            log_entry TEXT
        )
        """
    )
    conn.commit()

    process = subprocess.Popen(
        ['aws', 'logs', 'tail', log_group, '--follow', '--output', 'json', '--since', '15d'],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        universal_newlines=True
    )
    for line in process.stdout:
        try:
            timestamp, log_group, log_entry = line.strip().split(maxsplit=2)
            process_log_entry(timestamp, log_group, log_entry)
            IPython.display.clear_output()
            print(timestamp)
        except Exception as e:
            print(f"Error processing log entry: {e}")
            continue
    conn.close()
    process.kill()

if __name__ == "__main__":
    main("<log-group-name>")