Created
December 11, 2024 20:39
-
-
Save cnolanminich/3ff622566d0f9d0ddf84a4477aa575fd to your computer and use it in GitHub Desktop.
example postgres resource
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
from dagster import asset, ResourceParam | |
from sqlalchemy import create_engine | |
# Define the PostgreSQL resource | |
class PostgresResource: | |
def __init__(self, connection_string): | |
self.connection_string = connection_string | |
def get_connection(self): | |
return create_engine(self.connection_string).connect() | |
# Define the asset that ingests data from PostgreSQL | |
@asset | |
def ingest_data_from_postgres(postgres: ResourceParam[PostgresResource]): | |
with postgres.get_connection() as conn: | |
query = "SELECT * FROM your_table" | |
df = pd.read_sql_query(query, conn) | |
# Perform any additional data processing here | |
return df | |
# Example of how to configure the resource | |
postgres_resource = PostgresResource(connection_string="postgresql://user:password@localhost:5432/your_database") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment