embed_and_store.py

@action(reads=["markdown_content"], writes=[])
def embed_and_store(state: State, webpage_url: str) -> State:
    """Embed and store the Markdown content."""
    markdown_content = state["markdown_content"]

    chunks = get_text_chunks(markdown_content)

    # Embed and store the chunks using LanceDB
    con = lancedb.connect("./webpages")
    table = con.create_table("chunks", exist_ok=True, schema=TextDocument)
    table.add([
        {"text": c, "url": webpage_url, "position": i}
        for i, c in enumerate(chunks)
    ])

    return state