Skip to content

Instantly share code, notes, and snippets.

@manzt

manzt/anywidget-arrow.ipynb

Last active December 4, 2024 22:28
Show Gist options
  • Save manzt/5c5e3de6d2cea65d2bb68af03db88249 to your computer and use it in GitHub Desktop.
Save manzt/5c5e3de6d2cea65d2bb68af03db88249 to your computer and use it in GitHub Desktop.
Download ZIP
anywidget-arrow.ipynb
Display the source blob
Display the rendered blob
Raw
anywidget-arrow.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/manzt/5c5e3de6d2cea65d2bb68af03db88249/anywidget-arrow.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "187f0e02-b9db-43a8-b211-379b96e5f1b9",
"metadata": {
"tags": [],
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "187f0e02-b9db-43a8-b211-379b96e5f1b9",
"outputId": "47917e5c-99ec-411b-cbd0-f8beb8d468f8"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.2/63.2 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25h"
]
}
],
"source": [
"# Uncomment below if in Colab\n",
"! pip install --quiet anywidget pandas pyarrow fastparquet duckdb"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "0c1e3433-02ef-4ca2-b57c-fca8b355bfd6",
"metadata": {
"tags": [],
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "0c1e3433-02ef-4ca2-b57c-fca8b355bfd6",
"outputId": "f5cf976e-fcd7-4944-a7c1-ec1259118a63"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"--2023-04-25 22:09:29-- https://raw.githubusercontent.com/vega/vega-datasets/main/data/population.json\n",
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
"HTTP request sent, awaiting response... 200 OK\n",
"Length: 27665 (27K) [text/plain]\n",
"Saving to: ‘population.json’\n",
"\n",
"\rpopulation.json 0%[ ] 0 --.-KB/s \rpopulation.json 100%[===================>] 27.02K --.-KB/s in 0s \n",
"\n",
"2023-04-25 22:09:29 (71.4 MB/s) - ‘population.json’ saved [27665/27665]\n",
"\n"
]
}
],
"source": [
"!wget https://raw.githubusercontent.com/vega/vega-datasets/main/data/population.json"
]
},
{
"cell_type": "markdown",
"id": "9f4bce7e-a279-4bbb-81b9-7a899b1490e2",
"metadata": {
"tags": [],
"id": "9f4bce7e-a279-4bbb-81b9-7a899b1490e2"
},
"source": [
"## IPC with traitlets"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "b318a902-5794-4d86-b558-a58bfe706295",
"metadata": {
"tags": [],
"colab": {
"base_uri": "https://localhost:8080/",
"height": 91,
"referenced_widgets": [
"7d5d84ba48af429e86fe029736adafcc",
"f9219907b86849c79e7734e129e2e69b"
]
},
"id": "b318a902-5794-4d86-b558-a58bfe706295",
"outputId": "ab54f77f-5d98-46cc-9cc9-67fefed9c0eb"
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"TraitsExample()"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "7d5d84ba48af429e86fe029736adafcc"
}
},
"metadata": {
"application/vnd.jupyter.widget-view+json": {
"colab": {
"custom_widget_manager": {
"url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/b3e629b1971e1542/manager.min.js"
}
}
}
}
}
],
"source": [
"import pathlib\n",
"\n",
"import pandas as pd\n",
"import pyarrow as pa\n",
"\n",
"import anywidget\n",
"import traitlets\n",
"\n",
"\n",
"class TraitsExample(anywidget.AnyWidget):\n",
" _esm = \"\"\"\n",
" import * as arrow from \"https://cdn.jsdelivr.net/npm/apache-arrow@11/+esm\";\n",
" import * as Inputs from \"https://cdn.jsdelivr.net/npm/@observablehq/inputs/+esm\";\n",
"\n",
" export function render(view) {\n",
" view.model.on(\"change:_ipc\", () => {\n",
" let table = arrow.tableFromIPC(view.model.get(\"_ipc\").buffer);\n",
" let el = Inputs.table(table);\n",
" if (view.el.firstChild) {\n",
" view.el.firstChild.replaceWith(el);\n",
" } else {\n",
" view.el.appendChild(el);\n",
" }\n",
" });\n",
" let ipc = view.model.get(\"_ipc\");\n",
" if (!ipc) return;\n",
" let table = arrow.tableFromIPC(ipc.buffer);\n",
" view.el.appendChild(Inputs.table(table));\n",
" }\n",
" \"\"\"\n",
" _ipc = traitlets.Bytes().tag(sync=True)\n",
" df = traitlets.Any()\n",
" \n",
" @traitlets.observe(\"df\")\n",
" def change(self, change):\n",
" table = pa.Table.from_pandas(change.new)\n",
" sink = pa.BufferOutputStream()\n",
" with pa.ipc.new_stream(sink, table.schema) as writer:\n",
" writer.write(table)\n",
" buf = sink.getvalue()\n",
" self._ipc = buf.to_pybytes()\n",
"\n",
" \n",
"df = pd.read_json('./population.json')\n",
"w = TraitsExample(df=df)\n",
"w"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "5eabc7d0-6a9a-411e-83a7-8465fe3b2134",
"metadata": {
"id": "5eabc7d0-6a9a-411e-83a7-8465fe3b2134"
},
"outputs": [],
"source": [
"# the df property is \"reactive\", and assignment triggers the \n",
"# 'change:_ipc' event handler in the frontend\n",
"w.df = df.iloc[0:2]"
]
},
{
"cell_type": "markdown",
"id": "9d643579-bdd4-4724-8dea-3ad893dcb055",
"metadata": {
"id": "9d643579-bdd4-4724-8dea-3ad893dcb055"
},
"source": [
"## IPC with custom messages"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "f509d423-8986-46b3-9814-619bb531d8d8",
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 625,
"referenced_widgets": [
"5ff885222cb049129189c5eed522511b",
"21ea4bf92cda49efbada22e2216a5927"
]
},
"id": "f509d423-8986-46b3-9814-619bb531d8d8",
"outputId": "b7594092-e3f9-42c6-a072-f69d2b8d851c"
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"DuckDB()"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "5ff885222cb049129189c5eed522511b"
}
},
"metadata": {
"application/vnd.jupyter.widget-view+json": {
"colab": {
"custom_widget_manager": {
"url": "https://ssl.gstatic.com/colaboratory-static/widgets/colab-cdn-widget-manager/b3e629b1971e1542/manager.min.js"
}
}
}
}
}
],
"source": [
"import duckdb\n",
"\n",
"class DuckDB(anywidget.AnyWidget):\n",
" _esm = \"\"\"\n",
" import * as arrow from \"https://cdn.jsdelivr.net/npm/apache-arrow@11/+esm\";\n",
" import * as Inputs from \"https://cdn.jsdelivr.net/npm/@observablehq/inputs/+esm\";\n",
" \n",
" function send(query, view, { timeout = 3000 } = {}) {\n",
" let uuid = globalThis.crypto.randomUUID();\n",
" return new Promise((resolve, reject) => {\n",
" let timer = setTimeout(() => {\n",
" reject(new Error(`Promise timed out after ${timeout} ms`));\n",
" view.model.off(\"msg:custom\", handler);\n",
" }, timeout);\n",
" function handler(msg, buffers) {\n",
" if (!(msg.uuid === uuid)) return;\n",
" clearTimeout(timer)\n",
" resolve(buffers[0].buffer);\n",
" view.model.off(\"msg:custom\", handler);\n",
" }\n",
" view.model.on(\"msg:custom\", handler)\n",
" view.model.send({ query, uuid });\n",
" })\n",
" }\n",
"\n",
" export function render(view) {\n",
" let textarea = Object.assign(document.createElement(\"textarea\"), {\n",
" value: `SELECT * from \"population.json\"\\n WHERE sex == 1`,\n",
" });\n",
" textarea.style.width = \"300px\";\n",
" let button = Object.assign(document.createElement(\"button\"), {\n",
" innerText: \"submit\",\n",
" })\n",
" let table_el;\n",
" button.addEventListener(\"click\", async () => {\n",
" let buffer = await send(textarea.value, view);\n",
" let table = arrow.tableFromIPC(buffer);\n",
" if (table_el) table_el.remove();\n",
" table_el = Inputs.table(table);\n",
" view.el.appendChild(table_el);\n",
" });\n",
" view.el.appendChild(textarea);\n",
" view.el.appendChild(button);\n",
" }\n",
" \"\"\"\n",
" \n",
" def __init__(self, *args, **kwargs):\n",
" super().__init__(*args, **kwargs)\n",
" self.on_msg(self._handle_custom_msg)\n",
"\n",
" def _handle_custom_msg(self, msg: dict, buffers: list):\n",
" table = duckdb.sql(msg[\"query\"]).arrow()\n",
" sink = pa.BufferOutputStream()\n",
" with pa.ipc.new_stream(sink, table.schema) as writer:\n",
" writer.write(table) \n",
" self.send({ \"uuid\": msg[\"uuid\"] }, [sink.getvalue().to_pybytes()]) \n",
" \n",
"w = DuckDB()\n",
"w"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "94119aca-e4b0-417e-9b63-d079ee39c1f2",
"metadata": {
"id": "94119aca-e4b0-417e-9b63-d079ee39c1f2"
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0370115-a5ae-4609-84de-7015fcbcdc04",
"metadata": {
"id": "d0370115-a5ae-4609-84de-7015fcbcdc04"
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
},
"colab": {
"provenance": [],
"include_colab_link": true
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"7d5d84ba48af429e86fe029736adafcc": {
"model_module": "anywidget",
"model_name": "AnyModel",
"model_module_version": "0.2.3",
"state": {
"_dom_classes": [],
"_model_module": "anywidget",
"_model_module_version": "0.2.3",
"_model_name": "AnyModel",
"_view_count": null,
"_view_module": "anywidget",
"_view_module_version": "0.2.3",
"_view_name": "AnyView",
"layout": "IPY_MODEL_f9219907b86849c79e7734e129e2e69b",
"_esm": "\n import * as arrow from \"https://cdn.jsdelivr.net/npm/apache-arrow@11/+esm\";\n import * as Inputs from \"https://cdn.jsdelivr.net/npm/@observablehq/inputs/+esm\";\n\n export function render(view) {\n view.model.on(\"change:_ipc\", () => {\n let table = arrow.tableFromIPC(view.model.get(\"_ipc\").buffer);\n let el = Inputs.table(table);\n if (view.el.firstChild) {\n view.el.firstChild.replaceWith(el);\n } else {\n view.el.appendChild(el);\n }\n });\n let ipc = view.model.get(\"_ipc\");\n if (!ipc) return;\n let table = arrow.tableFromIPC(ipc.buffer);\n view.el.appendChild(Inputs.table(table));\n }\n ",
"_anywidget_id": "__main__.TraitsExample"
}
},
"f9219907b86849c79e7734e129e2e69b": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"5ff885222cb049129189c5eed522511b": {
"model_module": "anywidget",
"model_name": "AnyModel",
"model_module_version": "0.2.3",
"state": {
"_dom_classes": [],
"_model_module": "anywidget",
"_model_module_version": "0.2.3",
"_model_name": "AnyModel",
"_view_count": null,
"_view_module": "anywidget",
"_view_module_version": "0.2.3",
"_view_name": "AnyView",
"layout": "IPY_MODEL_21ea4bf92cda49efbada22e2216a5927",
"_esm": "\n import * as arrow from \"https://cdn.jsdelivr.net/npm/apache-arrow@11/+esm\";\n import * as Inputs from \"https://cdn.jsdelivr.net/npm/@observablehq/inputs/+esm\";\n \n function send(query, view, { timeout = 3000 } = {}) {\n let uuid = globalThis.crypto.randomUUID();\n return new Promise((resolve, reject) => {\n let timer = setTimeout(() => {\n reject(new Error(`Promise timed out after ${timeout} ms`));\n view.model.off(\"msg:custom\", handler);\n }, timeout);\n function handler(msg, buffers) {\n if (!(msg.uuid === uuid)) return;\n clearTimeout(timer)\n resolve(buffers[0].buffer);\n view.model.off(\"msg:custom\", handler);\n }\n view.model.on(\"msg:custom\", handler)\n view.model.send({ query, uuid });\n })\n }\n\n export function render(view) {\n let textarea = Object.assign(document.createElement(\"textarea\"), {\n value: `SELECT * from \"population.json\"\n WHERE sex == 1`,\n });\n textarea.style.width = \"300px\";\n let button = Object.assign(document.createElement(\"button\"), {\n innerText: \"submit\",\n })\n let table_el;\n button.addEventListener(\"click\", async () => {\n let buffer = await send(textarea.value, view);\n let table = arrow.tableFromIPC(buffer);\n if (table_el) table_el.remove();\n table_el = Inputs.table(table);\n view.el.appendChild(table_el);\n });\n view.el.appendChild(textarea);\n view.el.appendChild(button);\n }\n ",
"_anywidget_id": "__main__.DuckDB"
}
},
"21ea4bf92cda49efbada22e2216a5927": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
}
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment