Skip to content

Instantly share code, notes, and snippets.

@astrojuanlu

astrojuanlu/experiments-delta-cdf.ipynb

Created July 14, 2024 21:00
Show Gist options
  • Save astrojuanlu/41add9bb28f11a220496f9ead1943deb to your computer and use it in GitHub Desktop.
Save astrojuanlu/41add9bb28f11a220496f9ead1943deb to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
experiments-delta-cdf.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "de06530c-995d-46d5-8a50-ef59066a9cb4",
"showTitle": false,
"title": ""
}
},
"source": [
"Source: https://web.archive.org/web/20230202153818/https://www.databricks.com/notebooks/delta-lake-cdf.html\n",
"\n",
"![CDF](https://databricks.com/wp-content/uploads/2021/05/cdf-blog-img-1-rev.png)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "84279827-d1fe-4e6d-b52d-610be255aafa",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr></tr></thead><tbody></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": null
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"CREATE TABLE bronze_eps \n",
" (date STRING, stock_symbol STRING, analyst INT, estimated_eps DOUBLE) \n",
" USING DELTA\n",
" TBLPROPERTIES (delta.enableChangeDataFeed = true);\n",
"\n",
"CREATE TABLE silver_eps\n",
" (date STRING, stock_symbol STRING, analyst INT, estimated_eps DOUBLE) \n",
" USING DELTA\n",
" TBLPROPERTIES (delta.enableChangeDataFeed = true);\n",
"\n",
"CREATE TABLE gold_consensus_eps \n",
" (date STRING, stock_symbol STRING, consensus_eps DOUBLE) \n",
" USING DELTA\n",
" TBLPROPERTIES (delta.enableChangeDataFeed = true);"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "60da249c-f6ec-49ce-8a86-8ec8297cca27",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": [
"df = spark.createDataFrame(\n",
" [\n",
" ('3/1/2021', 'a', 1, 2.2),\n",
" ('3/1/2021', 'a', 2, 2.0),\n",
" ('3/1/2021', 'b', 1, 1.3),\n",
" ('3/1/2021', 'b', 2, 1.2),\n",
" ('3/1/2021', 'c', 1, 3.5),\n",
" ('3/1/2021', 'c', 2, 2.6)\n",
" ],\n",
" ('date', 'stock_symbol', 'analyst', 'estimated_eps')\n",
")\n",
"\n",
"df.createOrReplaceTempView(\"bronze_eps_march_dataset\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "1e2ab69f-fbcb-4107-b7ff-c762dd5a3606",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>6</td><td>6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
6,
6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 2
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"INSERT INTO bronze_eps TABLE bronze_eps_march_dataset"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "c5bfee9c-fbe1-4b46-b055-f382f9cf4f09",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"a",
2,
2.0
],
[
"3/1/2021",
"b",
1,
1.3
],
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 4
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM bronze_eps"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "e5db197d-adc9-4bbb-bac4-438d9f0e3216",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th><th>_change_type</th><th>_commit_version</th><th>_commit_timestamp</th></tr></thead><tbody><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td><td>insert</td><td>1</td><td>2024-07-14T16:07:59Z</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"b",
2,
1.2,
"insert",
1,
"2024-07-14T16:07:59Z"
],
[
"3/1/2021",
"a",
1,
2.2,
"insert",
1,
"2024-07-14T16:07:59Z"
],
[
"3/1/2021",
"a",
2,
2.0,
"insert",
1,
"2024-07-14T16:07:59Z"
],
[
"3/1/2021",
"c",
1,
3.5,
"insert",
1,
"2024-07-14T16:07:59Z"
],
[
"3/1/2021",
"c",
2,
2.6,
"insert",
1,
"2024-07-14T16:07:59Z"
],
[
"3/1/2021",
"b",
1,
1.3,
"insert",
1,
"2024-07-14T16:07:59Z"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 3
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
},
{
"metadata": "{}",
"name": "_change_type",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "_commit_version",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "_commit_timestamp",
"type": "\"timestamp\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM table_changes('bronze_eps', 1)"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "65d13004-c56d-4f48-b771-b68f43b9af2b",
"showTitle": false,
"title": ""
}
},
"source": [
"---"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "70a9ffa2-c70a-461e-9ea5-36b1ecf6d0e5",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>catalog</th></tr></thead><tbody><tr><td>spark_catalog</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"spark_catalog"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 9
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "catalog",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SHOW catalogs;"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "67d647b2-0319-4bd8-93fb-db13233fe7cc",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>current_catalog()</th></tr></thead><tbody><tr><td>spark_catalog</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"spark_catalog"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 6
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{\"__autoGeneratedAlias\":\"true\"}",
"name": "current_catalog()",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT current_catalog()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "668f07a2-6d6f-433e-b978-d8c906429d4d",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>databaseName</th></tr></thead><tbody><tr><td>default</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"default"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 10
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "databaseName",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SHOW schemas;"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "7f3a8f75-b891-47b9-8dc4-b0577e08966a",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>current_schema()</th></tr></thead><tbody><tr><td>default</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"default"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 7
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{\"__autoGeneratedAlias\":\"true\"}",
"name": "current_schema()",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT current_database()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "4c709ac8-12b5-41a9-ab2f-0fbafc3fbc03",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"org.apache.spark.sql.AnalysisException: [OPERATION_REQUIRES_UNITY_CATALOG] Operation CURRENT_METASTORE requires Unity Catalog enabled. SQLSTATE: 0AKUD\n",
"\tat com.databricks.sql.acl.InlineUserInfoExpressions.com$databricks$sql$acl$InlineUserInfoExpressions$$validateUcEnabled(InlineUserInfoExpressions.scala:200)\n",
"\tat com.databricks.sql.acl.InlineUserInfoExpressions$$anonfun$rewrite$2.applyOrElse(InlineUserInfoExpressions.scala:148)\n",
"\tat com.databricks.sql.acl.InlineUserInfoExpressions$$anonfun$rewrite$2.applyOrElse(InlineUserInfoExpressions.scala:85)\n",
"\tat org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:477)\n",
"\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n",
"\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:477)\n",
"\tat org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$3(TreeNode.scala:482)\n",
"\tat org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1279)\n",
"\tat org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1278)\n",
"\tat org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:663)\n",
"\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:482)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDownWithPruning$1(QueryPlan.scala:174)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:215)\n",
"\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:215)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:226)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$3(QueryPlan.scala:231)\n",
"\tat scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)\n",
"\tat scala.collection.immutable.List.foreach(List.scala:431)\n",
"\tat scala.collection.TraversableLike.map(TraversableLike.scala:286)\n",
"\tat scala.collection.TraversableLike.map$(TraversableLike.scala:279)\n",
"\tat scala.collection.immutable.List.map(List.scala:305)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:231)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:236)\n",
"\tat org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:323)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:236)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDownWithPruning(QueryPlan.scala:174)\n",
"\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsWithPruning(QueryPlan.scala:145)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$$anonfun$resolveExpressionsWithPruning$1.applyOrElse(AnalysisHelper.scala:298)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$$anonfun$resolveExpressionsWithPruning$1.applyOrElse(AnalysisHelper.scala:297)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsDownWithPruning$2(AnalysisHelper.scala:219)\n",
"\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsDownWithPruning$1(AnalysisHelper.scala:219)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:400)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsDownWithPruning(AnalysisHelper.scala:217)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsDownWithPruning$(AnalysisHelper.scala:213)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsDownWithPruning(LogicalPlan.scala:39)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsWithPruning(AnalysisHelper.scala:102)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsWithPruning$(AnalysisHelper.scala:99)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsWithPruning(LogicalPlan.scala:39)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveExpressionsWithPruning(AnalysisHelper.scala:297)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveExpressionsWithPruning$(AnalysisHelper.scala:295)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveExpressionsWithPruning(LogicalPlan.scala:39)\n",
"\tat com.databricks.sql.acl.InlineUserInfoExpressions.rewrite(InlineUserInfoExpressions.scala:85)\n",
"\tat com.databricks.sql.acl.InlineUserInfoExpressions.rewrite(InlineUserInfoExpressions.scala:56)\n",
"\tat com.databricks.sql.optimizer.DatabricksEdgeRule.apply(DatabricksEdgeRule.scala:36)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$4(RuleExecutor.scala:309)\n",
"\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$3(RuleExecutor.scala:309)\n",
"\tat scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)\n",
"\tat scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)\n",
"\tat scala.collection.immutable.List.foldLeft(List.scala:91)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:306)\n",
"\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n",
"\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.executeBatch$1(RuleExecutor.scala:289)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$9(RuleExecutor.scala:382)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$9$adapted(RuleExecutor.scala:382)\n",
"\tat scala.collection.immutable.List.foreach(List.scala:431)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:382)\n",
"\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:256)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.executeSameContext(Analyzer.scala:415)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$execute$1(Analyzer.scala:408)\n",
"\tat org.apache.spark.sql.catalyst.analysis.AnalysisContext$.withNewAnalysisContext(Analyzer.scala:322)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:408)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:341)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:248)\n",
"\tat org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:166)\n",
"\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:248)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$executeAndCheck$1(Analyzer.scala:393)\n",
"\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:407)\n",
"\tat org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:392)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$analyzed$1(QueryExecution.scala:247)\n",
"\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n",
"\tat org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:394)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$4(QueryExecution.scala:576)\n",
"\tat org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:1097)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:576)\n",
"\tat com.databricks.util.LexicalThreadLocal$Handle.runWith(LexicalThreadLocal.scala:63)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:572)\n",
"\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:572)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:241)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:240)\n",
"\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:222)\n",
"\tat org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:126)\n",
"\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n",
"\tat org.apache.spark.sql.SparkSession.$anonfun$withActiveAndFrameProfiler$1(SparkSession.scala:1182)\n",
"\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n",
"\tat org.apache.spark.sql.SparkSession.withActiveAndFrameProfiler(SparkSession.scala:1182)\n",
"\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:116)\n",
"\tat org.apache.spark.sql.SparkSession.$anonfun$sql$4(SparkSession.scala:954)\n",
"\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n",
"\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:942)\n",
"\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:977)\n",
"\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:1010)\n",
"\tat org.apache.spark.sql.SQLContext.sql(SQLContext.scala:696)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal$DbClassicStrategy.executeSQLQuery(DriverLocal.scala:277)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.executeSQLSubCommand(DriverLocal.scala:367)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$executeSql$1(DriverLocal.scala:388)\n",
"\tat scala.collection.immutable.List.map(List.scala:293)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.executeSql(DriverLocal.scala:383)\n",
"\tat com.databricks.backend.daemon.driver.JupyterDriverLocal.repl(JupyterDriverLocal.scala:970)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$29(DriverLocal.scala:1108)\n",
"\tat com.databricks.unity.EmptyHandle$.runWith(UCSHandle.scala:128)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$24(DriverLocal.scala:1099)\n",
"\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:426)\n",
"\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n",
"\tat com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:216)\n",
"\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:424)\n",
"\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:418)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:88)\n",
"\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:472)\n",
"\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:455)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:88)\n",
"\tat com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:1044)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$2(DriverWrapper.scala:786)\n",
"\tat scala.util.Try$.apply(Try.scala:213)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:778)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:806)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:685)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:730)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:556)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:482)\n",
"\tat com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:290)\n",
"\tat java.lang.Thread.run(Thread.java:750)\n"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"arguments": {},
"datasetInfos": [],
"jupyterProps": null,
"metadata": {
"errorSummary": "[OPERATION_REQUIRES_UNITY_CATALOG] Operation CURRENT_METASTORE requires Unity Catalog enabled. SQLSTATE: 0AKUD"
},
"removedWidgets": [],
"sqlProps": {
"errorClass": "OPERATION_REQUIRES_UNITY_CATALOG",
"sqlState": "0AKUD",
"startIndex": null,
"stopIndex": null
},
"stackFrames": [
"org.apache.spark.sql.AnalysisException: [OPERATION_REQUIRES_UNITY_CATALOG] Operation CURRENT_METASTORE requires Unity Catalog enabled. SQLSTATE: 0AKUD\n\tat com.databricks.sql.acl.InlineUserInfoExpressions.com$databricks$sql$acl$InlineUserInfoExpressions$$validateUcEnabled(InlineUserInfoExpressions.scala:200)\n\tat com.databricks.sql.acl.InlineUserInfoExpressions$$anonfun$rewrite$2.applyOrElse(InlineUserInfoExpressions.scala:148)\n\tat com.databricks.sql.acl.InlineUserInfoExpressions$$anonfun$rewrite$2.applyOrElse(InlineUserInfoExpressions.scala:85)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:477)\n\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:477)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$3(TreeNode.scala:482)\n\tat org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1279)\n\tat org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1278)\n\tat org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:663)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:482)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDownWithPruning$1(QueryPlan.scala:174)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:215)\n\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:215)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:226)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$3(QueryPlan.scala:231)\n\tat scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)\n\tat scala.collection.immutable.List.foreach(List.scala:431)\n\tat scala.collection.TraversableLike.map(TraversableLike.scala:286)\n\tat scala.collection.TraversableLike.map$(TraversableLike.scala:279)\n\tat scala.collection.immutable.List.map(List.scala:305)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:231)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:236)\n\tat org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:323)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:236)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDownWithPruning(QueryPlan.scala:174)\n\tat org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsWithPruning(QueryPlan.scala:145)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$$anonfun$resolveExpressionsWithPruning$1.applyOrElse(AnalysisHelper.scala:298)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$$anonfun$resolveExpressionsWithPruning$1.applyOrElse(AnalysisHelper.scala:297)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsDownWithPruning$2(AnalysisHelper.scala:219)\n\tat org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:83)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.$anonfun$resolveOperatorsDownWithPruning$1(AnalysisHelper.scala:219)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.allowInvokingTransformsInAnalyzer(AnalysisHelper.scala:400)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsDownWithPruning(AnalysisHelper.scala:217)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsDownWithPruning$(AnalysisHelper.scala:213)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsDownWithPruning(LogicalPlan.scala:39)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsWithPruning(AnalysisHelper.scala:102)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveOperatorsWithPruning$(AnalysisHelper.scala:99)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveOperatorsWithPruning(LogicalPlan.scala:39)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveExpressionsWithPruning(AnalysisHelper.scala:297)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.resolveExpressionsWithPruning$(AnalysisHelper.scala:295)\n\tat org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveExpressionsWithPruning(LogicalPlan.scala:39)\n\tat com.databricks.sql.acl.InlineUserInfoExpressions.rewrite(InlineUserInfoExpressions.scala:85)\n\tat com.databricks.sql.acl.InlineUserInfoExpressions.rewrite(InlineUserInfoExpressions.scala:56)\n\tat com.databricks.sql.optimizer.DatabricksEdgeRule.apply(DatabricksEdgeRule.scala:36)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$4(RuleExecutor.scala:309)\n\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$3(RuleExecutor.scala:309)\n\tat scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)\n\tat scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)\n\tat scala.collection.immutable.List.foldLeft(List.scala:91)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$2(RuleExecutor.scala:306)\n\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.executeBatch$1(RuleExecutor.scala:289)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$9(RuleExecutor.scala:382)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$9$adapted(RuleExecutor.scala:382)\n\tat scala.collection.immutable.List.foreach(List.scala:431)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$execute$1(RuleExecutor.scala:382)\n\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:256)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.executeSameContext(Analyzer.scala:415)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$execute$1(Analyzer.scala:408)\n\tat org.apache.spark.sql.catalyst.analysis.AnalysisContext$.withNewAnalysisContext(Analyzer.scala:322)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:408)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.execute(Analyzer.scala:341)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.$anonfun$executeAndTrack$1(RuleExecutor.scala:248)\n\tat org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:166)\n\tat org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:248)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.$anonfun$executeAndCheck$1(Analyzer.scala:393)\n\tat org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$.markInAnalyzer(AnalysisHelper.scala:407)\n\tat org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:392)\n\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$analyzed$1(QueryExecution.scala:247)\n\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n\tat org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:394)\n\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$4(QueryExecution.scala:576)\n\tat org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:1097)\n\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:576)\n\tat com.databricks.util.LexicalThreadLocal$Handle.runWith(LexicalThreadLocal.scala:63)\n\tat org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:572)\n\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n\tat org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:572)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:241)\n\tat org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:240)\n\tat org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:222)\n\tat org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:126)\n\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n\tat org.apache.spark.sql.SparkSession.$anonfun$withActiveAndFrameProfiler$1(SparkSession.scala:1182)\n\tat com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:94)\n\tat org.apache.spark.sql.SparkSession.withActiveAndFrameProfiler(SparkSession.scala:1182)\n\tat org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:116)\n\tat org.apache.spark.sql.SparkSession.$anonfun$sql$4(SparkSession.scala:954)\n\tat org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:1175)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:942)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:977)\n\tat org.apache.spark.sql.SparkSession.sql(SparkSession.scala:1010)\n\tat org.apache.spark.sql.SQLContext.sql(SQLContext.scala:696)\n\tat com.databricks.backend.daemon.driver.DriverLocal$DbClassicStrategy.executeSQLQuery(DriverLocal.scala:277)\n\tat com.databricks.backend.daemon.driver.DriverLocal.executeSQLSubCommand(DriverLocal.scala:367)\n\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$executeSql$1(DriverLocal.scala:388)\n\tat scala.collection.immutable.List.map(List.scala:293)\n\tat com.databricks.backend.daemon.driver.DriverLocal.executeSql(DriverLocal.scala:383)\n\tat com.databricks.backend.daemon.driver.JupyterDriverLocal.repl(JupyterDriverLocal.scala:970)\n\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$29(DriverLocal.scala:1108)\n\tat com.databricks.unity.EmptyHandle$.runWith(UCSHandle.scala:128)\n\tat com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$24(DriverLocal.scala:1099)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:426)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:216)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:424)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:418)\n\tat com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:88)\n\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:472)\n\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:455)\n\tat com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:88)\n\tat com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:1044)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$2(DriverWrapper.scala:786)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:778)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:806)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:685)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:730)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:556)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:482)\n\tat com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:290)\n\tat java.lang.Thread.run(Thread.java:750)\n"
],
"type": "baseError"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT current_metastore()"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "f251a709-48e4-44dd-a671-e9e49a2a6b4a",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"a",
2,
2.0
],
[
"3/1/2021",
"b",
1,
1.3
],
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 8
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"USE CATALOG spark_catalog;\n",
"USE SCHEMA default;\n",
"SELECT * FROM bronze_eps;"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "13d723c1-b399-4079-881b-2d8eecb9f2eb",
"showTitle": false,
"title": ""
}
},
"source": [
"---\n",
"\n",
"Since the silver and gold tables have no rows yet, we will insert the values for now:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "79a4977c-da4e-4283-bb28-892bddc570f8",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>6</td><td>6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
6,
6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 11
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"INSERT INTO silver_eps\n",
"SELECT date, stock_symbol, analyst, estimated_eps\n",
"FROM table_changes('bronze_eps', 1)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "36ae4a7f-dc01-458a-8289-ad89d0c9f4d8",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th></tr></thead><tbody><tr><td>3/1/2021</td><td>b</td></tr><tr><td>3/1/2021</td><td>a</td></tr><tr><td>3/1/2021</td><td>c</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"b"
],
[
"3/1/2021",
"a"
],
[
"3/1/2021",
"c"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 12
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT DISTINCT date, stock_symbol FROM table_changes('silver_eps', 1);"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "9c290386-2793-444c-ac4a-c5dfe99f2215",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>3</td><td>3</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
3,
3
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 13
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"INSERT INTO gold_consensus_eps \n",
"SELECT silver_eps.date, silver_eps.stock_symbol, AVG(estimated_eps) as consensus_eps\n",
"FROM silver_eps\n",
"INNER JOIN (SELECT DISTINCT date, stock_symbol FROM table_changes('silver_eps', 1)) AS silver_cdf\n",
" ON silver_eps.date = silver_cdf.date\n",
" AND silver_eps.stock_symbol = silver_cdf.stock_symbol\n",
"GROUP BY silver_eps.date, silver_eps.stock_symbol"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "9a645d06-d988-4244-9cfb-e4961c1a5911",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>consensus_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>b</td><td>1.25</td></tr><tr><td>3/1/2021</td><td>a</td><td>2.1</td></tr><tr><td>3/1/2021</td><td>c</td><td>3.05</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"b",
1.25
],
[
"3/1/2021",
"a",
2.1
],
[
"3/1/2021",
"c",
3.05
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 14
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "consensus_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM gold_consensus_eps"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "29a92809-2700-4338-a79f-197c7643d296",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": [
"df = spark.createDataFrame(\n",
" [\n",
" ('3/1/2021', 'a', 2, 2.4),\n",
" ('4/1/2021', 'a', 1, 2.3),\n",
" ('4/1/2021', 'a', 2, 2.1),\n",
" ('4/1/2021', 'b', 1, 1.3),\n",
" ('4/1/2021', 'b', 2, 1.2),\n",
" ('4/1/2021', 'c', 1, 3.5),\n",
" ('4/1/2021', 'c', 2, 2.6)\n",
" ],\n",
" ('date', 'stock_symbol', 'analyst', 'estimated_eps')\n",
")\n",
"\n",
"df.createOrReplaceTempView(\"bronze_eps_april_dataset\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "89b49460-91d6-436e-8eed-442343b3f2ba",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>7</td><td>7</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
7,
7
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 16
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"INSERT INTO bronze_eps TABLE bronze_eps_april_dataset"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "e67f3e0e-c07d-4936-b8c9-6f1d0789df82",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th><th>_change_type</th><th>_commit_version</th><th>_commit_timestamp</th></tr></thead><tbody><tr><td>4/1/2021</td><td>a</td><td>2</td><td>2.1</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>4/1/2021</td><td>c</td><td>2</td><td>2.6</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>4/1/2021</td><td>c</td><td>1</td><td>3.5</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>4/1/2021</td><td>b</td><td>2</td><td>1.2</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>4/1/2021</td><td>b</td><td>1</td><td>1.3</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>4/1/2021</td><td>a</td><td>1</td><td>2.3</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.4</td><td>insert</td><td>2</td><td>2024-07-14T16:28:25Z</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"4/1/2021",
"a",
2,
2.1,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"4/1/2021",
"c",
2,
2.6,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"4/1/2021",
"c",
1,
3.5,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"4/1/2021",
"b",
2,
1.2,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"4/1/2021",
"b",
1,
1.3,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"4/1/2021",
"a",
1,
2.3,
"insert",
2,
"2024-07-14T16:28:25Z"
],
[
"3/1/2021",
"a",
2,
2.4,
"insert",
2,
"2024-07-14T16:28:25Z"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 17
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
},
{
"metadata": "{}",
"name": "_change_type",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "_commit_version",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "_commit_timestamp",
"type": "\"timestamp\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM table_changes('bronze_eps', 2)"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "d51cc928-01ed-4f29-8d1f-ebccd1bc09c8",
"showTitle": false,
"title": ""
}
},
"source": [
"Most of the new data is for a new month but as you can see, there is an update too:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "c520cc29-daee-488d-94f7-d9f1248c8063",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_updated_rows</th><th>num_deleted_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>7</td><td>1</td><td>0</td><td>6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
7,
1,
0,
6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 18
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_updated_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_deleted_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"MERGE INTO silver_eps\n",
"USING \n",
" (SELECT * FROM table_changes('bronze_eps', 2))\n",
" AS bronze_cdf\n",
"ON bronze_cdf.date = silver_eps.date AND\n",
" bronze_cdf.stock_symbol = silver_eps.stock_symbol AND\n",
" bronze_cdf.analyst = silver_eps.analyst\n",
"WHEN MATCHED THEN\n",
" UPDATE SET silver_eps.estimated_eps = bronze_cdf.estimated_eps\n",
"WHEN NOT MATCHED\n",
" THEN INSERT (date, stock_symbol, analyst, estimated_eps) VALUES (date, stock_symbol, analyst, estimated_eps)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "ccf40e60-6701-4781-a3d2-b61bce983f3b",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th><th>_change_type</th><th>_commit_version</th><th>_commit_timestamp</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td><td>update_preimage</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.4</td><td>update_postimage</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>a</td><td>2</td><td>2.1</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>c</td><td>2</td><td>2.6</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>c</td><td>1</td><td>3.5</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>b</td><td>2</td><td>1.2</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>b</td><td>1</td><td>1.3</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr><tr><td>4/1/2021</td><td>a</td><td>1</td><td>2.3</td><td>insert</td><td>2</td><td>2024-07-14T16:30:22Z</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
2,
2.0,
"update_preimage",
2,
"2024-07-14T16:30:22Z"
],
[
"3/1/2021",
"a",
2,
2.4,
"update_postimage",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"a",
2,
2.1,
"insert",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"c",
2,
2.6,
"insert",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"c",
1,
3.5,
"insert",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"b",
2,
1.2,
"insert",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"b",
1,
1.3,
"insert",
2,
"2024-07-14T16:30:22Z"
],
[
"4/1/2021",
"a",
1,
2.3,
"insert",
2,
"2024-07-14T16:30:22Z"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 19
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
},
{
"metadata": "{}",
"name": "_change_type",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "_commit_version",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "_commit_timestamp",
"type": "\"timestamp\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM table_changes('silver_eps', 2)"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "64d9c7b8-08fb-4555-a5e4-8abda9a3c809",
"showTitle": false,
"title": ""
}
},
"source": [
"From https://docs.delta.io/2.0.0/delta-change-data-feed.html#change-data-event-schema, \"`preimage` is the value before the update, `postimage` is the value after the update\". So, only one row was updated on the Silver table."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "56b9f909-4522-42a4-b2cb-c647b6cc237d",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td></tr><tr><td>4/1/2021</td><td>a</td></tr><tr><td>4/1/2021</td><td>c</td></tr><tr><td>4/1/2021</td><td>b</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a"
],
[
"4/1/2021",
"a"
],
[
"4/1/2021",
"c"
],
[
"4/1/2021",
"b"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 20
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT DISTINCT date, stock_symbol FROM table_changes('silver_eps', 2)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "a3475c05-e821-4a13-9e84-559bfabc6056",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>num_affected_rows</th><th>num_updated_rows</th><th>num_deleted_rows</th><th>num_inserted_rows</th></tr></thead><tbody><tr><td>4</td><td>1</td><td>0</td><td>3</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
4,
1,
0,
3
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 21
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "num_affected_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_updated_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_deleted_rows",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "num_inserted_rows",
"type": "\"long\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"MERGE INTO gold_consensus_eps \n",
"USING\n",
" (SELECT silver_eps.date, silver_eps.stock_symbol, AVG(estimated_eps) as consensus_eps\n",
" FROM silver_eps\n",
" INNER JOIN (SELECT DISTINCT date, stock_symbol FROM table_changes('silver_eps', 2)) AS silver_cdf\n",
" ON silver_eps.date = silver_cdf.date\n",
" AND silver_eps.stock_symbol = silver_cdf.stock_symbol\n",
" GROUP BY silver_eps.date, silver_eps.stock_symbol) as silver_cdf_agg\n",
"ON silver_cdf_agg.date = gold_consensus_eps.date AND\n",
" silver_cdf_agg.stock_symbol = gold_consensus_eps.stock_symbol\n",
"WHEN MATCHED THEN\n",
" UPDATE SET gold_consensus_eps.consensus_eps = silver_cdf_agg.consensus_eps\n",
"WHEN NOT MATCHED\n",
" THEN INSERT (date, stock_symbol, consensus_eps) VALUES (date, stock_symbol, consensus_eps)"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "ba3b2c74-a54f-4eb4-ae4d-cbbe3aeee9c3",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>consensus_eps</th><th>_change_type</th><th>_commit_version</th><th>_commit_timestamp</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>2.1</td><td>update_preimage</td><td>2</td><td>2024-07-14T16:33:41Z</td></tr><tr><td>3/1/2021</td><td>a</td><td>2.3</td><td>update_postimage</td><td>2</td><td>2024-07-14T16:33:41Z</td></tr><tr><td>4/1/2021</td><td>a</td><td>2.2</td><td>insert</td><td>2</td><td>2024-07-14T16:33:41Z</td></tr><tr><td>4/1/2021</td><td>c</td><td>3.05</td><td>insert</td><td>2</td><td>2024-07-14T16:33:41Z</td></tr><tr><td>4/1/2021</td><td>b</td><td>1.25</td><td>insert</td><td>2</td><td>2024-07-14T16:33:41Z</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
2.1,
"update_preimage",
2,
"2024-07-14T16:33:41Z"
],
[
"3/1/2021",
"a",
2.3,
"update_postimage",
2,
"2024-07-14T16:33:41Z"
],
[
"4/1/2021",
"a",
2.2,
"insert",
2,
"2024-07-14T16:33:41Z"
],
[
"4/1/2021",
"c",
3.05,
"insert",
2,
"2024-07-14T16:33:41Z"
],
[
"4/1/2021",
"b",
1.25,
"insert",
2,
"2024-07-14T16:33:41Z"
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 22
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "consensus_eps",
"type": "\"double\""
},
{
"metadata": "{}",
"name": "_change_type",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "_commit_version",
"type": "\"long\""
},
{
"metadata": "{}",
"name": "_commit_timestamp",
"type": "\"timestamp\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM table_changes('gold_consensus_eps', 2)"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "23f2de55-2a4e-4080-9b2a-9ebc7351f936",
"showTitle": false,
"title": ""
}
},
"source": [
"Inspect versions:"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "55f8d6fa-de0f-48f0-ace4-7b994e384a9c",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>consensus_eps</th></tr></thead><tbody></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 24
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "consensus_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM gold_consensus_eps@v0"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "67daf6c6-6660-43c4-925e-10b4b3b96304",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>consensus_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>b</td><td>1.25</td></tr><tr><td>3/1/2021</td><td>a</td><td>2.1</td></tr><tr><td>3/1/2021</td><td>c</td><td>3.05</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"b",
1.25
],
[
"3/1/2021",
"a",
2.1
],
[
"3/1/2021",
"c",
3.05
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 25
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "consensus_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM gold_consensus_eps@v1"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "5c69cc40-c62e-4494-9379-96903573170b",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>consensus_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>2.3</td></tr><tr><td>4/1/2021</td><td>a</td><td>2.2</td></tr><tr><td>4/1/2021</td><td>c</td><td>3.05</td></tr><tr><td>4/1/2021</td><td>b</td><td>1.25</td></tr><tr><td>3/1/2021</td><td>b</td><td>1.25</td></tr><tr><td>3/1/2021</td><td>c</td><td>3.05</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
2.3
],
[
"4/1/2021",
"a",
2.2
],
[
"4/1/2021",
"c",
3.05
],
[
"4/1/2021",
"b",
1.25
],
[
"3/1/2021",
"b",
1.25
],
[
"3/1/2021",
"c",
3.05
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 26
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "consensus_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM gold_consensus_eps@v2"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "0a4ac7b4-ba15-4314-892c-ef3e5dd3bef4",
"showTitle": false,
"title": ""
}
},
"source": [
"What happened to the Bronze (raw) data?"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "25284656-cbe9-4d82-80b0-36057a9dc8e5",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 27
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM bronze_eps@v0"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "fca550fc-a0ab-418c-a7a4-b48fee3e4a2c",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"a",
2,
2.0
],
[
"3/1/2021",
"b",
1,
1.3
],
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 28
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM bronze_eps@v1"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "037b9904-1433-4cc1-921b-13143ee7244e",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr><tr><td>4/1/2021</td><td>a</td><td>1</td><td>2.3</td></tr><tr><td>4/1/2021</td><td>a</td><td>2</td><td>2.1</td></tr><tr><td>4/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>4/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>4/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>4/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.4</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"a",
2,
2.0
],
[
"3/1/2021",
"b",
1,
1.3
],
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
],
[
"4/1/2021",
"a",
1,
2.3
],
[
"4/1/2021",
"a",
2,
2.1
],
[
"4/1/2021",
"b",
1,
1.3
],
[
"4/1/2021",
"b",
2,
1.2
],
[
"4/1/2021",
"c",
1,
3.5
],
[
"4/1/2021",
"c",
2,
2.6
],
[
"3/1/2021",
"a",
2,
2.4
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 29
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM bronze_eps@v2"
]
},
{
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "e61d4620-f979-4cb7-8ba3-c4ec6ebb8059",
"showTitle": false,
"title": ""
}
},
"source": [
"All the rows were mindlessly inserted, nice."
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "d5372db6-0da0-41c3-98de-9165c2541909",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 30
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM silver_eps@v0"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "4676e095-99d5-4db7-8c7f-1df4bdd2ea60",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.0</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"a",
2,
2.0
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
],
[
"3/1/2021",
"b",
1,
1.3
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 31
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM silver_eps@v1"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"implicitDf": true,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "827458d3-d158-4022-812d-ef32599ab72b",
"showTitle": false,
"title": ""
}
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/html": [
"<style scoped>\n",
" .table-result-container {\n",
" max-height: 300px;\n",
" overflow: auto;\n",
" }\n",
" table, th, td {\n",
" border: 1px solid black;\n",
" border-collapse: collapse;\n",
" }\n",
" th, td {\n",
" padding: 5px;\n",
" }\n",
" th {\n",
" text-align: left;\n",
" }\n",
"</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>date</th><th>stock_symbol</th><th>analyst</th><th>estimated_eps</th></tr></thead><tbody><tr><td>4/1/2021</td><td>a</td><td>2</td><td>2.1</td></tr><tr><td>4/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr><tr><td>4/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>4/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>4/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr><tr><td>4/1/2021</td><td>a</td><td>1</td><td>2.3</td></tr><tr><td>3/1/2021</td><td>a</td><td>2</td><td>2.4</td></tr><tr><td>3/1/2021</td><td>b</td><td>2</td><td>1.2</td></tr><tr><td>3/1/2021</td><td>a</td><td>1</td><td>2.2</td></tr><tr><td>3/1/2021</td><td>c</td><td>1</td><td>3.5</td></tr><tr><td>3/1/2021</td><td>c</td><td>2</td><td>2.6</td></tr><tr><td>3/1/2021</td><td>b</td><td>1</td><td>1.3</td></tr></tbody></table></div>"
]
},
"metadata": {
"application/vnd.databricks.v1+output": {
"addedWidgets": {},
"aggData": [],
"aggError": "",
"aggOverflow": false,
"aggSchema": [],
"aggSeriesLimitReached": false,
"aggType": "",
"arguments": {},
"columnCustomDisplayInfos": {},
"data": [
[
"4/1/2021",
"a",
2,
2.1
],
[
"4/1/2021",
"c",
2,
2.6
],
[
"4/1/2021",
"c",
1,
3.5
],
[
"4/1/2021",
"b",
2,
1.2
],
[
"4/1/2021",
"b",
1,
1.3
],
[
"4/1/2021",
"a",
1,
2.3
],
[
"3/1/2021",
"a",
2,
2.4
],
[
"3/1/2021",
"b",
2,
1.2
],
[
"3/1/2021",
"a",
1,
2.2
],
[
"3/1/2021",
"c",
1,
3.5
],
[
"3/1/2021",
"c",
2,
2.6
],
[
"3/1/2021",
"b",
1,
1.3
]
],
"datasetInfos": [],
"dbfsResultPath": null,
"isJsonSchema": true,
"metadata": {
"dataframeName": "_sqldf",
"executionCount": 32
},
"overflow": false,
"plotOptions": {
"customPlotOptions": {},
"displayType": "table",
"pivotAggregation": null,
"pivotColumns": null,
"xColumns": null,
"yColumns": null
},
"removedWidgets": [],
"schema": [
{
"metadata": "{}",
"name": "date",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "stock_symbol",
"type": "\"string\""
},
{
"metadata": "{}",
"name": "analyst",
"type": "\"integer\""
},
{
"metadata": "{}",
"name": "estimated_eps",
"type": "\"double\""
}
],
"type": "table"
}
},
"output_type": "display_data"
}
],
"source": [
"%sql\n",
"SELECT * FROM silver_eps@v2"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"inputWidgets": {},
"nuid": "de2412ed-9bec-4206-aa8d-a37f53c5d0bf",
"showTitle": false,
"title": ""
}
},
"outputs": [],
"source": []
}
],
"metadata": {
"application/vnd.databricks.v1+notebook": {
"dashboards": [],
"environmentMetadata": null,
"language": "python",
"notebookMetadata": {
"mostRecentlyExecutedCommandWithImplicitDF": {
"commandId": 4116221871035421,
"dataframes": [
"_sqldf"
]
},
"pythonIndentUnit": 4
},
"notebookName": "Experiments with Delta CDF",
"widgets": {}
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment