rnbtechnology’s gists

rnbtechnology / sample_delete_record.json

Created February 18, 2021 00:54

GoldenGate Sample Records

rnbtechnology / glue_hudi_0.5.3_bundle_deletions_pom.xml

Created January 14, 2021 19:42

<include>org.apache.httpcomponents:httpclient</include>

rnbtechnology / glue_hudi_0.5.3_bundle_additions_pom.xml

Created January 14, 2021 19:40

	<relocation>
	<pattern>org.eclipse.jetty.</pattern>
	<shadedPattern>org.apache.hudi.org.eclipse.jetty.</shadedPattern>
	</relocation>

rnbtechnology / glue_hudi_0.5.3_bundle_pom.xml

Created January 14, 2021 19:22

	<?xml version="1.0" encoding="UTF-8"?>
	<!--
	Licensed to the Apache Software Foundation (ASF) under one or more
	contributor license agreements. See the NOTICE file distributed with
	this work for additional information regarding copyright ownership.
	The ASF licenses this file to You under the Apache License, Version 2.0
	(the "License"); you may not use this file except in compliance with
	the License. You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

rnbtechnology / write_to_s3.py

Last active November 8, 2021 08:39

	import copy

	# write to a path using the Hudi format
	def hudi_write(df, schema, table, path, mode, hudi_options):
	hudi_options = {
	"hoodie.datasource.write.recordkey.field": "recordkey",
	"hoodie.datasource.write.precombine.field": "precombine_field",
	"hoodie.datasource.write.partitionpath.field": "partitionpath_field",
	"hoodie.datasource.write.operation": "write_operaion",
	"hoodie.datasource.write.table.type": "table_type",

rnbtechnology / deserialize_kafka.py

Last active January 7, 2021 23:29

	from pyspark import SparkContext
	from pyspark.sql.column import Column, _to_java_column
	from pyspark.sql.functions import col

	# instantiates a Scala Map containing configurations for communicating with Schema Regsitry APIs
	def get_schema_registry_conf_map(spark, schema_registry_url, topic_name):
	sc = spark.SparkContext
	jvm_gateway = sc._gateway.jvm

	schema_registry_config_dict = {

rnbtechnology / read_from_kafka.py

Last active January 7, 2021 23:30

	TOPIC_NAME = "topic_name"
	KAFKA_BOOTSTRAP_SERVERS = "host1:port1,host2:port2"

	# read data from Kafka
	df = (
	spark.readStream.format("kafka")
	.option("kafka.bootstrap.servers", KAFKA_BOOTSTRAP_SERVERS)
	.option("subscribe", TOPIC_NAME)
	.load()
	)

rnbtechnology / sample_delete_record.json

Created January 3, 2021 21:34

rnbtechnology / sample_update_record.json

Created January 3, 2021 21:33

rnbtechnology / sample_insert_record.json

Created January 3, 2021 21:31