LogsSchema.scala

package microsoft.opentelemetrycollector.otlp.json.v1

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.StructType

// @formatter:off
/** Represents a schema for a Logs payload.
  *
  * @param spark
  *   The SparkSession for schema inference.
  *
  * The source of truth is stored in the OpenTelemetry Protocol (OTLP) specification:
  *
  *  >>> Protobuf: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/logs/v1/logs.proto
  *  >>> Sample: https://github.com/open-telemetry/opentelemetry-proto/blob/main/examples/logs.json
  */
// @formatter:on
class LogsSchema(spark: SparkSession) {

  /** Sample data.
    */
  private val Sample = Seq(
    ("""{
           "resourceLogs":[
              {
                 "resource":{
                    "attributes":[
                       {
                          "key":"service.name",
                          "value":{
                             "stringValue":"my.service"
                          }
                       }
                    ]
                 },
                 "scopeLogs":[
                    {
                       "scope":{
                          "name":"my.library",
                          "version":"1.0.0",
                          "attributes":[
                             {
                                "key":"my.scope.attribute",
                                "value":{
                                   "stringValue":"some scope attribute"
                                }
                             }
                          ]
                       },
                       "logRecords":[
                          {
                             "timeUnixNano":"1544712660300000000",
                             "observedTimeUnixNano":"1544712660300000000",
                             "severityNumber":10,
                             "severityText":"Information",
                             "traceId":"5B8EFFF798038103D269B633813FC60C",
                             "spanId":"EEE19B7EC3C1B174",
                             "body":{
                                "stringValue":"Example log record"
                             },
                             "attributes":[
                                {
                                   "key":"string.attribute",
                                   "value":{
                                      "stringValue":"some string"
                                   }
                                },
                                {
                                   "key":"boolean.attribute",
                                   "value":{
                                      "boolValue":true
                                   }
                                },
                                {
                                   "key":"int.attribute",
                                   "value":{
                                      "intValue":"10"
                                   }
                                },
                                {
                                   "key":"double.attribute",
                                   "value":{
                                      "doubleValue":637.704
                                   }
                                },
                                {
                                   "key":"array.attribute",
                                   "value":{
                                      "arrayValue":{
                                         "values":[
                                            {
                                               "stringValue":"many"
                                            },
                                            {
                                               "stringValue":"values"
                                            }
                                         ]
                                      }
                                   }
                                },
                                {
                                   "key":"map.attribute",
                                   "value":{
                                      "kvlistValue":{
                                         "values":[
                                            {
                                               "key":"some.map.key",
                                               "value":{
                                                  "stringValue":"some value"
                                               }
                                            }
                                         ]
                                      }
                                   }
                                }
                             ]
                          }
                       ]
                    }
                 ]
              }
           ]
        }
    """)
  )

  /** @inheritdoc
    */
  override val schema = {
    import spark.implicits._
    StructType(spark.read.json(Sample.toDS).schema)
  }
}

/** Companion object for [[LogsSchema]].
  */
object LogsSchema {

  /** Constructor.
    *
    * @param spark
    *   The SparkSession for schema inference.
    */
  def apply(spark: SparkSession): LogsSchema =
    new LogsSchema(spark)
}

Schema in Spark:

root
 |-- parsed: struct (nullable = true)
 |    |-- resourceLogs: array (nullable = true)
 |    |    |-- element: struct (containsNull = true)
 |    |    |    |-- resource: struct (nullable = true)
 |    |    |    |    |-- attributes: array (nullable = true)
 |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |-- key: string (nullable = true)
 |    |    |    |    |    |    |-- value: struct (nullable = true)
 |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |-- scopeLogs: array (nullable = true)
 |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |-- logRecords: array (nullable = true)
 |    |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |    |-- attributes: array (nullable = true)
 |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |    |    |    |-- key: string (nullable = true)
 |    |    |    |    |    |    |    |    |    |-- value: struct (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- arrayValue: struct (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |-- values: array (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- boolValue: boolean (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- doubleValue: double (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- intValue: string (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- kvlistValue: struct (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |-- values: array (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |    |-- key: string (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |    |-- value: struct (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |    |    |    |    |-- body: struct (nullable = true)
 |    |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |    |    |    |    |-- observedTimeUnixNano: string (nullable = true)
 |    |    |    |    |    |    |    |-- severityNumber: long (nullable = true)
 |    |    |    |    |    |    |    |-- severityText: string (nullable = true)
 |    |    |    |    |    |    |    |-- spanId: string (nullable = true)
 |    |    |    |    |    |    |    |-- timeUnixNano: string (nullable = true)
 |    |    |    |    |    |    |    |-- traceId: string (nullable = true)
 |    |    |    |    |    |-- scope: struct (nullable = true)
 |    |    |    |    |    |    |-- attributes: array (nullable = true)
 |    |    |    |    |    |    |    |-- element: struct (containsNull = true)
 |    |    |    |    |    |    |    |    |-- key: string (nullable = true)
 |    |    |    |    |    |    |    |    |-- value: struct (nullable = true)
 |    |    |    |    |    |    |    |    |    |-- stringValue: string (nullable = true)
 |    |    |    |    |    |    |-- name: string (nullable = true)
 |    |    |    |    |    |    |-- version: string (nullable = true)