Mako Mori

gistfile1.txt

---
dashboards:
  - name: R Dev
    index_name: c-t-dev-1
  - name: P Demo
    index_name: c-t-demo-7
  - name: P Prod
    index_name: c-t-prod-1
  - name: P Stage
    index_name: c-t-stage-0
  - name: P Validation
    index_name: c-t-validation-1
  - name: Customer1
    index_name: customer1-t-customer1-1
  - name: Customer2
    index_name: customer2-t-customer2-1

visualizations:
  - name: telemetry-pg-batch-size
    es_query: telemetry_postgres_batch_size
    description: Telemetry Postgres Batch Size
  - name: telemetry-pg-message-rate
    es_query: telemetry_postgres_message_rate
    description: Telemetry Postgres Message Rate
  - name: influxdb-response-time
    es_query: influxdb_response_time
    description: InfluxDB response times

searches:
  - name: telemetry-postgres
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: telemetry

  - name: trp-control
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: trpcontrol

  - name: nats-streaming
    matchers:
      - field: kubernetes.labels.io-cattle-field/appId
        value: nats-streaming

  - name: nats-js-stream
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: nats-js-stream

  - name: data-api
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: t-data-api
    must_not:
      - field: log
        value: ELB-HealthChecker/2.0

  - name: grafana
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: grafana

  - name: influxdb
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: influxdb

  - name: insight-operator
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: insight-operator

  - name: timescale
    matchers:
      - field: kubernetes.labels.io-cattle-field/appId
        value: timescale

  - name: mqtt
    matchers:
      - field: kubernetes.labels.app-kubernetes-io/name
        value: t-mosquitto

  - name: jhub
    matchers:
      - field: kubernetes.labels.app
        value: jupyterhub
    must_not:
      - field: log
        value: healthz

  - name: nats
    matchers:
      - field: kubernetes.labels.app
        value: nats

make_objects.py

#!/usr/bin/env python3
import json
import yaml
import time
import requests
from pprint import PrettyPrinter
from jinja2 import Environment, FileSystemLoader

base_url = "https://kibana.fqdn.com/api"
headers = {'kbn-xsrf': "true", 'Content-Type': 'application/json'}


class KibanaDashboard:
    def __init__(self, dashboard_config, config):
        self.config = config
        self.dashboard_config = dashboard_config

        self.env = Environment(loader=FileSystemLoader('./timelion_expressions/'))

        self.searches = self.config["searches"]
        self.p = PrettyPrinter(indent=4)
        self.thing_cache = {}

    def find_dashboard(self, dashboard_name):
        check = requests.get(
            f"{base_url}/saved_objects/_find?type=dashboard&search_fields=title&search={dashboard_name}",
            headers=headers)
        res = json.loads(check.text)

        for saved_object in res["saved_objects"]:
            # self.p.pprint(saved_object["attributes"]["title"])
            if dashboard_name == saved_object["attributes"]["title"]:
                return saved_object

        return False

    def find_thing(self, thing_type, name, search_field="title"):
        if type not in self.thing_cache:
            self.thing_cache[thing_type] = {}

        if name in self.thing_cache[thing_type]:
            return self.thing_cache[thing_type][name]

        url = f"{base_url}/saved_objects/_find?type={thing_type}&search_fields={search_field}&search={name}"
        print(f"URL: {url}")
        check = requests.get(url, headers=headers)
        res = json.loads(check.text)

        for saved_object in res["saved_objects"]:
            # self.p.pprint(saved_object["attributes"]["title"])
            if name == saved_object["attributes"]["title"]:
                self.thing_cache[thing_type][name] = saved_object
                return saved_object

        return False

    def find_viz(self, title):
        return self.find_thing("visualization", title)

    def find_index(self, index_name):
        check = requests.get(
            f"{base_url}/saved_objects/_find?type=index-pattern&search_fields=title&search={index_name}",
            headers=headers)
        res = json.loads(check.text)

        for saved_object in res["saved_objects"]:
            # self.p.pprint(saved_object["attributes"]["title"])
            if f"{index_name}*" == saved_object["attributes"]["title"]:
                return saved_object

        return False

    def find_search(self, title):
        check = requests.get(f"{base_url}/saved_objects/_find?type=search&search_fields=title&search={title}",
                             headers=headers)
        res = json.loads(check.text)

        for saved_object in res["saved_objects"]:
            if title == saved_object["attributes"]["title"]:
                return saved_object

        return False

    def compile(self):
        self.make_searches()
        self.make_dashboard(
            title=self.dashboard_config["name"],
            things=self.config["searches"],
            thing_type="search",
            description="Logs from all services")

        self.make_visualizations()
        self.make_dashboard(
            title=f"{self.dashboard_config['name']} - Telemetry",
            things=self.config["visualizations"],
            thing_type="visualization",
            description="Metrics on telemetry and influx")

    def make_dashboard(self, **kwargs):
        things = kwargs["things"]
        thing_type = kwargs["thing_type"]
        description = kwargs["description"]
        dashboard_title = kwargs["title"]

        width = 24
        height = 15

        panel_ref_id = 0
        panel_x = 0
        panel_y = 0

        dashboard = {
            "attributes": {
                "description": description,
                "hits": 0,
                "optionsJSON": '{"hidePanelTitles":false,"useMargins":true}',
                "panelsJSON": "",
                "timeRestore": False,
                "title": dashboard_title,
                "version": 1
            },
            "references": []
        }

        panels = []

        for thing in things:
            title = self.saved_search_title(thing["name"])
            # if thing_type == "search":
            # saved_thing = self.find_thing(title)
            # elif thing_type == "visualization":
            # saved_thing = self.find_viz(title)
            # else:
            # raise Exception(f"Unknown type of thing: {thing_type}")
            saved_thing = self.find_thing(thing_type, title)

            if not saved_thing:
                print(f"Unable to find thing: {title} / {thing_type}")
                continue

            panel = {
                "embeddableConfig": {},
                "gridData": {
                    "h": 15,
                    "i": saved_thing["id"],
                    "w": 24,
                    "x": panel_x,
                    "y": panel_y
                },
                "panelIndex": saved_thing["id"],
                "panelRefName": f"panel_{panel_ref_id}",
                "version": "7.9.2"
            }

            ref = {
                "id": saved_thing["id"],
                "name": f"panel_{panel_ref_id}",
                "type": thing_type
            }

            panels.append(panel)
            dashboard["references"].append(ref)

            panel_ref_id += 1
            # print(f"{title} / {panel_ref_id} / {panel_x} x {panel_y} / {panel_ref_id % 2}")

            if panel_ref_id % 2 == 0:
                panel_y += height
                panel_x = 0
            else:
                panel_x += width

        dashboard["attributes"]["panelsJSON"] = json.dumps(panels)

        existing = self.find_thing("dashboard", dashboard_title)

        if existing:
            url = f"{base_url}/saved_objects/dashboard/{existing['id']}"
            print(f"Updating dashboard: {dashboard_title}")
            r = requests.put(url, headers=headers, data=json.dumps(dashboard))
            print(r.status_code)
        else:
            url = f"{base_url}/saved_objects/dashboard"
            print(f"Creating dashboard: {dashboard_title}")
            r = requests.post(url, headers=headers, data=json.dumps(dashboard))
            print(r.status_code)

    def viz_title(self, name):
        return f"{self.dashboard_config['index_name']}-{name}"

    def saved_search_title(self, name):
        return f"{self.dashboard_config['index_name']}-{name}"

    def make_visualizations(self):
        for viz_config in self.config["visualizations"]:
            viz_title = self.viz_title(viz_config["name"])
            v = self.find_viz(viz_title)

            vis = json.loads(open("vis.json").read())
            vis_state = json.loads(open("vis_state.json").read())

            t = self.env.get_template(f"{viz_config['es_query']}.j2")

            rendered = t.render({"index_name": f"{self.dashboard_config['index_name']}*"})

            vis_state["params"]["expression"] = rendered
            vis_state["title"] = viz_title

            vis["attributes"]["visState"] = json.dumps(vis_state)
            vis["attributes"]["title"] = viz_title

            if not v:
                print(f"Creating viz: {viz_title}")
                url = f"{base_url}/saved_objects/visualization"
                r = requests.post(url, headers=headers, data=json.dumps(vis))
                if r.status_code == 200:
                    print(f"Create complete: {r.status_code}")
                else:
                    raise Exception(r.text)

            else:
                print(f"Updating viz: {viz_title}")
                url = f"{base_url}/saved_objects/visualization/{v['id']}"
                r = requests.put(url, headers=headers, data=json.dumps(vis))
                if r.status_code == 200:
                    print(f"Update complete: {r.status_code}")
                else:
                    raise Exception(r.text)

        return True

    def make_searches(self):
        for search in self.searches:
            search_js = json.loads(open("search.json").read())
            search_source_js = json.loads(open("search_source.json").read())

            title = self.saved_search_title(search["name"])
            print(f"Starting search: {title}")

            saved_search = self.find_search(title)
            es_index_id = self.find_index(self.dashboard_config["index_name"])["id"]

            search_js["attributes"]["title"] = title
            search_js["attributes"]["description"] = title

            for i, ref in enumerate(search_js["references"]):
                search_js["references"][i]["id"] = es_index_id

            for matcher in search["matchers"]:
                search_source_js["filter"][0]["meta"]["key"] = matcher['field']
                search_source_js["filter"][0]["meta"]["params"]["query"] = matcher["value"]

                search_source_js["filter"][0]["query"]["bool"]["filter"]["match_phrase"] = {
                    matcher["field"]: matcher["value"]}

            if "must_not" in search:
                search_source_js["filter"][0]["query"]["bool"]["must_not"] = []
                for must_not in search["must_not"]:
                    search_source_js["filter"][0]["query"]["bool"]["must_not"].append({
                        "match_phrase": {
                            must_not["field"]: must_not["value"]
                        }
                    })

            search_js["attributes"]["kibanaSavedObjectMeta"]["searchSourceJSON"] = json.dumps(search_source_js)

            if saved_search:
                print(f"Updating search: {title}")
                url = f"{base_url}/saved_objects/search/{saved_search['id']}"
                r = requests.put(url, headers=headers, data=json.dumps(search_js))
                print(f"Update complete: {r.status_code}")
            else:
                print(f"Creating search: {title}")
                url = f"{base_url}/saved_objects/search"
                r = requests.post(url, headers=headers, data=json.dumps(search_js))
                print(f"Create complete: {r.status_code}")

            time.sleep(5)


if __name__ == "__main__":
    config = yaml.safe_load(open("etc/searches.yaml").read())

    for dashboard in config["dashboards"]:
        kd = KibanaDashboard(dashboard, config)
        # dash = kd.find_dashboard("TRP Demo - SRE")
        # kd.p.pprint(dash)
        kd.compile()
        # v = kd.find_viz("Telemetry - Postgres - Batch Size")
        # kd.p.pprint(v)
        # print(json.dumps(json.loads(v["attributes"]["visState"])))

search.json

{
  "attributes": {
    "title": "",
    "description": "",
    "hits": 0,
    "columns": [
      "log"
    ],
    "sort": [],
    "version": 1,
    "kibanaSavedObjectMeta": {
      "searchSourceJSON": ""
    }
  },
  "references": [
    {
      "name": "kibanaSavedObjectMeta.searchSourceJSON.index",
      "type": "index-pattern",
      "id": "b8fa11e0-1887-11eb-95a5-a76632cb0428"
    },
    {
      "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
      "type": "index-pattern",
      "id": "b8fa11e0-1887-11eb-95a5-a76632cb0428"
    }
  ]
}

search_source.json

{
  "highlightAll": true,
  "version": true,
  "filter": [
    {
      "meta": {
        "alias": null,
        "negate": false,
        "disabled": false,
        "type": "phrase",
        "key": "kubernetes.labels.app_kubernetes_io/name",
        "params": {
          "query": "appcontrol"
        },
        "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index"
      },
      "query": {
        "bool": {
          "filter": {
            "match_phrase": {
              "kubernetes.labels.app_kubernetes_io/name": "appcontrol"
            }
          }
        }
      },
      "$state": {
        "store": "appState"
      }
    }
  ],
  "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index"
}

vis.json

{
  "attributes": {
    "title": "",
    "visState": "{}",
    "uiStateJSON": "{}",
    "description": "",
    "kibanaSavedObjectMeta": {
      "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
    }
  },
  "references": []
}

vis_state.json

{
  "title": "",
  "type": "timelion",
  "aggs": [],
  "params": {
    "expression": "timelion",
    "interval": "auto"
  }
}