Alloy log

gistfile1.txt

ts=2025-01-30T08:59:35.298859187Z level=info "boringcrypto enabled"=false
ts=2025-01-30T08:59:35.29786316Z level=info source=/go/pkg/mod/github.com/!kim!machine!gun/[email protected]/memlimit/memlimit.go:170 msg="memory is not limited, skipping" package=github.com/KimMachineGun/automemlimit/memlimit
ts=2025-01-30T08:59:35.298890774Z level=info msg="no peer discovery configured: both join and discover peers are empty" service=cluster
ts=2025-01-30T08:59:35.298894656Z level=info msg="running usage stats reporter"
ts=2025-01-30T08:59:35.298896901Z level=warn msg="this stdlib function is deprecated; please refer to the documentation for updated usage and alternatives" controller_path=/ controller_id="" function=env
ts=2025-01-30T08:59:35.2989009Z level=warn msg="this stdlib function is deprecated; please refer to the documentation for updated usage and alternatives" controller_path=/ controller_id="" function=env
ts=2025-01-30T08:59:35.298903588Z level=info msg="starting complete graph evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0
ts=2025-01-30T08:59:35.298912352Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=logging duration=76.16µs
ts=2025-01-30T08:59:35.298919966Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otel duration=1.255µs
ts=2025-01-30T08:59:35.299309243Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.write.hm_loki duration=383.836µs
ts=2025-01-30T08:59:35.2994654Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.process.pod_logs duration=145.694µs
ts=2025-01-30T08:59:35.29951065Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.process.cluster_events duration=36.978µs
ts=2025-01-30T08:59:35.299656903Z level=info msg="Using pod service account via in-cluster config" component_path=/ component_id=loki.source.kubernetes_events.cluster_events
ts=2025-01-30T08:59:35.299665507Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.source.kubernetes_events.cluster_events duration=147.329µs
ts=2025-01-30T08:59:35.299763755Z level=info msg="Using pod service account via in-cluster config" component_path=/ component_id=discovery.kubernetes.pod
ts=2025-01-30T08:59:35.299966203Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=discovery.kubernetes.pod duration=295.049µs
ts=2025-01-30T08:59:35.300131666Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=discovery.relabel.pod_logs duration=158.558µs
ts=2025-01-30T08:59:35.300230347Z level=info msg="Using pod service account via in-cluster config" component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:35.300328487Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.source.kubernetes.pod_logs duration=189.478µs
ts=2025-01-30T08:59:35.300347944Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=tracing duration=11.455µs
ts=2025-01-30T08:59:35.300380505Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=remotecfg duration=26.458µs
ts=2025-01-30T08:59:35.30039233Z level=info msg="applying non-TLS config to HTTP server" service=http
ts=2025-01-30T08:59:35.300396282Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=http duration=8.966µs
ts=2025-01-30T08:59:35.3004122Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=cluster duration=5.482µs
ts=2025-01-30T08:59:35.300425281Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=labelstore duration=6.075µs
ts=2025-01-30T08:59:35.300623136Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.exporter.otlp.hm_tempo duration=193.605µs
ts=2025-01-30T08:59:35.30074566Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.processor.batch.default duration=114.751µs
ts=2025-01-30T08:59:35.300821231Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.connector.servicegraph.default duration=61.09µs
ts=2025-01-30T08:59:35.300860986Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.connector.spanlogs.default duration=32.767µs
ts=2025-01-30T08:59:35.300952352Z level=info msg="Building spanmetrics connector" component_path=/ component_id=otelcol.connector.spanmetrics.default
ts=2025-01-30T08:59:35.300967247Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.connector.spanmetrics.default duration=96.958µs
ts=2025-01-30T08:59:35.30116053Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=otelcol.receiver.otlp.default duration=187.624µs
ts=2025-01-30T08:59:35.301222795Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=beyla.ebpf.default duration=51.882µs
ts=2025-01-30T08:59:35.301265382Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=local.file_match.node_logs duration=37.234µs
ts=2025-01-30T08:59:35.301393519Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=loki.source.file.node_logs duration=122.646µs
ts=2025-01-30T08:59:35.301406227Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=livedebugging duration=6.872µs
ts=2025-01-30T08:59:35.301428598Z level=info msg="finished node evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 node_id=ui duration=17.376µs
ts=2025-01-30T08:59:35.301436486Z level=info msg="finished complete graph evaluation" controller_path=/ controller_id="" trace_id=2d094aad48c2885b1b91a43de8db45d0 duration=2.819459ms
ts=2025-01-30T08:59:35.301502606Z level=info msg="scheduling loaded components and services"
ts=2025-01-30T08:59:35.301615366Z level=info msg="Starting spanmetrics connector" component_path=/ component_id=otelcol.connector.spanmetrics.default
ts=2025-01-30T08:59:35.301728382Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks." component_path=/ component_id=otelcol.receiver.otlp.default documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2025-01-30T08:59:35.302269129Z level=info msg="Starting GRPC server" component_path=/ component_id=otelcol.receiver.otlp.default endpoint=0.0.0.0:4317
ts=2025-01-30T08:59:35.302312936Z level=info msg="starting cluster node" service=cluster peers_count=0 peers="" advertise_addr=127.0.0.1:12345
ts=2025-01-30T08:59:35.302725561Z level=info msg="peers changed" service=cluster peers_count=1 peers=hm-alloy-swpk5
ts=2025-01-30T08:59:35.302755826Z level=info msg="Started servicegraphconnector" component_path=/ component_id=otelcol.connector.servicegraph.default
ts=2025-01-30T08:59:35.303141491Z level=info msg="watching events for namespace" component_path=/ component_id=loki.source.kubernetes_events.cluster_events namespace=""
ts=2025-01-30T08:59:35.303374924Z level=info msg="now listening for http traffic" service=http addr=0.0.0.0:12345
ts=2025-01-30T08:59:35.303393072Z level=warn msg="Using the 0.0.0.0 address exposes this server to every network interface, which may facilitate Denial of Service attacks." component_path=/ component_id=otelcol.receiver.otlp.default documentation=https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security-best-practices.md#safeguards-against-denial-of-service-attacks
ts=2025-01-30T08:59:35.303531061Z level=info msg="Starting HTTP server" component_path=/ component_id=otelcol.receiver.otlp.default endpoint=0.0.0.0:4318
2025/01/30 08:59:35 INFO got host ID component=ContextInfo func=fetchHostID fetcher=AWS hostID=i-0e4edfe2095ec1a22
2025/01/30 08:59:35 INFO starting Beyla in Application Observability mode
2025/01/30 08:59:35 INFO trace context propagation disabled due to missing capability CAP_SYS_ADMIN component=go.Tracer
2025/01/30 08:59:35 INFO using hostname component=traces.ReadDecorator function=instance_ID_hostNamePIDDecorator hostname=hm-alloy-swpk5
2025/01/30 08:59:35 ERROR Unable to load eBPF watcher for process events component=discover.ProcessWatcher interval=5s error="loading and assigning BPF objects: field BeylaKprobeSysBind: program beyla_kprobe_sys_bind: map watch_events: map create: operation not permitted (MEMLOCK may be too low, consider rlimit.RemoveMemlock)"
2025/01/30 08:59:37 WARN can't fetch Kubernetes Cluster Name. Network metrics won't contain k8s.cluster.name attribute unless you explicitly set  the BEYLA_KUBE_CLUSTER_NAME environment variable component=transform.KubernetesDecorator func=KubeClusterName
2025/01/30 08:59:37 INFO Starting main node component=beyla.Instrumenter
ts=2025-01-30T08:59:40.31282272Z level=info msg="Using pod service account via in-cluster config" component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318262297Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-s3 component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318442074Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318525918Z level=info msg="tailer running" target=kube-system/ebs-csi-node-ndqx5:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318578026Z level=info msg="tailer running" target=kube-system/aws-node-b42q9:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318592194Z level=info msg="tailer running" target=production-hm-loki/loki-backend-2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318532183Z level=info msg="tailer running" target=kube-system/aws-node-2sxjs:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318642268Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-distributor-6f579f694c-665x8:distributor component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318777276Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-notifications-controller-785c45558f79w49:notifications-controller component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318787695Z level=info msg="tailer running" target=kube-system/s3-csi-node-49gzb:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318783347Z level=info msg="tailer running" target=production-hm-loki/loki-backend-0:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318790003Z level=info msg="tailer running" target=kube-system/aws-node-nrtx6:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318838278Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318879881Z level=info msg="tailer running" target=production-hm-loki/loki-canary-7cklq:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318863015Z level=info msg="tailer running" target=kube-system/aws-node-b42q9:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318923592Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318946018Z level=info msg="tailer running" target=production-hm-loki/loki-canary-nx89v:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318947687Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-fjkwv:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318921792Z level=info msg="tailer running" target=kube-system/aws-node-f886f:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31898527Z level=info msg="tailer running" target=kube-system/aws-node-2q5gn:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.318994611Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-ingester-0:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319003102Z level=info msg="tailer running" target=kube-system/aws-node-n7jvf:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31900432Z level=info msg="tailer running" target=kube-system/s3-csi-node-74jn5:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319053199Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:upgrade-db-auth component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319093012Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319123666Z level=info msg="tailer running" target=kube-system/kube-proxy-4txs8:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319128277Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319151587Z level=info msg="tailer running" target=kube-system/aws-node-htfw5:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319166009Z level=info msg="tailer running" target=production-hm-opencost/hm-opencost-8586b654fd-ls5zw:hm-opencost component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319186743Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319192305Z level=info msg="tailer running" target=production-hm-valkey/hm-valkey-replicas-1:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319221689Z level=info msg="tailer running" target=kube-system/s3-csi-node-c5jv6:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319245512Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-ww74b:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319242045Z level=info msg="tailer running" target=kube-system/aws-node-v6f6g:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31926335Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319281307Z level=info msg="tailer running" target=kube-system/aws-node-4fwp7:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319287155Z level=info msg="tailer running" target=kube-system/ebs-csi-node-brxpg:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319295939Z level=info msg="tailer running" target=development-hm-open-webui/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31931576Z level=info msg="tailer running" target=kube-system/s3-csi-node-q64sq:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319337109Z level=info msg="tailer running" target=kube-system/aws-node-v6f6g:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319347918Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319336796Z level=info msg="tailer running" target=kube-system/aws-node-v6f6g:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319356014Z level=info msg="tailer running" target=kube-system/ebs-csi-node-k5jjz:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31938479Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-compactor-77b694d464-qptdc:compactor component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319393778Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xmp5r:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319405331Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-querier-75fb7d4b96-2nbw8:querier component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319429533Z level=info msg="tailer running" target=kube-system/s3-csi-node-s57m2:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319428557Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-application-controller-0:application-controller component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319452026Z level=info msg="tailer running" target=kube-system/s3-csi-node-s57m2:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319467808Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319483825Z level=info msg="tailer running" target=production-hm-open-webui-united-airlines/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319499996Z level=info msg="tailer running" target=production-hm-loki/loki-canary-wf2sv:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319527086Z level=info msg="tailer running" target=kube-system/s3-csi-node-p8l8c:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319554911Z level=info msg="tailer running" target=kube-system/aws-node-g7vzz:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319562627Z level=info msg="tailer running" target=kube-system/aws-node-n7jvf:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319530621Z level=info msg="tailer running" target=kube-system/s3-csi-node-vhn8s:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319597092Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-pod-sweeper-pod-sweeper-765ffd4d5c-4wj22:airbyte-pod-sweeper component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319605286Z level=info msg="tailer running" target=kube-system/aws-node-qkf4l:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319608581Z level=info msg="tailer running" target=kube-system/aws-node-n7jvf:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319650757Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-xsw6t:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319647451Z level=info msg="tailer running" target=kube-system/kube-proxy-dgd84:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319666134Z level=info msg="tailer running" target=kube-system/ebs-csi-node-k5jjz:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319666632Z level=info msg="tailer running" target=production-hm-loki/loki-backend-0:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31969019Z level=info msg="tailer running" target=kube-system/kube-proxy-rfksb:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319695006Z level=info msg="tailer running" target=kube-system/s3-csi-node-k22qf:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319712547Z level=info msg="tailer running" target=production-hm-loki/hm-loki-gateway-84cc597d99-cbhfx:nginx component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319722412Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xhgwd:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319730822Z level=info msg="tailer running" target=production-hm-akhq/hm-akhq-6fd5579f4f-ghgkg:akhq component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319751195Z level=info msg="tailer running" target=kube-system/aws-node-2sxjs:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31975656Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319755963Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-worker-849b8f7886-v4kxb:airbyte-worker-container component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319784646Z level=info msg="tailer running" target=kube-system/aws-node-g7vzz:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319792802Z level=info msg="tailer running" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319801175Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31980422Z level=info msg="tailer running" target=kube-system/ebs-csi-node-m6dgw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319838931Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xhgwd:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319862215Z level=info msg="tailer running" target=kube-system/ebs-csi-node-wrlgw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319874239Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:upgrade-db-auth component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31987572Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-attacher component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319910107Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-memcached-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319911624Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.31993671Z level=info msg="tailer running" target=kube-system/s3-csi-node-p8l8c:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319937619Z level=info msg="tailer running" target=kube-system/ebs-csi-node-ndqx5:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.319986245Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-attacher component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320014265Z level=info msg="tailer running" target=production-hm-redpanda-console/hm-redpanda-console-5c5897b79-cmvl6:console component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320034014Z level=info msg="tailer running" target=production-hm-cert-manager/hm-cert-manager-cainjector-78b5955fd4-shbfm:cert-manager-cainjector component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320037205Z level=info msg="tailer running" target=kube-system/s3-csi-node-plz8r:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320058117Z level=info msg="tailer running" target=kube-system/aws-node-b46vs:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320089787Z level=info msg="tailer running" target=kube-system/ebs-csi-node-v8j6h:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320097372Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-temporal-68d49dbf9-dcstc:airbyte-temporal component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320117838Z level=info msg="tailer running" target=production-hm-sealed-secrets/hm-sealed-secrets-c88b4697d-gtp5d:controller component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32012628Z level=info msg="tailer running" target=production-hm-open-webui/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320138703Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-8879j:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320156774Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320163439Z level=info msg="tailer running" target=kube-system/s3-csi-node-plz8r:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320187595Z level=info msg="tailer running" target=kube-system/s3-csi-node-c7s6f:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32019406Z level=info msg="tailer running" target=production-hm-loki/hm-loki-results-cache-0:exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320204364Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-tkr4x:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320209903Z level=info msg="tailer running" target=production-hm-loki/loki-canary-bjfnw:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32023869Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-run-69747567-s8bn2:wait-for-tracking component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320233295Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-repo-server-844c74bbd8-75qn9:copyutil component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320255713Z level=info msg="tailer running" target=kube-system/s3-csi-node-74jn5:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320253263Z level=info msg="tailer running" target=kube-system/s3-csi-node-c5jv6:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32027589Z level=info msg="tailer running" target=kube-system/aws-node-qlwkj:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32027818Z level=info msg="tailer running" target=kube-system/ebs-csi-node-wrlgw:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320327359Z level=info msg="tailer running" target=production-hm-open-webui/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320326127Z level=info msg="tailer running" target=kube-system/s3-csi-node-vhn8s:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320373088Z level=info msg="tailer running" target=kube-system/s3-csi-node-c5jv6:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320379049Z level=info msg="tailer running" target=kube-system/aws-node-l8sdz:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320388216Z level=info msg="tailer running" target=kube-system/aws-node-qlwkj:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320411287Z level=info msg="tailer running" target=kube-system/s3-csi-node-plz8r:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320439103Z level=info msg="tailer running" target=kube-system/aws-node-qkf4l:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320455116Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-kube-pr-operator-845764d5fb-fbwt6:kube-prometheus-stack component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32044412Z level=info msg="tailer running" target=production-hm-loki/loki-canary-mzms6:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32047912Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:wait-for-database component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320515344Z level=info msg="tailer running" target=kube-system/aws-node-htfw5:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320503501Z level=info msg="tailer running" target=production-hm-loki/loki-read-6d6bdf744f-tvzm2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320525009Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-webapp-57b8ffb99-bhx7x:airbyte-webapp-container component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320541626Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-j5j6h:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320577189Z level=info msg="tailer running" target=kube-system/aws-node-4fwp7:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320614851Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard0-2:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320632726Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-resizer component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320653726Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xpq5z:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320658621Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320660768Z level=info msg="tailer running" target=production-hm-open-webui-pipelines-united-airlines/hm-open-webui-pipelines-6987ffc9f7-8j544:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320678198Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320680822Z level=info msg="tailer running" target=kube-system/ebs-csi-node-wrlgw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320697034Z level=info msg="tailer running" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:init-config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320706736Z level=info msg="tailer running" target=kube-system/kube-proxy-jvz2x:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320726003Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-provisioner component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320725166Z level=info msg="tailer running" target=kube-system/s3-csi-node-7xmdw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320751881Z level=info msg="tailer running" target=kube-system/aws-node-jdpv5:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320753201Z level=info msg="tailer running" target=kube-system/s3-csi-node-s57m2:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320791336Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-run-69747567-s8bn2:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320796963Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-cron-6764f64866-t6779:airbyte-cron component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320819155Z level=info msg="tailer running" target=production-hm-goldilocks/hm-goldilocks-dashboard-65867d6548-lzngr:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320827546Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-bj5jf:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.3208517Z level=info msg="tailer running" target=production-hm-traefik/hm-traefik-7765cbd49c-k2gmz:hm-traefik component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320866069Z level=info msg="tailer running" target=kube-system/ebs-csi-node-fq6nn:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320898704Z level=info msg="tailer running" target=kube-system/aws-node-f886f:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320910525Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-connector-builder-server-f4b78f75b-gk4fg:airbyte-connector-builder-server component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320934787Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:get-default-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320921228Z level=info msg="tailer running" target=kube-system/s3-csi-node-2vgvl:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320950934Z level=info msg="tailer running" target=production-hm-loki/loki-canary-gwcxw:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320961928Z level=info msg="tailer running" target=production-hm-loki/loki-canary-ffqr7:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.320977917Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-wq5fg:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32101726Z level=info msg="tailer running" target=production-hm-loki/hm-loki-chunks-cache-0:exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32101823Z level=info msg="tailer running" target=kube-system/kube-proxy-xdc92:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321020123Z level=info msg="tailer running" target=kube-system/aws-node-b42q9:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32107168Z level=info msg="tailer running" target=production-hm-loki/loki-write-0:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32108743Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321072556Z level=info msg="tailer running" target=kube-system/kube-proxy-9xtfx:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321083461Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-dex-server-d9f78f7b7-pbjz5:copyutil component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321110103Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard1-2:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321137213Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xhgwd:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321138223Z level=info msg="tailer running" target=kube-system/s3-csi-node-p8l8c:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321131445Z level=info msg="tailer running" target=kube-system/s3-csi-node-p8l8c:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321154683Z level=info msg="tailer running" target=kube-system/s3-csi-node-s5lgl:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321163243Z level=info msg="tailer running" target=kube-system/s3-csi-node-s5lgl:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321189863Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-swpk5:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321188421Z level=info msg="tailer running" target=production-hm-loki/loki-backend-1:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321206066Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-phkmk:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321252802Z level=info msg="tailer running" target=kube-system/ebs-csi-node-tlkdd:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32125576Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321267746Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-redis-65cfc58454-4hvf4:redis component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321288602Z level=info msg="tailer running" target=kube-system/s3-csi-node-kz6hm:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321298108Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321300511Z level=info msg="tailer running" target=kube-system/aws-node-nrtx6:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321307963Z level=info msg="tailer running" target=production-hm-loki/loki-read-6d6bdf744f-pjnjg:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321280373Z level=info msg="tailer running" target=production-hm-loki/loki-backend-2:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321335892Z level=info msg="tailer running" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:prometheus component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321345487Z level=info msg="tailer running" target=kube-system/kube-proxy-f9vqh:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321353666Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-l2tgj:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321371792Z level=info msg="tailer running" target=kube-system/kube-proxy-rngsd:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321380137Z level=info msg="tailer running" target=kube-system/s3-csi-node-49gzb:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32138216Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-swvxl:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321395462Z level=info msg="tailer running" target=kube-system/s3-csi-node-k22qf:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321407557Z level=info msg="tailer running" target=production-hm-valkey/hm-valkey-replicas-0:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321417769Z level=info msg="tailer running" target=kube-system/ebs-csi-node-ndqx5:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321424629Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321427472Z level=info msg="tailer running" target=kube-system/ebs-csi-node-fq6nn:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321454117Z level=info msg="tailer running" target=kube-system/ebs-csi-node-fq6nn:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321459989Z level=info msg="tailer running" target=kube-system/s3-csi-node-vhn8s:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321478722Z level=info msg="tailer running" target=kube-system/s3-csi-node-q64sq:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321490759Z level=info msg="tailer running" target=production-hm-valkey/hm-valkey-primary-0:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32150488Z level=info msg="tailer running" target=kube-system/s3-csi-node-c5jv6:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321505356Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-fjzwj:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321521313Z level=info msg="tailer running" target=production-hm-confluent-schema-registry/hm-confluent-schema-registry-deployment-6d8f7454d-cttqw:confluent-schema-registry component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321522689Z level=info msg="tailer running" target=kube-system/s3-csi-node-kz6hm:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321562217Z level=info msg="tailer running" target=kube-system/s3-csi-node-6wbqw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321570112Z level=info msg="tailer running" target=kube-system/aws-node-jdpv5:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321534595Z level=info msg="tailer running" target=kube-system/s3-csi-node-q64sq:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321548587Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321609053Z level=info msg="tailer running" target=kube-system/ebs-csi-node-m6dgw:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321596829Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-database component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321621894Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321637972Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-phkmk:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321642509Z level=info msg="tailer running" target=kube-system/ebs-csi-node-tlkdd:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321642967Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:get-default-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321670723Z level=info msg="tailer running" target=production-hm-loki/loki-backend-1:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321675391Z level=info msg="tailer running" target=production-hm-kuberay-operator/kuberay-operator-6774c79bc9-n6r2t:kuberay-operator component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321685797Z level=info msg="tailer running" target=kube-system/s3-csi-node-k22qf:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321722628Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-bk26w:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32172817Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-webapp-57b8ffb99-7wkzj:airbyte-webapp-container component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321717997Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-ingester-2:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321718928Z level=info msg="tailer running" target=production-hm-loki/hm-loki-chunks-cache-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321745486Z level=info msg="tailer running" target=production-hm-loki/loki-canary-xrffm:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321770646Z level=info msg="tailer running" target=kube-system/ebs-csi-node-4cq7f:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321776737Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-swvxl:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321798911Z level=info msg="tailer running" target=kube-system/s3-csi-node-74jn5:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32183259Z level=info msg="tailer running" target=production-hm-metrics-server/hm-metrics-server-6dff7bdfc6-92k6w:metrics-server component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32185577Z level=info msg="tailer running" target=production-hm-loki/loki-write-1:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321898137Z level=info msg="tailer running" target=production-hm-goldilocks/hm-goldilocks-dashboard-65867d6548-crkpd:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321931039Z level=info msg="tailer running" target=kube-system/ebs-csi-node-dg9mj:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321936489Z level=info msg="tailer running" target=kube-system/s3-csi-node-s5lgl:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321974016Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard1-0:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.321996243Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322024396Z level=info msg="tailer running" target=kube-system/ebs-csi-node-p4zr6:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322032499Z level=info msg="tailer running" target=production-hm-litellm/hm-litellm-deployment-7d754b56f9-c6j7p:litellm component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322073941Z level=info msg="tailer running" target=production-hm-loki/loki-canary-g8bsf:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322103253Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-nz9d9:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322123019Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-kube-state-metrics-99b8b9877-d62k8:kube-state-metrics component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322125878Z level=info msg="tailer running" target=production-hm-open-webui-pipelines/hm-open-webui-pipelines-b5594dd7c-bqtlq:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322140521Z level=info msg="tailer running" target=kube-system/ebs-csi-node-4cq7f:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322142299Z level=info msg="tailer running" target=production-hm-airbyte/hm-airbyte-server-6d8fb769c9-8b9bk:airbyte-server-container component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322158781Z level=info msg="tailer running" target=kube-system/s3-csi-node-q64sq:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322175345Z level=info msg="tailer running" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:alertmanager component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322190715Z level=info msg="tailer running" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-admission-controller-5vz4p4:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322208327Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-dex-server-d9f78f7b7-pbjz5:dex-server component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322234185Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-kwgsp:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32226556Z level=info msg="tailer running" target=kube-system/s3-csi-node-7xmdw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322289235Z level=info msg="tailer running" target=kube-system/ebs-csi-node-m6dgw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32229322Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322315574Z level=info msg="tailer running" target=kube-system/s3-csi-node-2vgvl:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322314634Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xpq5z:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322326631Z level=info msg="tailer running" target=production-hm-loki/loki-read-6d6bdf744f-b4wpx:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322332162Z level=info msg="tailer running" target=kube-system/aws-node-2sxjs:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322364751Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xmp5r:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322384375Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32242447Z level=info msg="tailer running" target=production-hm-opencost/hm-opencost-8586b654fd-ls5zw:opencost-ui component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322442035Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-snapshotter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322453105Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-l8f2p:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32246093Z level=info msg="tailer running" target=kube-system/aws-node-f886f:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322483587Z level=info msg="tailer running" target=production-hm-external-dns/hm-external-dns-6ddcbb57cb-dt5zs:external-dns component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322497299Z level=info msg="tailer running" target=kube-system/ebs-csi-node-p4zr6:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32251273Z level=info msg="tailer running" target=kube-system/s3-csi-node-k22qf:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322511579Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-nz9d9:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322520251Z level=info msg="tailer running" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-updater-76fdd4f9d6-hjrmm:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322534106Z level=info msg="tailer running" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-recommender-6679db88bfk7458:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322558395Z level=info msg="tailer running" target=kube-system/s3-csi-node-49gzb:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32255821Z level=info msg="tailer running" target=production-hm-loki/loki-canary-q4fbl:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322601565Z level=info msg="tailer running" target=kube-system/s3-csi-node-plz8r:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322608405Z level=info msg="tailer running" target=kube-system/kube-proxy-c5gnj:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32261284Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-msvxz:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322634085Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32265214Z level=info msg="tailer running" target=kube-system/aws-node-jdpv5:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322648081Z level=info msg="tailer running" target=kube-system/aws-node-qlwkj:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322678542Z level=info msg="tailer running" target=kube-system/aws-node-qkf4l:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322722483Z level=info msg="tailer running" target=kube-system/kube-proxy-l58gp:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322724981Z level=info msg="tailer running" target=kube-system/ebs-csi-node-njghm:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322738904Z level=info msg="tailer running" target=kube-system/ebs-csi-node-v8j6h:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322750215Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322755446Z level=info msg="tailer running" target=production-hm-loki/loki-canary-x2dmg:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322769383Z level=info msg="tailer running" target=kube-system/aws-node-2q5gn:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322771853Z level=info msg="tailer running" target=kube-system/ebs-csi-node-njghm:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322784287Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322797529Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard0-0:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322829336Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-zqxhz:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322864316Z level=info msg="tailer running" target=kube-system/kube-proxy-vzxgz:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322882359Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-ingester-1:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322913861Z level=info msg="tailer running" target=kube-system/aws-node-l8sdz:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322925836Z level=info msg="tailer running" target=kube-system/s3-csi-node-6wbqw:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322942254Z level=info msg="tailer running" target=kube-system/kube-proxy-hjshg:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322960329Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322968125Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322986614Z level=info msg="tailer running" target=production-hm-cert-manager/hm-cert-manager-webhook-ddc57b566-jhj9h:cert-manager-webhook component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.322980332Z level=info msg="tailer running" target=kube-system/s3-csi-node-vhn8s:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32300617Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard1-1:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323017519Z level=info msg="tailer running" target=production-hm-loki/loki-canary-chnz5:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323028748Z level=info msg="tailer running" target=production-hm-qdrant/hm-qdrant-0:qdrant component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32303547Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323052932Z level=info msg="tailer running" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32306338Z level=info msg="tailer running" target=kube-system/ebs-csi-node-dg9mj:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323069959Z level=info msg="tailer running" target=production-hm-cert-manager/hm-cert-manager-694698b599-s2rsb:cert-manager-controller component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323071891Z level=info msg="tailer running" target=kube-system/s3-csi-node-2vgvl:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323081589Z level=info msg="tailer running" target=kube-system/s3-csi-node-c7s6f:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323111618Z level=info msg="tailer running" target=kube-system/s3-csi-node-7xmdw:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323111849Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323112943Z level=info msg="tailer running" target=kube-system/s3-csi-node-c7s6f:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323123323Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-bj5jf:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323141822Z level=info msg="tailer running" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:wait-for-s3 component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323163367Z level=info msg="tailer running" target=kube-system/s3-csi-node-49gzb:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323173147Z level=info msg="tailer running" target=kube-system/aws-node-b46vs:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323197872Z level=info msg="tailer running" target=kube-system/s3-csi-node-2vgvl:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323199112Z level=info msg="tailer running" target=kube-system/aws-node-nrtx6:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323214492Z level=info msg="tailer running" target=kube-system/aws-node-g7vzz:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32323178Z level=info msg="tailer running" target=kube-system/ebs-csi-node-brxpg:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323239188Z level=info msg="tailer running" target=production-hm-qdrant/hm-qdrant-0:ensure-dir-ownership component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323255907Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323269021Z level=info msg="tailer running" target=kube-system/aws-node-2q5gn:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323269538Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-provisioner component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323289743Z level=info msg="tailer running" target=kube-system/aws-node-4fwp7:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323290054Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xmp5r:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323571858Z level=info msg="tailer running" target=kube-system/kube-proxy-v2wjl:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323655592Z level=info msg="tailer running" target=production-hm-goldilocks/hm-goldilocks-controller-7b7bcb855d-8hshv:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323662826Z level=info msg="tailer running" target=kube-system/ebs-csi-node-njghm:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323609363Z level=info msg="tailer running" target=kube-system/s3-csi-node-kz6hm:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323632664Z level=info msg="tailer running" target=kube-system/kube-proxy-dnc79:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323775335Z level=info msg="tailer running" target=production-hm-loki/loki-canary-jw5zj:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323818373Z level=info msg="tailer running" target=kube-system/ebs-csi-node-dg9mj:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323849354Z level=info msg="tailer running" target=kube-system/coredns-84fd8c67df-lkp8t:coredns component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323890585Z level=info msg="tailer running" target=production-hm-loki/loki-canary-g5jcb:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323917842Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323948776Z level=info msg="tailer running" target=kube-system/s3-csi-node-6wbqw:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323976112Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-msvxz:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.323999735Z level=info msg="tailer running" target=development-hm-open-webui-pipelines/hm-open-webui-pipelines-b5594dd7c-hx9lx:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324032521Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324025684Z level=info msg="tailer running" target=kube-system/ebs-csi-node-k5jjz:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324090936Z level=info msg="tailer running" target=kube-system/ebs-csi-node-v8j6h:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324104269Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324162053Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-server-78699c8c78-tkssp:server component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324165927Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-swpk5:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324229463Z level=info msg="tailer running" target=production-hm-litellm/hm-litellm-deployment-7d754b56f9-5wqwf:litellm component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324234081Z level=info msg="tailer running" target=production-hm-open-webui-united-airlines/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324290511Z level=info msg="tailer running" target=kube-system/ebs-csi-node-xpq5z:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324314901Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-snapshotter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324379836Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-query-frontend-57564b8d4d-5sddh:query-frontend component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324395151Z level=info msg="tailer running" target=kube-system/ebs-csi-node-p4zr6:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324430684Z level=info msg="tailer running" target=development-hm-open-webui/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324418886Z level=info msg="tailer running" target=kube-system/s3-csi-node-kz6hm:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32446606Z level=info msg="tailer running" target=kube-system/coredns-84fd8c67df-lgv7r:coredns component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324474984Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-resizer component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324505891Z level=info msg="tailer running" target=kube-system/kube-proxy-jmsdz:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324528345Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324580733Z level=info msg="tailer running" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-head-p5gm9:ray-head component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324569566Z level=info msg="tailer running" target=kube-system/s3-csi-node-6wbqw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324633295Z level=info msg="tailer running" target=kube-system/ebs-csi-node-tlkdd:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324641559Z level=info msg="tailer running" target=production-hm-argo-cd/hm-argo-cd-argocd-repo-server-844c74bbd8-75qn9:repo-server component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324695212Z level=info msg="tailer running" target=production-hm-clickhouse/hm-clickhouse-shard0-1:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32474956Z level=info msg="tailer running" target=kube-system/s3-csi-node-c7s6f:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324765197Z level=info msg="tailer running" target=kube-system/s3-csi-node-s57m2:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324800977Z level=info msg="tailer running" target=kube-system/s3-csi-node-74jn5:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324830468Z level=info msg="tailer running" target=production-hm-tempo/hm-tempo-metrics-generator-56fb849f7f-vhqtq:metrics-generator component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324847088Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-mn54c:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.32490063Z level=info msg="tailer running" target=production-hm-loki/hm-loki-results-cache-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.3249401Z level=info msg="tailer running" target=kube-system/ebs-csi-node-brxpg:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.324982144Z level=info msg="tailer running" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325006969Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325043425Z level=info msg="tailer running" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325065706Z level=info msg="tailer running" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-w6f99:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325101088Z level=info msg="tailer running" target=kube-system/s3-csi-node-s5lgl:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325191149Z level=info msg="tailer running" target=production-hm-loki/loki-write-2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325119707Z level=info msg="tailer running" target=kube-system/aws-node-b46vs:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325264279Z level=info msg="tailer running" target=production-hm-valkey/hm-valkey-replicas-2:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325129275Z level=info msg="tailer running" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:init-config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325140696Z level=info msg="tailer running" target=kube-system/aws-node-htfw5:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325158784Z level=info msg="tailer running" target=kube-system/aws-node-l8sdz:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325157949Z level=info msg="tailer running" target=kube-system/ebs-csi-node-4cq7f:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325240792Z level=info msg="tailer running" target=kube-system/s3-csi-node-7xmdw:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325260468Z level=info msg="tailer running" target=production-hm-loki/loki-canary-pbjbl:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:40.325983651Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.327039175Z level=info msg="opened log stream" target=kube-system/aws-node-b42q9:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.327802758Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-notifications-controller-785c45558f79w49:notifications-controller component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.328142998Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-ndqx5:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.328552237Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.328610863Z level=info msg="opened log stream" target=kube-system/s3-csi-node-49gzb:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.328879236Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-distributor-6f579f694c-665x8:distributor component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.328990858Z level=info msg="opened log stream" target=kube-system/aws-node-2sxjs:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.32913912Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-s3 component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.329738701Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.525399364Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-0:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.632312521Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 2 errors like: entry for stream '{app=\"mlflow\", cluster=\"hm-eks-cluster\", container=\"wait-for-s3\", container_runtime=\"containerd\", instance=\"production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-s3\", job=\"production-hm-mlflow/wait-for-s3\", namespace=\"production-hm-mlflow\", pod=\"hm-mlflow-tracking-6669f6d7d-chzdz\", service_name=\"mlflow\"}' has timestamp too old: 2025-01-20T01:32:10Z, oldest acceptable timestamp is: 2025-01-23T08:59:40Z; 4 errors like: entry for stream '{app=\"aws-node\", cluster=\"hm-eks-cluster\", container=\"aws-node\", container_runtime=\"containerd\", instance=\"kube-system/aws-node-b42q9:aws-node\", job=\"kube-system/aws-node\", namespace=\"kube-system\", pod=\"aws-node-b42q9\", service_name=\"aws-node\"}' has timestamp too old: 2025-01-20T01:31:59Z, oldest acceptable timestamp is: 2025-01-23T08:59:40Z; 3 errors like: entry for stream '{app=\"aws-ebs-csi-driver\", cluster=\"hm-eks-cluster\", container=\"liveness-probe\", container_runtime=\"containerd\", instance=\""
ts=2025-01-30T08:59:40.726704297Z level=info msg="opened log stream" target=kube-system/aws-node-nrtx6:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:40.925266977Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-7cklq:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:41.124079349Z level=info msg="opened log stream" target=kube-system/aws-node-b42q9:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:41.32827943Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:41.329517012Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 04:52:12.754974803 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T04:52:12Z, oldest acceptable timestamp is: 2025-01-30T07:58:22Z',"
ts=2025-01-30T08:59:41.527042128Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-fjkwv:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:41.726285735Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-nx89v:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:41.924030628Z level=info msg="opened log stream" target=kube-system/aws-node-f886f:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.029327895Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 7 errors like: entry for stream '{app=\"aws-node\", cluster=\"hm-eks-cluster\", container=\"aws-vpc-cni-init\", container_runtime=\"containerd\", instance=\"kube-system/aws-node-b42q9:aws-vpc-cni-init\", job=\"kube-system/aws-vpc-cni-init\", namespace=\"kube-system\", pod=\"aws-node-b42q9\", service_name=\"aws-node\"}' has timestamp too old: 2025-01-20T01:31:56Z, oldest acceptable timestamp is: 2025-01-23T08:59:41Z"
ts=2025-01-30T08:59:42.126545491Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-ingester-0:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.326647696Z level=info msg="opened log stream" target=kube-system/aws-node-2q5gn:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.525976027Z level=info msg="opened log stream" target=kube-system/aws-node-n7jvf:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.728338754Z level=info msg="opened log stream" target=kube-system/s3-csi-node-74jn5:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.925855153Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:upgrade-db-auth component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:42.932515559Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 4 errors like: entry for stream '{app=\"aws-node\", cluster=\"hm-eks-cluster\", container=\"aws-node\", container_runtime=\"containerd\", instance=\"kube-system/aws-node-n7jvf:aws-node\", job=\"kube-system/aws-node\", namespace=\"kube-system\", pod=\"aws-node-n7jvf\", service_name=\"aws-node\"}' has timestamp too old: 2025-01-20T01:31:49Z, oldest acceptable timestamp is: 2025-01-23T08:59:42Z; 2 errors like: entry for stream '{app=\"aws-node\", cluster=\"hm-eks-cluster\", container=\"aws-node\", container_runtime=\"containerd\", instance=\"kube-system/aws-node-n7jvf:aws-node\", job=\"kube-system/aws-node\", namespace=\"kube-system\", pod=\"aws-node-n7jvf\", service_name=\"aws-node\"}' has timestamp too old: 2025-01-20T01:31:56Z, oldest acceptable timestamp is: 2025-01-23T08:59:42Z"
ts=2025-01-30T08:59:43.125197125Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:43.325965403Z level=info msg="opened log stream" target=kube-system/kube-proxy-4txs8:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:43.349343096Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 03:43:02.352030393 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T03:43:02Z, oldest acceptable timestamp is: 2025-01-30T07:58:21Z',"
ts=2025-01-30T08:59:43.526360926Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:43.72638425Z level=info msg="opened log stream" target=kube-system/aws-node-htfw5:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:43.737866679Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 02:29:22.068693441 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T02:29:22Z, oldest acceptable timestamp is: 2025-01-30T07:58:37Z',"
ts=2025-01-30T08:59:43.926192036Z level=info msg="opened log stream" target=production-hm-opencost/hm-opencost-8586b654fd-ls5zw:hm-opencost component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:44.12792155Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:44.325029784Z level=info msg="opened log stream" target=production-hm-valkey/hm-valkey-replicas-1:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:44.526580531Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c5jv6:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:44.72499631Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-ww74b:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:44.729436747Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 06:19:23.255248233 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T06:19:23Z, oldest acceptable timestamp is: 2025-01-30T07:58:22Z',"
ts=2025-01-30T08:59:44.927763409Z level=info msg="opened log stream" target=kube-system/aws-node-v6f6g:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:45.12648523Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:45.31632627Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-clpk6:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:45.316409534Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-wsh2k:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:45.316600451Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-clpk6:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:45.316701116Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-wsh2k:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:45.326994979Z level=info msg="opened log stream" target=kube-system/aws-node-4fwp7:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:45.33209621Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 1 errors like: entry for stream '{app=\"argocd-applicationset-controller\", cluster=\"hm-eks-cluster\", container=\"applicationset-controller\", container_runtime=\"containerd\", instance=\"production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller\", job=\"production-hm-argo-cd/applicationset-controller\", namespace=\"production-hm-argo-cd\", pod=\"hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5\", service_name=\"argocd-applicationset-controller\"}' has timestamp too old: 2025-01-02T01:55:34Z, oldest acceptable timestamp is: 2025-01-23T08:59:44Z; 1 errors like: entry for stream '{app=\"argocd-applicationset-controller\", cluster=\"hm-eks-cluster\", container=\"applicationset-controller\", container_runtime=\"containerd\", instance=\"production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller\", job=\"production-hm-argo-cd/applicationset-controller\", namespace=\"production-hm-argo"
ts=2025-01-30T08:59:45.525176998Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-brxpg:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:45.724948347Z level=info msg="opened log stream" target=development-hm-open-webui/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:45.926462858Z level=info msg="opened log stream" target=kube-system/s3-csi-node-q64sq:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:46.126990319Z level=info msg="opened log stream" target=kube-system/aws-node-v6f6g:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:46.229473487Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 07:13:35.825498664 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T07:13:35Z, oldest acceptable timestamp is: 2025-01-30T07:58:41Z',"
ts=2025-01-30T08:59:46.32868002Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:46.526010728Z level=info msg="opened log stream" target=kube-system/aws-node-v6f6g:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:46.725593655Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-k5jjz:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:46.926463317Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-compactor-77b694d464-qptdc:compactor component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:47.125263426Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xmp5r:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:47.326194936Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-querier-75fb7d4b96-2nbw8:querier component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:47.527126633Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s57m2:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:47.725804598Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-application-controller-0:application-controller component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:47.738885897Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 1 errors like: entry for stream '{app=\"argocd-applicationset-controller\", cluster=\"hm-eks-cluster\", container=\"applicationset-controller\", container_runtime=\"containerd\", instance=\"production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller\", job=\"production-hm-argo-cd/applicationset-controller\", namespace=\"production-hm-argo-cd\", pod=\"hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5\", service_name=\"argocd-applicationset-controller\"}' has timestamp too old: 2025-01-14T11:45:34Z, oldest acceptable timestamp is: 2025-01-23T08:59:46Z; 1 errors like: entry for stream '{app=\"argocd-applicationset-controller\", cluster=\"hm-eks-cluster\", container=\"applicationset-controller\", container_runtime=\"containerd\", instance=\"production-hm-argo-cd/hm-argo-cd-argocd-applicationset-controller-6bd4f8b66rcmq5:applicationset-controller\", job=\"production-hm-argo-cd/applicationset-controller\", namespace=\"production-hm-argo"
ts=2025-01-30T08:59:47.925220431Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s57m2:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:48.126489918Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:48.325473124Z level=info msg="opened log stream" target=production-hm-open-webui-united-airlines/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:48.335650927Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 06:36:10.21116004 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T06:36:10Z, oldest acceptable timestamp is: 2025-01-30T07:57:10Z',"
ts=2025-01-30T08:59:48.526766626Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-wf2sv:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:48.724663818Z level=info msg="opened log stream" target=kube-system/s3-csi-node-p8l8c:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:48.926588554Z level=info msg="opened log stream" target=kube-system/aws-node-g7vzz:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:49.12705532Z level=info msg="opened log stream" target=kube-system/s3-csi-node-vhn8s:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:49.137304262Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-23 09:05:34.235476715 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-23T09:05:34Z, oldest acceptable timestamp is: 2025-01-30T07:55:34Z',"
ts=2025-01-30T08:59:49.327625878Z level=info msg="opened log stream" target=kube-system/aws-node-n7jvf:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:49.525539637Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-pod-sweeper-pod-sweeper-765ffd4d5c-4wj22:airbyte-pod-sweeper component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:49.743208267Z level=info msg="opened log stream" target=kube-system/aws-node-qkf4l:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:49.75533683Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 3 errors like: entry for stream '{app=\"aws-mountpoint-s3-csi-driver\", cluster=\"hm-eks-cluster\", container=\"install-mountpoint\", container_runtime=\"containerd\", instance=\"kube-system/s3-csi-node-vhn8s:install-mountpoint\", job=\"kube-system/install-mountpoint\", namespace=\"kube-system\", pod=\"s3-csi-node-vhn8s\", service_name=\"aws-mountpoint-s3-csi-driver\"}' has timestamp too old: 2024-10-10T15:37:55Z, oldest acceptable timestamp is: 2025-01-23T08:59:49Z"
ts=2025-01-30T08:59:49.927011725Z level=info msg="opened log stream" target=kube-system/aws-node-n7jvf:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:50.12629169Z level=info msg="opened log stream" target=kube-system/kube-proxy-dgd84:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:50.316397111Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-bnzmn:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316501741Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-r4xhv:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316498156Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-lvlw4:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316558538Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-r4xhv:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316690807Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-dfvmr:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316750837Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-bnzmn:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316795997Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-pqqps:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.31688835Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-lvlw4:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316893449Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-pqqps:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.316916432Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-dfvmr:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:50.324612943Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-xsw6t:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:50.525356216Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-k5jjz:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:50.72477846Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-0:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:50.925671787Z level=info msg="opened log stream" target=kube-system/kube-proxy-rfksb:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:51.125066212Z level=info msg="opened log stream" target=kube-system/s3-csi-node-k22qf:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:51.333861875Z level=info msg="opened log stream" target=production-hm-loki/hm-loki-gateway-84cc597d99-cbhfx:nginx component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:51.524190937Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xhgwd:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:51.724575613Z level=info msg="opened log stream" target=production-hm-akhq/hm-akhq-6fd5579f4f-ghgkg:akhq component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:51.927018509Z level=info msg="opened log stream" target=kube-system/aws-node-2sxjs:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:52.125592672Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:52.325693594Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-worker-849b8f7886-v4kxb:airbyte-worker-container component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:52.527070882Z level=info msg="opened log stream" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:52.725126544Z level=info msg="opened log stream" target=kube-system/aws-node-g7vzz:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:52.924311751Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:53.126951144Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-m6dgw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:53.325467661Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xhgwd:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:53.524407115Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-wrlgw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:53.727316201Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:upgrade-db-auth component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:53.925851004Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-attacher component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:54.031249314Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 05:00:18.340149001 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T05:00:18Z, oldest acceptable timestamp is: 2025-01-30T07:58:21Z',"
ts=2025-01-30T08:59:54.124079796Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-memcached-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:54.327594348Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:54.532746932Z level=info msg="opened log stream" target=kube-system/s3-csi-node-p8l8c:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:54.724666069Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-ndqx5:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:54.925804791Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-attacher component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:55.12613377Z level=info msg="opened log stream" target=production-hm-redpanda-console/hm-redpanda-console-5c5897b79-cmvl6:console component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:55.316500172Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-qqhsn:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316532678Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-qqhsn:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316629976Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-qjvm4:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316663914Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-4b869:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316796653Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-d9lbv:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316813345Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-4b869:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316839863Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-d9lbv:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.316924837Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-2gdbf:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317040651Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-2gdbf:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317053754Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-qjvm4:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317080263Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-5cnjf:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317122726Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-hvq5p:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317157184Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-5cnjf:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.317163104Z level=info msg="tailer running" target=production-hm-alloy/hm-alloy-hvq5p:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T08:59:55.326137118Z level=info msg="opened log stream" target=kube-system/s3-csi-node-plz8r:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:55.524257063Z level=info msg="opened log stream" target=production-hm-cert-manager/hm-cert-manager-cainjector-78b5955fd4-shbfm:cert-manager-cainjector component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:55.724211637Z level=info msg="opened log stream" target=kube-system/aws-node-b46vs:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:55.925595694Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-v8j6h:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:56.12422181Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-temporal-68d49dbf9-dcstc:airbyte-temporal component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:56.324896605Z level=info msg="opened log stream" target=production-hm-sealed-secrets/hm-sealed-secrets-c88b4697d-gtp5d:controller component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:56.524453008Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-8879j:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:56.724936757Z level=info msg="opened log stream" target=production-hm-open-webui/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:56.924097716Z level=info msg="opened log stream" target=kube-system/s3-csi-node-plz8r:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:57.124632832Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:57.323484013Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c7s6f:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:57.524408699Z level=info msg="opened log stream" target=production-hm-loki/hm-loki-results-cache-0:exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:57.724259911Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-tkr4x:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:57.92576732Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-bjfnw:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:58.124198145Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-run-69747567-s8bn2:wait-for-tracking component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:58.325045826Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-repo-server-844c74bbd8-75qn9:copyutil component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:58.525007485Z level=info msg="opened log stream" target=kube-system/s3-csi-node-74jn5:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:58.724849676Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c5jv6:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:58.923897266Z level=info msg="opened log stream" target=kube-system/aws-node-qlwkj:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:59.124630484Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-wrlgw:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:59.324448852Z level=info msg="opened log stream" target=production-hm-open-webui/open-webui-0:open-webui component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:59.524772006Z level=info msg="opened log stream" target=kube-system/s3-csi-node-vhn8s:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:59.725917195Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c5jv6:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T08:59:59.829413735Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 02:57:22.058928017 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T02:57:22Z, oldest acceptable timestamp is: 2025-01-30T07:58:37Z',"
ts=2025-01-30T08:59:59.924006239Z level=info msg="opened log stream" target=kube-system/aws-node-qlwkj:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:00.122901663Z level=info msg="opened log stream" target=kube-system/aws-node-l8sdz:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:00.324416787Z level=info msg="opened log stream" target=kube-system/s3-csi-node-plz8r:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:00.524423373Z level=info msg="opened log stream" target=kube-system/aws-node-qkf4l:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:00.724228826Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-kube-pr-operator-845764d5fb-fbwt6:kube-prometheus-stack component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:00.923684166Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-mzms6:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:01.127287272Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:wait-for-database component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:01.325223421Z level=info msg="opened log stream" target=kube-system/aws-node-htfw5:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:01.525966627Z level=info msg="opened log stream" target=production-hm-loki/loki-read-6d6bdf744f-tvzm2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:01.72569544Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-webapp-57b8ffb99-bhx7x:airbyte-webapp-container component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:01.924928285Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-j5j6h:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:02.128003263Z level=info msg="opened log stream" target=kube-system/aws-node-4fwp7:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:02.325241845Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard0-2:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:02.524511129Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-resizer component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:02.724885945Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xpq5z:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:02.924369589Z level=info msg="opened log stream" target=production-hm-open-webui-pipelines-united-airlines/hm-open-webui-pipelines-6987ffc9f7-8j544:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:03.127340368Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:03.324763241Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:03.524381416Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-wrlgw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:03.725555882Z level=info msg="opened log stream" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:init-config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:03.925044176Z level=info msg="opened log stream" target=kube-system/kube-proxy-jvz2x:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:04.124972415Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-provisioner component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:04.325425966Z level=info msg="opened log stream" target=kube-system/s3-csi-node-7xmdw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:04.526225879Z level=info msg="opened log stream" target=kube-system/aws-node-jdpv5:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:04.726210678Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s57m2:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:04.736627855Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 04:23:35.369548508 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T04:23:35Z, oldest acceptable timestamp is: 2025-01-30T07:58:36Z',"
ts=2025-01-30T09:00:04.926172672Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-cron-6764f64866-t6779:airbyte-cron component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:05.125786214Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-run-69747567-s8bn2:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:05.328815679Z level=info msg="opened log stream" target=production-hm-goldilocks/hm-goldilocks-dashboard-65867d6548-lzngr:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:05.525679746Z level=info msg="opened log stream" target=production-hm-traefik/hm-traefik-7765cbd49c-k2gmz:hm-traefik component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:05.725066895Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-bj5jf:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:05.925910439Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-fq6nn:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:06.125494678Z level=info msg="opened log stream" target=kube-system/aws-node-f886f:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:06.32572497Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-connector-builder-server-f4b78f75b-gk4fg:airbyte-connector-builder-server component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:06.526421476Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:get-default-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:06.725773152Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-gwcxw:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:06.925047559Z level=info msg="opened log stream" target=kube-system/s3-csi-node-2vgvl:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:07.124769883Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-ffqr7:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:07.324030494Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-wq5fg:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:07.523844731Z level=info msg="opened log stream" target=kube-system/kube-proxy-xdc92:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:07.725015036Z level=info msg="opened log stream" target=production-hm-loki/hm-loki-chunks-cache-0:exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:07.924718879Z level=info msg="opened log stream" target=kube-system/aws-node-b42q9:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:08.123722611Z level=info msg="opened log stream" target=production-hm-loki/loki-write-0:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:08.326073439Z level=info msg="opened log stream" target=kube-system/kube-proxy-9xtfx:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:08.524363038Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:mlflow component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:08.724021105Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-dex-server-d9f78f7b7-pbjz5:copyutil component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:08.831899874Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-29 18:46:03.762647579 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-29T18:46:03Z, oldest acceptable timestamp is: 2025-01-30T07:58:18Z',"
ts=2025-01-30T09:00:08.923381753Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard1-2:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:09.123295551Z level=info msg="opened log stream" target=kube-system/s3-csi-node-p8l8c:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:09.32494762Z level=info msg="opened log stream" target=kube-system/s3-csi-node-p8l8c:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:09.524660249Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xhgwd:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:09.724289454Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s5lgl:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:09.923965873Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s5lgl:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:10.126286927Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-swpk5:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:10.326512108Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-1:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:10.523935075Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-phkmk:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:10.726203342Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:10.926418698Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-tlkdd:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:11.125534637Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-redis-65cfc58454-4hvf4:redis component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:11.325052136Z level=info msg="opened log stream" target=kube-system/s3-csi-node-kz6hm:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:11.526705618Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:11.72614359Z level=info msg="opened log stream" target=kube-system/aws-node-nrtx6:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:11.925701321Z level=info msg="opened log stream" target=production-hm-loki/loki-read-6d6bdf744f-pjnjg:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:12.126668163Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-2:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:12.230657596Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 06:44:08.314808979 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T06:44:08Z, oldest acceptable timestamp is: 2025-01-30T07:58:21Z',"
ts=2025-01-30T09:00:12.3260806Z level=info msg="opened log stream" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:prometheus component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:12.524341132Z level=info msg="opened log stream" target=kube-system/kube-proxy-f9vqh:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:12.726900864Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-l2tgj:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:12.926774895Z level=info msg="opened log stream" target=kube-system/kube-proxy-rngsd:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:13.124685849Z level=info msg="opened log stream" target=kube-system/s3-csi-node-49gzb:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:13.326380923Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-swvxl:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:13.526177248Z level=info msg="opened log stream" target=kube-system/s3-csi-node-k22qf:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:13.726158536Z level=info msg="opened log stream" target=production-hm-valkey/hm-valkey-replicas-0:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:13.927470353Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-ndqx5:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:14.124592223Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:14.325866966Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-fq6nn:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:14.52564964Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-fq6nn:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:14.72442086Z level=info msg="opened log stream" target=kube-system/s3-csi-node-vhn8s:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:14.925775513Z level=info msg="opened log stream" target=kube-system/s3-csi-node-q64sq:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:15.125656907Z level=info msg="opened log stream" target=production-hm-valkey/hm-valkey-primary-0:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:15.325826495Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c5jv6:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:15.526589892Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-fjzwj:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:15.726043459Z level=info msg="opened log stream" target=production-hm-confluent-schema-registry/hm-confluent-schema-registry-deployment-6d8f7454d-cttqw:confluent-schema-registry component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:15.924123931Z level=info msg="opened log stream" target=kube-system/s3-csi-node-kz6hm:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:16.125473622Z level=info msg="opened log stream" target=kube-system/s3-csi-node-6wbqw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:16.326275287Z level=info msg="opened log stream" target=kube-system/aws-node-jdpv5:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:16.526737688Z level=info msg="opened log stream" target=kube-system/s3-csi-node-q64sq:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:16.725542746Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:16.925568385Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-m6dgw:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:17.128246503Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:17.230722043Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 4 errors like: entry for stream '{app=\"console\", cluster=\"hm-eks-cluster\", container=\"console\", container_runtime=\"containerd\", instance=\"production-hm-redpanda-console/hm-redpanda-console-5c5897b79-cmvl6:console\", job=\"production-hm-redpanda-console/console\", namespace=\"production-hm-redpanda-console\", pod=\"hm-redpanda-console-5c5897b79-cmvl6\", service_name=\"console\"}' has timestamp too old: 2025-01-22T11:53:34Z, oldest acceptable timestamp is: 2025-01-23T09:00:12Z; 1 errors like: entry for stream '{app=\"traefik\", cluster=\"hm-eks-cluster\", container=\"hm-traefik\", container_runtime=\"containerd\", instance=\"production-hm-traefik/hm-traefik-7765cbd49c-k2gmz:hm-traefik\", job=\"production-hm-traefik/hm-traefik\", namespace=\"production-hm-traefik\", pod=\"hm-traefik-7765cbd49c-k2gmz\", service_name=\"traefik\"}' has timestamp too old: 2025-01-11T08:35:44Z, oldest acceptable timestamp is: 2025-01-23T09:00:12Z; 1 errors like: entry for stream '{app=\""
ts=2025-01-30T09:00:17.324698487Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-tlkdd:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:17.525204873Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:get-default-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:17.727294101Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-database component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:17.924014334Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-phkmk:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:18.126517931Z level=info msg="opened log stream" target=production-hm-loki/loki-backend-1:loki-sc-rules component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:18.328443235Z level=info msg="opened log stream" target=production-hm-kuberay-operator/kuberay-operator-6774c79bc9-n6r2t:kuberay-operator component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:18.525185975Z level=info msg="opened log stream" target=kube-system/s3-csi-node-k22qf:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:18.725383484Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-bk26w:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:18.927989107Z level=info msg="opened log stream" target=production-hm-loki/hm-loki-chunks-cache-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:19.127957333Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-ingester-2:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:19.326280528Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-webapp-57b8ffb99-7wkzj:airbyte-webapp-container component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:19.527124472Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-xrffm:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:19.723328022Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-4cq7f:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:19.926622609Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-swvxl:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:20.125934041Z level=info msg="opened log stream" target=kube-system/s3-csi-node-74jn5:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:20.324631345Z level=info msg="opened log stream" target=production-hm-metrics-server/hm-metrics-server-6dff7bdfc6-92k6w:metrics-server component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:20.525423481Z level=info msg="opened log stream" target=production-hm-loki/loki-write-1:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:20.725698983Z level=info msg="opened log stream" target=production-hm-goldilocks/hm-goldilocks-dashboard-65867d6548-crkpd:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:20.925660559Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-dg9mj:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:21.125866592Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s5lgl:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:21.326142717Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard1-0:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:21.52591853Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:21.726467447Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-p4zr6:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:21.925688508Z level=info msg="opened log stream" target=production-hm-litellm/hm-litellm-deployment-7d754b56f9-c6j7p:litellm component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:22.124929183Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-g8bsf:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:22.326162635Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-nz9d9:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:22.526138108Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-kube-state-metrics-99b8b9877-d62k8:kube-state-metrics component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:22.724855358Z level=info msg="opened log stream" target=production-hm-open-webui-pipelines/hm-open-webui-pipelines-b5594dd7c-bqtlq:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:22.927005783Z level=info msg="opened log stream" target=production-hm-airbyte/hm-airbyte-server-6d8fb769c9-8b9bk:airbyte-server-container component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:23.124709587Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-4cq7f:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:23.229340138Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 00:22:48.67167646 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T00:22:48Z, oldest acceptable timestamp is: 2025-01-30T07:54:09Z',"
ts=2025-01-30T09:00:23.324597358Z level=info msg="opened log stream" target=kube-system/s3-csi-node-q64sq:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:23.52696086Z level=info msg="opened log stream" target=production-hm-prometheus/alertmanager-hm-prometheus-kube-pr-alertmanager-0:alertmanager component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:23.725694719Z level=info msg="opened log stream" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-admission-controller-5vz4p4:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:23.926959845Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-dex-server-d9f78f7b7-pbjz5:dex-server component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:24.126805923Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-kwgsp:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:24.32674773Z level=info msg="opened log stream" target=kube-system/s3-csi-node-7xmdw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:24.525668778Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-m6dgw:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:24.726279579Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:24.926276337Z level=info msg="opened log stream" target=kube-system/s3-csi-node-2vgvl:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:25.125671051Z level=info msg="opened log stream" target=production-hm-loki/loki-read-6d6bdf744f-b4wpx:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:25.327875911Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xpq5z:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:25.527171355Z level=info msg="opened log stream" target=kube-system/aws-node-2sxjs:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:25.726202131Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xmp5r:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:25.924538175Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:26.12633591Z level=info msg="opened log stream" target=production-hm-opencost/hm-opencost-8586b654fd-ls5zw:opencost-ui component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:26.32598578Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-snapshotter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:26.524997617Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-l8f2p:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:26.726283234Z level=info msg="opened log stream" target=kube-system/aws-node-f886f:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:26.925447207Z level=info msg="opened log stream" target=production-hm-external-dns/hm-external-dns-6ddcbb57cb-dt5zs:external-dns component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:27.125356729Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-p4zr6:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:27.326598098Z level=info msg="opened log stream" target=kube-system/s3-csi-node-k22qf:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:27.524677035Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-nz9d9:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:27.725586731Z level=info msg="opened log stream" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-updater-76fdd4f9d6-hjrmm:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:27.926652183Z level=info msg="opened log stream" target=production-hm-vertical-pod-autoscaler/hm-vertical-pod-autoscaler-vpa-recommender-6679db88bfk7458:vpa component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:28.123852279Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-q4fbl:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:28.325606137Z level=info msg="opened log stream" target=kube-system/s3-csi-node-49gzb:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:28.527046092Z level=info msg="opened log stream" target=kube-system/s3-csi-node-plz8r:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:28.724318275Z level=info msg="opened log stream" target=kube-system/kube-proxy-c5gnj:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:28.925194279Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-msvxz:ray-worker component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:29.03861125Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-23 17:08:29.060999544 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-23T17:08:29Z, oldest acceptable timestamp is: 2025-01-30T07:58:39Z',"
ts=2025-01-30T09:00:29.12692304Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:29.324426026Z level=info msg="opened log stream" target=kube-system/aws-node-jdpv5:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:29.525674642Z level=info msg="opened log stream" target=kube-system/aws-node-qlwkj:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:29.727071631Z level=info msg="opened log stream" target=kube-system/aws-node-qkf4l:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:29.925198211Z level=info msg="opened log stream" target=kube-system/kube-proxy-l58gp:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:30.125195569Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-njghm:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:30.325695945Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-v8j6h:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:30.528353362Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-z87sl:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:30.726430285Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-x2dmg:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:30.924013483Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-njghm:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:31.124863053Z level=info msg="opened log stream" target=kube-system/aws-node-2q5gn:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:31.326351962Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard0-0:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:31.524773814Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-28f6b:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:31.6400699Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 03:48:44.310856045 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T03:48:44Z, oldest acceptable timestamp is: 2025-01-30T07:58:40Z',"
ts=2025-01-30T09:00:31.726202911Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-zqxhz:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:31.925707962Z level=info msg="opened log stream" target=kube-system/kube-proxy-vzxgz:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:32.125669487Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-ingester-1:ingester component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:32.326411683Z level=info msg="opened log stream" target=kube-system/aws-node-l8sdz:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:32.526739705Z level=info msg="opened log stream" target=kube-system/s3-csi-node-6wbqw:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:32.724790981Z level=info msg="opened log stream" target=kube-system/kube-proxy-hjshg:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:32.925948773Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-4vzr6:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:33.12649118Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:33.32485538Z level=info msg="opened log stream" target=kube-system/s3-csi-node-vhn8s:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:33.525945312Z level=info msg="opened log stream" target=production-hm-cert-manager/hm-cert-manager-webhook-ddc57b566-jhj9h:cert-manager-webhook component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:33.724089848Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard1-1:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:33.924941737Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-chnz5:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:34.125203006Z level=info msg="opened log stream" target=production-hm-qdrant/hm-qdrant-0:qdrant component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:34.326916826Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-lltj9:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:34.526408226Z level=info msg="opened log stream" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:34.727143501Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-dg9mj:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:34.925130428Z level=info msg="opened log stream" target=production-hm-cert-manager/hm-cert-manager-694698b599-s2rsb:cert-manager-controller component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:35.124744647Z level=info msg="opened log stream" target=kube-system/s3-csi-node-2vgvl:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:35.325603428Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c7s6f:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:35.526296129Z level=info msg="opened log stream" target=kube-system/s3-csi-node-7xmdw:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:35.727822863Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c7s6f:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:35.925962946Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:36.124843143Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-bj5jf:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:36.325403064Z level=info msg="opened log stream" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:wait-for-s3 component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:36.525793543Z level=info msg="opened log stream" target=kube-system/s3-csi-node-49gzb:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:36.529412731Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 01:57:49.082273555 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T01:57:49Z, oldest acceptable timestamp is: 2025-01-30T07:57:49Z',"
ts=2025-01-30T09:00:36.725559801Z level=info msg="opened log stream" target=kube-system/aws-node-b46vs:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:36.923791913Z level=info msg="opened log stream" target=kube-system/s3-csi-node-2vgvl:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:37.126788485Z level=info msg="opened log stream" target=kube-system/aws-node-nrtx6:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:37.3256558Z level=info msg="opened log stream" target=kube-system/aws-node-g7vzz:aws-node component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:37.523502818Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-brxpg:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:37.726608539Z level=info msg="opened log stream" target=production-hm-qdrant/hm-qdrant-0:ensure-dir-ownership component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:37.924947605Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:38.124703896Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-provisioner component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:38.327059875Z level=info msg="opened log stream" target=kube-system/aws-node-2q5gn:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:38.526087862Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xmp5r:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:38.724362368Z level=info msg="opened log stream" target=kube-system/aws-node-4fwp7:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:38.926855167Z level=info msg="opened log stream" target=production-hm-goldilocks/hm-goldilocks-controller-7b7bcb855d-8hshv:goldilocks component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:39.125549327Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-njghm:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:39.323938822Z level=info msg="opened log stream" target=kube-system/s3-csi-node-kz6hm:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:39.525072621Z level=info msg="opened log stream" target=kube-system/kube-proxy-dnc79:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:39.726164266Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-jw5zj:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:39.924732478Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-dg9mj:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:40.124967156Z level=info msg="opened log stream" target=kube-system/coredns-84fd8c67df-lkp8t:coredns component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:40.324962029Z level=info msg="opened log stream" target=kube-system/kube-proxy-v2wjl:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:40.525308289Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-g5jcb:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:40.725824591Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-jpzc2:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:40.927261195Z level=info msg="opened log stream" target=kube-system/s3-csi-node-6wbqw:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:41.125631565Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-workergroup-worker-msvxz:wait-gcs-ready component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:41.32393106Z level=info msg="opened log stream" target=development-hm-open-webui-pipelines/hm-open-webui-pipelines-b5594dd7c-hx9lx:pipelines component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:41.526293452Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:41.725857522Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-k5jjz:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:41.83289018Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): 1 errors like: entry for stream '{app=\"aws-ebs-csi-driver\", cluster=\"hm-eks-cluster\", container=\"ebs-plugin\", container_runtime=\"containerd\", instance=\"kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:ebs-plugin\", job=\"kube-system/ebs-plugin\", namespace=\"kube-system\", pod=\"ebs-csi-controller-7585b4bfb5-4vrv4\", service_name=\"aws-ebs-csi-driver\"}' has timestamp too old: 2024-10-27T05:18:28Z, oldest acceptable timestamp is: 2025-01-23T09:00:37Z; 3 errors like: entry for stream '{cluster=\"hm-eks-cluster\", container=\"kube-proxy\", container_runtime=\"containerd\", instance=\"kube-system/kube-proxy-9xtfx:kube-proxy\", job=\"kube-system/kube-proxy\", namespace=\"kube-system\", pod=\"kube-proxy-9xtfx\", service_name=\"kube-proxy\"}' has timestamp too old: 2024-11-28T16:14:46Z, oldest acceptable timestamp is: 2025-01-23T09:00:37Z; 2 errors like: entry for stream '{app=\"traefik\", cluster=\"hm-eks-cluster\", container=\"hm-traefik\", container_runtime=\"containerd\", instance=\"production-hm-traefik/hm-traefik-"
ts=2025-01-30T09:00:41.924963059Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-v8j6h:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:42.125569788Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:grafana-sc-dashboard component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:42.328052735Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-server-78699c8c78-tkssp:server component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:42.524135221Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-swpk5:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:42.726809139Z level=info msg="opened log stream" target=production-hm-litellm/hm-litellm-deployment-7d754b56f9-5wqwf:litellm component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:42.924803836Z level=info msg="opened log stream" target=production-hm-open-webui-united-airlines/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:43.126015087Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-xpq5z:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:43.326876951Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:csi-snapshotter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:43.523507541Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-query-frontend-57564b8d4d-5sddh:query-frontend component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:43.726414065Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-p4zr6:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:43.926349116Z level=info msg="opened log stream" target=development-hm-open-webui/open-webui-0:copy-app-data component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:44.123757562Z level=info msg="opened log stream" target=kube-system/s3-csi-node-kz6hm:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:44.325080154Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-4vrv4:csi-resizer component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:44.52695821Z level=info msg="opened log stream" target=kube-system/kube-proxy-jmsdz:kube-proxy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:44.724173265Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-q5dks:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:44.925009406Z level=info msg="opened log stream" target=kube-system/s3-csi-node-6wbqw:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:45.127150938Z level=info msg="opened log stream" target=production-hm-ray-cluster/hm-ray-cluster-kuberay-head-p5gm9:ray-head component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:45.323767037Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-tlkdd:liveness-probe component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:45.524846596Z level=info msg="opened log stream" target=production-hm-argo-cd/hm-argo-cd-argocd-repo-server-844c74bbd8-75qn9:repo-server component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:45.72919246Z level=info msg="opened log stream" target=production-hm-clickhouse/hm-clickhouse-shard0-1:clickhouse component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:45.923353606Z level=info msg="opened log stream" target=kube-system/s3-csi-node-c7s6f:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:46.125188132Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s57m2:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:46.325976465Z level=info msg="opened log stream" target=kube-system/s3-csi-node-74jn5:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:46.525128193Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-mn54c:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:46.725982326Z level=info msg="opened log stream" target=production-hm-tempo/hm-tempo-metrics-generator-56fb849f7f-vhqtq:metrics-generator component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:46.926084737Z level=info msg="opened log stream" target=production-hm-loki/hm-loki-results-cache-0:memcached component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:47.12467594Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-brxpg:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:47.326823989Z level=info msg="opened log stream" target=kube-system/ebs-csi-controller-7585b4bfb5-46m8t:ebs-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:47.52575677Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-rx66s:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:47.532566849Z level=error msg="final error sending batch" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=400 tenant="" error="server returned HTTP status 400 Bad Request (400): entry with timestamp 2025-01-30 06:43:20.581785485 +0000 UTC ignored, reason: 'entry too far behind, entry timestamp is: 2025-01-30T06:43:20Z, oldest acceptable timestamp is: 2025-01-30T07:58:41Z',"
ts=2025-01-30T09:00:47.725046887Z level=info msg="opened log stream" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pxz9q:grafana component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:47.926388372Z level=info msg="opened log stream" target=production-hm-prometheus/hm-prometheus-prometheus-node-exporter-w6f99:node-exporter component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:48.126188381Z level=info msg="opened log stream" target=kube-system/coredns-84fd8c67df-lgv7r:coredns component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:48.325676563Z level=info msg="opened log stream" target=production-hm-loki/loki-write-2:loki component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:48.526186795Z level=info msg="opened log stream" target=kube-system/s3-csi-node-s5lgl:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:48.726291219Z level=info msg="opened log stream" target=kube-system/aws-node-b46vs:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:48.925557669Z level=info msg="opened log stream" target=production-hm-valkey/hm-valkey-replicas-2:valkey component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:49.129356684Z level=info msg="opened log stream" target=production-hm-prometheus/prometheus-hm-prometheus-kube-pr-prometheus-0:init-config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:49.330613783Z level=info msg="opened log stream" target=kube-system/aws-node-htfw5:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:49.537237956Z level=info msg="opened log stream" target=kube-system/aws-node-l8sdz:aws-eks-nodeagent component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:49.724734936Z level=info msg="opened log stream" target=kube-system/ebs-csi-node-4cq7f:node-driver-registrar component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:49.926847246Z level=info msg="opened log stream" target=kube-system/s3-csi-node-7xmdw:s3-plugin component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:50.133668529Z level=info msg="opened log stream" target=production-hm-loki/loki-canary-pbjbl:loki-canary component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:50.323903888Z level=info msg="tailer exited" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:50.524528737Z level=info msg="tailer exited" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:wait-for-s3 component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:50.722374197Z level=info msg="tailer exited" target=kube-system/aws-node-nrtx6:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:50.922151339Z level=info msg="tailer exited" target=kube-system/aws-node-b42q9:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:51.122727081Z level=info msg="tailer exited" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-jvrrg:upgrade-db-auth component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:51.322571532Z level=info msg="tailer exited" target=kube-system/aws-node-htfw5:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:51.522017949Z level=info msg="tailer exited" target=production-hm-grafana/hm-grafana-67c5cdd9f8-sx8p5:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:51.726146634Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-clpk6:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:51.926400624Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-wsh2k:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:52.126000337Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-clpk6:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:52.325471164Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-wsh2k:alloy component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:52.522670964Z level=info msg="tailer exited" target=kube-system/aws-node-v6f6g:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:52.72309128Z level=info msg="tailer exited" target=kube-system/aws-node-4fwp7:aws-vpc-cni-init component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:52.923171315Z level=info msg="tailer exited" target=production-hm-grafana/hm-grafana-67c5cdd9f8-pb6pj:download-dashboards component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:53.123364004Z level=info msg="tailer exited" target=production-hm-mlflow/hm-mlflow-tracking-6669f6d7d-chzdz:render-auth-conf component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:53.322598827Z level=info msg="tailer exited" target=kube-system/s3-csi-node-vhn8s:install-mountpoint component_path=/ component_id=loki.source.kubernetes.pod_logs
ts=2025-01-30T09:00:53.526227962Z level=info msg="opened log stream" target=production-hm-alloy/hm-alloy-bnzmn:config-reloader component_path=/ component_id=loki.source.kubernetes.pod_logs "start time"=1970-01-01T00:00:00.000Z
ts=2025-01-30T09:00:53.52897616Z level=warn msg="error sending batch, will retry" component_path=/ component_id=loki.write.hm_loki component=client host=hm-loki-gateway.production-hm-loki:80 status=500 tenant="" error="server returned HTTP status 500 Internal Server Error (500): rpc error: code = DeadlineExceeded desc = context deadline exceeded"