psql-operator-bug

operator-bug

IN PSQL OPERATOR


time="2021-04-19T23:37:27Z" level=info msg="SYNC event has been queued" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=controller worker=1
time="2021-04-19T23:37:27Z" level=info msg="there are 1 clusters running" pkg=controller
time="2021-04-19T23:37:27Z" level=info msg="syncing of the cluster started" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=controller worker=1
time="2021-04-19T23:37:27Z" level=debug msg="team API is disabled" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="syncing secrets" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="secret try30/galaxydbuser.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do already exists, fetching its password" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="secret try30/standby.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do already exists, fetching its password" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="secret try30/postgres.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do already exists, fetching its password" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="syncing master service" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="syncing replica service" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="No load balancer created for the replica service" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="syncing volumes using \"pvc\" storage resize mode" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="volume claims do not require changes" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="syncing statefulsets" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="cached StatefulSet value exists, rollingUpdate flag is true" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="Generating Spilo container, environment variables" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="[{SCOPE galaxy-try30-gxy-rls-galaxy-postgres nil} {PGROOT /home/postgres/pgdata/pgroot nil} {POD_IP &EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:status.podIP,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,}} {POD_NAMESPACE &EnvVarSource{FieldRef:&ObjectFieldSelector{APIVersion:v1,FieldPath:metadata.namespace,},ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:nil,}} {PGUSER_SUPERUSER postgres nil} {KUBERNETES_SCOPE_LABEL cluster-name nil} {KUBERNETES_ROLE_LABEL spilo-role nil} {PGPASSWORD_SUPERUSER &EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:postgres.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do,},Key:password,Optional:nil,},}} {PGUSER_STANDBY standby nil} {PGPASSWORD_STANDBY &EnvVarSource{FieldRef:nil,ResourceFieldRef:nil,ConfigMapKeyRef:nil,SecretKeyRef:&SecretKeySelector{LocalObjectReference:LocalObjectReference{Name:standby.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do,},Key:password,Optional:nil,},}} {PAM_OAUTH2 https://info.example.com/oauth2/tokeninfo?access_token= uid realm=/employees nil} {HUMAN_ROLE zalandos nil} {PGVERSION 13 nil} {KUBERNETES_LABELS {\"application\":\"spilo\"} nil} {SPILO_CONFIGURATION {\"postgresql\":{},\"bootstrap\":{\"initdb\":[{\"auth-host\":\"md5\"},{\"auth-local\":\"trust\"}],\"users\":{\"zalandos\":{\"password\":\"\",\"options\":[\"CREATEDB\",\"NOLOGIN\"]}},\"dcs\":{}}} nil} {DCS_ENABLE_KUBERNETES_API true nil}]" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="set statefulset's rolling update annotation to false: caller/reason from cache" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="set statefulset's rolling update annotation to true: caller/reason statefulset changes" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="statefulset try30/galaxy-try30-gxy-rls-galaxy-postgres is not in the desired state and needs to be updated" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- terminationMessagePath: /dev/termination-log," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- terminationMessagePolicy: File," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- restartPolicy: Always," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- dnsPolicy: ClusterFirst," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- serviceAccount: postgres-pod," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- securityContext: {}," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- schedulerName: default-scheduler" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="+ securityContext: {}" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- kind: PersistentVolumeClaim," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- apiVersion: v1," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- status: {" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- phase: Pending" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- }" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="+ status: {}" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- }," cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- revisionHistoryLimit: 10" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="+ }" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="metadata.annotation are different" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="- zalando-postgres-operator-rolling-update-required: false" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="+ zalando-postgres-operator-rolling-update-required: true" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="reason: new statefulset containers's postgres (index 0) security context does not match the current one" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="updating statefulset" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="patching statefulset annotations" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="patching statefulset annotations" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="performing rolling update" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="there are 1 pods in the cluster to recreate" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="name=galaxy-try30-gxy-rls-galaxy-postgres-0 phase=Running ip=10.42.0.172" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=warning msg="cannot perform switch over before re-creating the pod: no replicas" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=info msg="recreating old master pod \"try30/galaxy-try30-gxy-rls-galaxy-postgres-0\"" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:27Z" level=debug msg="subscribing to pod \"try30/galaxy-try30-gxy-rls-galaxy-postgres-0\"" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=info msg="pod \"try30/galaxy-try30-gxy-rls-galaxy-postgres-0\" has been recreated" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=debug msg="unsubscribing from pod \"try30/galaxy-try30-gxy-rls-galaxy-postgres-0\" events" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=info msg="pods have been recreated" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=debug msg="set statefulset's rolling update annotation to false: caller/reason applyRollingUpdateFlag" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=debug msg="patching statefulset annotations" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=debug msg="syncing pod disruption budgets" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:37:54Z" level=debug msg="syncing roles" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=warning msg="could not connect to Postgres database: dial tcp 10.43.149.191:5432: i/o timeout" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="closing database connection" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="syncing databases" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="closing database connection" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="syncing prepared databases with schemas" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="syncing connection pooler from (nil, nil) to (nil, nil)" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=debug msg="could not get connection pooler secret pooler.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do: secrets \"pooler.galaxy-try30-gxy-rls-galaxy-postgres.credentials.postgresql.acid.zalan.do\" not found" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=cluster worker=1
time="2021-04-19T23:38:09Z" level=info msg="cluster has been synced" cluster-name=try30/galaxy-try30-gxy-rls-galaxy-postgres pkg=controller worker=1



IN KUBEAPI


I0419 22:29:53.493039 1 passthrough.go:48] ccResolverWrapper: sending update to cc: {[{https://127.0.0.1:2379 <nil> 0 <nil>}] <nil> <nil>}
I0419 22:29:53.493060 1 clientconn.go:948] ClientConn switching balancer to "pick_first"
E0419 22:30:27.007246 1 repair.go:238] the cluster IP 10.43.2.216 for service galaxy-cancer-galaxy-galaxy-postgres/galaxy is not allocated; repairing
E0419 22:30:27.007300 1 repair.go:238] the cluster IP 10.43.50.202 for service galaxy-cancer-galaxy-galaxy-postgres-repl/galaxy is not allocated; repairing
E0419 22:30:27.022110 1 event.go:264] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"galaxy-cancer-galaxy-galaxy-postgres.167762fc57a3357e", GenerateName:"", Namespace:"galaxy", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Service", Namespace:"galaxy", Name:"galaxy-cancer-galaxy-galaxy-postgres", UID:"ab517e60-9860-4db0-abab-f263904a73f4", APIVersion:"v1", ResourceVersion:"706111", FieldPath:""}, Reason:"ClusterIPNotAllocated", Message:"Cluster IP 10.43.2.216 is not allocated; repairing", Source:v1.EventSource{Component:"ipallocator-repair-controller", Host:""}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0179f00c045d77e, ext:7065379535602, loc:(*time.Location)(0x70afb40)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0179f00c045d77e, ext:7065379535602, loc:(*time.Location)(0x70afb40)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "galaxy-cancer-galaxy-galaxy-postgres.167762fc57a3357e" is forbidden: unable to create new content in namespace galaxy because it is being terminated' (will not retry!)
E0419 22:30:27.024350 1 event.go:264] Server rejected event '&v1.Event{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"galaxy-cancer-galaxy-galaxy-postgres-repl.167762fc57cc742b", GenerateName:"", Namespace:"galaxy", SelfLink:"", UID:"", ResourceVersion:"", Generation:0, CreationTimestamp:v1.Time{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, DeletionTimestamp:(*v1.Time)(nil), DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ClusterName:"", ManagedFields:[]v1.ManagedFieldsEntry(nil)}, InvolvedObject:v1.ObjectReference{Kind:"Service", Namespace:"galaxy", Name:"galaxy-cancer-galaxy-galaxy-postgres-repl", UID:"43d889de-d8cb-42c3-a159-1f31dbbc8762", APIVersion:"v1", ResourceVersion:"706113", FieldPath:""}, Reason:"ClusterIPNotAllocated", Message:"Cluster IP 10.43.50.202 is not allocated; repairing", Source:v1.EventSource{Component:"ipallocator-repair-controller", Host:""}, FirstTimestamp:v1.Time{Time:time.Time{wall:0xc0179f00c06f162b, ext:7065382238614, loc:(*time.Location)(0x70afb40)}}, LastTimestamp:v1.Time{Time:time.Time{wall:0xc0179f00c06f162b, ext:7065382238614, loc:(*time.Location)(0x70afb40)}}, Count:1, Type:"Warning", EventTime:v1.MicroTime{Time:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}}, Series:(*v1.EventSeries)(nil), Action:"", Related:(*v1.ObjectReference)(nil), ReportingController:"", ReportingInstance:""}': 'events "galaxy-cancer-galaxy-galaxy-postgres-repl.167762fc57cc742b" is forbidden: unable to create new content in namespace galaxy because it is being terminated' (will not retry!)
W0419 22:30:30.054139 1 handler_proxy.go:102] no RequestInfo found in the context
E0419 22:30:30.058155 1 controller.go:116] loading OpenAPI spec for "v1beta1.metrics.k8s.io" failed with: failed to retrieve openAPI spec, http error: ResponseCode: 503, Body: service unavailable
, Header: map[Content-Type:[text/plain; charset=utf-8] X-Content-Type-Options:[nosniff]]