CL3 csi provisioner obtize

I0216 14:10:02.789149       1 round_trippers.go:553] POST https://10.233.0.1:443/api/v1/namespaces/openstack/events 201 Created in 1 milliseconds
I0216 14:10:02.789165       1 round_trippers.go:570] HTTP Statistics: GetConnection 0 ms ServerProcessing 1 ms Duration 1 ms
I0216 14:10:02.789172       1 round_trippers.go:577] Response Headers:
I0216 14:10:02.789181       1 round_trippers.go:580]     X-Kubernetes-Pf-Flowschema-Uid: 9fad322e-e900-46c9-93c8-f55980e0364e
I0216 14:10:02.789188       1 round_trippers.go:580]     X-Kubernetes-Pf-Prioritylevel-Uid: 254dd887-da24-42e4-9b2b-1804a065dc28
I0216 14:10:02.789194       1 round_trippers.go:580]     Content-Length: 1178
I0216 14:10:02.789202       1 round_trippers.go:580]     Date: Thu, 16 Feb 2023 14:10:02 GMT
I0216 14:10:02.789208       1 round_trippers.go:580]     Audit-Id: 6c3b59c6-f04f-4654-8f79-ac7cf52d93ca
I0216 14:10:02.789216       1 round_trippers.go:580]     Cache-Control: no-cache, private
I0216 14:10:02.789223       1 round_trippers.go:580]     Content-Type: application/json
I0216 14:10:02.789250       1 request.go:1154] Response Body: {"kind":"Event","apiVersion":"v1","metadata":{"name":"mysql-data-mariadb-server-1.17445358e5a7dfbb","namespace":"openstack","uid":"680b4308-9782-40a2-8688-4446524384ca","resourceVersion":"480019","creationTimestamp":"2023-02-16T14:10:02Z","managedFields":[{"manager":"csi-provisioner","operation":"Update","apiVersion":"v1","time":"2023-02-16T14:10:02Z","fieldsType":"FieldsV1","fieldsV1":{"f:count":{},"f:firstTimestamp":{},"f:involvedObject":{},"f:lastTimestamp":{},"f:message":{},"f:reason":{},"f:source":{"f:component":{}},"f:type":{}}}]},"involvedObject":{"kind":"PersistentVolumeClaim","namespace":"openstack","name":"mysql-data-mariadb-server-1","uid":"829c9ccf-ca61-4a62-8807-a8d5538a48c4","apiVersion":"v1","resourceVersion":"477894"},"reason":"Provisioning","message":"External provisioner is provisioning volume for claim \"openstack/mysql-data-mariadb-server-1\"","source":{"component":"rook-ceph.rbd.csi.ceph.com_csi-rbdplugin-provisioner-546797bb69-rwmrq_d822d4a6-e875-44bf-b7d6-9ae098961be4"},"firstTimestamp":"2023-02-16T14:10:02Z","lastTimestamp":"2023-02-16T14:10:02Z","count":1,"type":"Normal","eventTime":null,"reportingComponent":"","reportingInstance":""}
I0216 14:10:02.883314       1 utils.go:195] ID: 18 Req-ID: pvc-51cb436d-e644-4436-83c0-535eb717421b GRPC call: /csi.v1.Controller/CreateVolume
I0216 14:10:02.883423       1 utils.go:195] ID: 19 Req-ID: pvc-829c9ccf-ca61-4a62-8807-a8d5538a48c4 GRPC call: /csi.v1.Controller/CreateVolume
I0216 14:10:02.883577       1 utils.go:206] ID: 18 Req-ID: pvc-51cb436d-e644-4436-83c0-535eb717421b GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"rook-ceph-role":"storage-node"}}],"requisite":[{"segments":{"rook-ceph-role":"storage-node"}}]},"capacity_range":{"required_bytes":5368709120},"name":"pvc-51cb436d-e644-4436-83c0-535eb717421b","parameters":{"clusterID":"rook-ceph-external","csi.storage.k8s.io/pv/name":"pvc-51cb436d-e644-4436-83c0-535eb717421b","csi.storage.k8s.io/pvc/name":"mysql-data-mariadb-server-2","csi.storage.k8s.io/pvc/namespace":"openstack","imageFeatures":"layering","imageFormat":"2","pool":"cloud-ceph-blockpool-it4i"},"secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
I0216 14:10:02.883666       1 utils.go:206] ID: 19 Req-ID: pvc-829c9ccf-ca61-4a62-8807-a8d5538a48c4 GRPC request: {"accessibility_requirements":{"preferred":[{"segments":{"rook-ceph-role":"storage-node"}}],"requisite":[{"segments":{"rook-ceph-role":"storage-node"}}]},"capacity_range":{"required_bytes":5368709120},"name":"pvc-829c9ccf-ca61-4a62-8807-a8d5538a48c4","parameters":{"clusterID":"rook-ceph-external","csi.storage.k8s.io/pv/name":"pvc-829c9ccf-ca61-4a62-8807-a8d5538a48c4","csi.storage.k8s.io/pvc/name":"mysql-data-mariadb-server-1","csi.storage.k8s.io/pvc/namespace":"openstack","imageFeatures":"layering","imageFormat":"2","pool":"cloud-ceph-blockpool-it4i"},"secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"access_mode":{"mode":1}}]}
I0216 14:10:02.883703       1 rbd_util.go:1279] ID: 18 Req-ID: pvc-51cb436d-e644-4436-83c0-535eb717421b setting disableInUseChecks: false image features: [layering] mounter: rbd
I0216 14:10:02.883782       1 rbd_util.go:1279] ID: 19 Req-ID: pvc-829c9ccf-ca61-4a62-8807-a8d5538a48c4 setting disableInUseChecks: false image features: [layering] mounter: rbd
E0216 14:10:02.887773       1 controllerserver.go:334] ID: 17 Req-ID: pvc-ee274ebf-402e-49b3-93fd-f2ff0bab0e39 failed to connect to volume : failed to get connection: connecting failed: rados: ret=-13, Permission denied
E0216 14:10:02.887811       1 utils.go:210] ID: 17 Req-ID: pvc-ee274ebf-402e-49b3-93fd-f2ff0bab0e39 GRPC error: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.887972       1 connection.go:186] GRPC response: {}
I0216 14:10:02.888047       1 connection.go:187] GRPC error: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.888071       1 controller.go:816] CreateVolume failed, supports topology = true, node selected false => may reschedule = false => state = Finished: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.888112       1 controller.go:1075] Final error received, removing PVC ee274ebf-402e-49b3-93fd-f2ff0bab0e39 from claims in progress
W0216 14:10:02.888124       1 controller.go:934] Retrying syncing claim "ee274ebf-402e-49b3-93fd-f2ff0bab0e39", failure 0
E0216 14:10:02.888152       1 controller.go:957] error syncing claim "ee274ebf-402e-49b3-93fd-f2ff0bab0e39": failed to provision volume with StorageClass "external-ceph-rbd-cloud-ceph-blockpool-it4i": rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.888184       1 event.go:285] Event(v1.ObjectReference{Kind:"PersistentVolumeClaim", Namespace:"openstack", Name:"mysql-data-mariadb-server-0", UID:"ee274ebf-402e-49b3-93fd-f2ff0bab0e39", APIVersion:"v1", ResourceVersion:"477882", FieldPath:""}): type: 'Warning' reason: 'ProvisioningFailed' failed to provision volume with StorageClass "external-ceph-rbd-cloud-ceph-blockpool-it4i": rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
E0216 14:10:02.890452       1 controllerserver.go:334] ID: 18 Req-ID: pvc-51cb436d-e644-4436-83c0-535eb717421b failed to connect to volume : failed to get connection: connecting failed: rados: ret=-13, Permission denied
E0216 14:10:02.890480       1 utils.go:210] ID: 18 Req-ID: pvc-51cb436d-e644-4436-83c0-535eb717421b GRPC error: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.888202       1 request.go:1154] Request Body: {"kind":"Event","apiVersion":"v1","metadata":{"name":"mysql-data-mariadb-server-0.17445358f8fb0d98","namespace":"openstack","creationTimestamp":null},"involvedObject":{"kind":"PersistentVolumeClaim","namespace":"openstack","name":"mysql-data-mariadb-server-0","uid":"ee274ebf-402e-49b3-93fd-f2ff0bab0e39","apiVersion":"v1","resourceVersion":"477882"},"reason":"ProvisioningFailed","message":"failed to provision volume with StorageClass \"external-ceph-rbd-cloud-ceph-blockpool-it4i\": rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied","source":{"component":"rook-ceph.rbd.csi.ceph.com_csi-rbdplugin-provisioner-546797bb69-rwmrq_d822d4a6-e875-44bf-b7d6-9ae098961be4"},"firstTimestamp":"2023-02-16T14:10:02Z","lastTimestamp":"2023-02-16T14:10:02Z","count":1,"type":"Warning","eventTime":null,"reportingComponent":"","reportingInstance":""}
I0216 14:10:02.890596       1 connection.go:186] GRPC response: {}
I0216 14:10:02.890629       1 connection.go:187] GRPC error: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied
I0216 14:10:02.890644       1 controller.go:816] CreateVolume failed, supports topology = true, node selected false => may reschedule = false => state = Finished: rpc error: code = Internal desc = failed to get connection: connecting failed: rados: ret=-13, Permission denied