1. SDC not in approved state; tenant flag = false, driver flag = true , SDC Approve Request is denied by Auth Proxy -------------------------------------------------------------------- driver logs>>>>>>> [root@master-1-ytIuu2Xkibg1Q helm]# k logs vxflexos-node-lx2hc -n vxflexos -c driver time="2023-02-20T10:40:33Z" level=info msg="configured 1a99af710210af0f" allSystemNames= endpoint="https://localhost:9400" isDefault=true password="********" skipCertificateValidation=true systemID=1a99af710210af0f user=admin time="2023-02-20T10:40:33Z" level=info msg="driver configuration file " file=/vxflexos-config-params/driver-config-params.yaml time="2023-02-20T10:40:33Z" level=info msg="Read CSI_LOG_FORMAT from log configuration file" format=text time="2023-02-20T10:40:33Z" level=info msg="Read CSI_LOG_LEVEL from log configuration file" fields.level=debug time="2023-02-20T10:40:33Z" level=info msg="array configuration file" file=/vxflexos-config/config time="2023-02-20T10:40:33Z" level=info msg="set SDC GUID" guid=1477AA2F-66B6-5F4A-B024-588726E00074 time="2023-02-20T10:40:33Z" level=info msg="Found connected system" ID=1a99af710210af0f time="2023-02-20T10:40:33Z" level=info msg="NodeInfo ID" node info=1477AA2F-66B6-5F4A-B024-588726E00074 time="2023-02-20T10:40:33Z" level=info msg="NodeInfo topologykeys" node info key=csi-vxflexos.dellemc.com/1a99af710210af0f time="2023-02-20T10:40:33Z" level=info msg="Probing all arrays. Number of arrays: 1" time="2023-02-20T10:40:33Z" level=error msg="array 1a99af710210af0f probe failed: rpc error: code = FailedPrecondition desc = unable to login to VxFlexOS Gateway: 401 Unauthorized" time="2023-02-20T10:40:33Z" level=info msg="configured csi-vxflexos.dellemc.com" IsApproveSDCEnabled=true IsHealthMonitorEnabled=false IsSdcRenameEnabled=false allowRWOMultiPodAccess=false autoprobe=true mode=node privatedir=/var/lib/kubelet/plugins/vxflexos.emc.dell.com/disks sdcGUID= sdcPrefix= thickprovision=false time="2023-02-20T10:40:33Z" level=info msg="removed sock file" path=/var/lib/kubelet/plugins/vxflexos.emc.dell.com/csi_sock time="2023-02-20T10:40:33Z" level=fatal msg="grpc failed" error="rpc error: code = FailedPrecondition desc = All arrays are not working. Could not proceed further: map[1a99af710210af0f:rpc error: code = FailedPrecondition desc = unable to login to VxFlexOS Gateway: 401 Unauthorized]" [root@master-1-ytIuu2Xkibg1Q helm]# >>>>>>>>auth proxy logs>>>>>>> time="2023-02-20T10:40:09Z" level=info msg="Serving 10.42.0.11:59254 GET /api/login/" time="2023-02-20T10:40:09Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:09Z" level=info msg="Serving 10.42.0.11:59254 GET /api/version/" time="2023-02-20T10:40:09Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:09Z" level=info msg="Serving 10.42.0.11:59254 GET /api/types/System/instances/" time="2023-02-20T10:40:09Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:09Z" level=info msg="Serving 10.42.0.11:59254 GET /api/instances/System::1a99af710210af0f/relationships/Sdc/" time="2023-02-20T10:40:09Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:09Z" level=info msg="Serving 10.42.0.11:59254 POST /api/instances/System::1a99af710210af0f/action/approveSdc/" time="2023-02-20T10:40:09Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:09Z" level=debug msg="Asking OPA..." time="2023-02-20T10:40:09Z" level=debug msg="OPA Response" opa_response="{\"result\":{\"allow\":true,\"claims\":{\"aud\":\"karavi\",\"exp\":1676889625,\"group\":\"HR\",\"iss\":\"com.dell.karavi\",\"roles\":\"HRRole\",\"sub\":\"karavi-tenant\"},\"deny\":[],\"response\":{\"allowed\":true}}}\n" time="2023-02-20T10:40:09Z" level=debug msg="proxy: writing error" code=403 message="sdc approve request denied" storage=powerflex time="2023-02-20T10:40:10Z" level=info msg="Serving 10.42.0.11:59254 GET /api/login/" time="2023-02-20T10:40:10Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:10Z" level=info msg="Serving 10.42.0.11:59254 GET /api/version/" time="2023-02-20T10:40:10Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:10Z" level=info msg="Serving 10.42.0.11:59254 GET /api/types/System/instances/" time="2023-02-20T10:40:10Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:10Z" level=info msg="Serving 10.42.0.11:59254 GET /api/instances/System::1a99af710210af0f/relationships/Sdc/" time="2023-02-20T10:40:10Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:10Z" level=info msg="Serving 10.42.0.11:59254 POST /api/instances/System::1a99af710210af0f/action/approveSdc/" time="2023-02-20T10:40:10Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:40:10Z" level=debug msg="Asking OPA..." time="2023-02-20T10:40:10Z" level=debug msg="OPA Response" opa_response="{\"result\":{\"allow\":true,\"claims\":{\"aud\":\"karavi\",\"exp\":1676889626,\"group\":\"HR\",\"iss\":\"com.dell.karavi\",\"roles\":\"HRRole\",\"sub\":\"karavi-tenant\"},\"deny\":[],\"response\":{\"allowed\":true}}}\n" time="2023-02-20T10:40:10Z" level=debug msg="proxy: writing error" code=403 message="sdc approve request denied" storage=powerflex time="2023-02-20T10:40:33Z" level=info msg="Serving 10.42.0.11:59254 GET /api/login/" time="2023-02-20T10:40:33Z" level=info msg="Serving 10.42.0.11:59254 POST /proxy/refresh-token/" time="2023-02-20T10:40:33Z" level=info msg="Refreshing token!" time="2023-02-20T10:40:36Z" level=info msg="Serving 10.42.0.11:59254 GET /api/login/" time="2023-02-20T10:40:36Z" level=info msg="Serving 10.42.0.11:59254 POST /proxy/refresh-token/" time="2023-02-20T10:40:36Z" level=info msg="Refreshing token!" [root@LGLW9242 ~]# --------------------------------------------------------------------------------------------------------------------------------------------- 2. SDC not in apprived state , tenant flag=true, driver flag=true, DC Approve Request is passed by Auth Proxy [root@master-1-ytIuu2Xkibg1Q dell-csi-helm-installer]# k logs vxflexos-node-h46w8 -n vxflexos -c driver time="2023-02-20T10:49:49Z" level=info msg="configured 1a99af710210af0f" allSystemNames= endpoint="https://localhost:9400" isDefault=true password="********" skipCertificateValidation=true systemID=1a99af710210af0f user=admin time="2023-02-20T10:49:49Z" level=info msg="driver configuration file " file=/vxflexos-config-params/driver-config-params.yaml time="2023-02-20T10:49:49Z" level=info msg="Read CSI_LOG_FORMAT from log configuration file" format=text time="2023-02-20T10:49:49Z" level=info msg="Read CSI_LOG_LEVEL from log configuration file" fields.level=debug time="2023-02-20T10:49:49Z" level=info msg="array configuration file" file=/vxflexos-config/config time="2023-02-20T10:49:49Z" level=info msg="set SDC GUID" guid=80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D time="2023-02-20T10:49:49Z" level=info msg="Found connected system" ID=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="NodeInfo ID" node info=80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D time="2023-02-20T10:49:49Z" level=info msg="NodeInfo topologykeys" node info key=csi-vxflexos.dellemc.com/1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Probing all arrays. Number of arrays: 1" time="2023-02-20T10:49:49Z" level=info msg="Found Name for system=1a99af710210af0f_pflex_system with ID=1a99af710210af0f" time="2023-02-20T10:49:49Z" level=info msg="default array is set to array ID: 1a99af710210af0f" time="2023-02-20T10:49:49Z" level=info msg="1a99af710210af0f is the default array, skipping VolumePrefixToSystems map update. \n" time="2023-02-20T10:49:49Z" level=info msg="array 1a99af710210af0f probed successfully" time="2023-02-20T10:49:49Z" level=info msg="set SDC GUID" guid=80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D time="2023-02-20T10:49:49Z" level=info msg="Approve SDC enabled" time="2023-02-20T10:49:49Z" level=info msg="SDC Approved, SDC Id: 2a2a4a4a00000780 and SDC GUID: 80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D" time="2023-02-20T10:49:49Z" level=info msg="configured csi-vxflexos.dellemc.com" IsApproveSDCEnabled=true IsHealthMonitorEnabled=false IsSdcRenameEnabled=false allowRWOMultiPodAccess=false autoprobe=true mode=node privatedir=/var/lib/kubelet/plugins/vxflexos.emc.dell.com/disks sdcGUID=80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D sdcPrefix= thickprovision=false time="2023-02-20T10:49:49Z" level=info msg="identity service registered" time="2023-02-20T10:49:49Z" level=info msg="node service registered" time="2023-02-20T10:49:49Z" level=info msg="Registering additional GRPC servers" time="2023-02-20T10:49:49Z" level=info msg=serving endpoint="unix:///var/lib/kubelet/plugins/vxflexos.emc.dell.com/csi_sock" time="2023-02-20T10:49:49Z" level=info msg="/csi.v1.Identity/GetPluginInfo: REQ 0001: XXX_NoUnkeyedLiteral={}, XXX_sizecache=0" time="2023-02-20T10:49:49Z" level=info msg="/csi.v1.Identity/GetPluginInfo: REP 0001: Name=csi-vxflexos.dellemc.com, VendorVersion=2.5.0+18+dirty, Manifest=map[commit:ae9719b77a034f2dd81fba19544d189a031c8633 formed:Fri, 03 Feb 2023 05:46:05 UTC semver:2.5.0+18+dirty url:http://github.com/dell/csi-vxflexos], XXX_NoUnkeyedLiteral={}, XXX_sizecache=0" time="2023-02-20T10:49:50Z" level=info msg="/csi.v1.Node/NodeGetInfo: REQ 0002: XXX_NoUnkeyedLiteral={}, XXX_sizecache=0" time="2023-02-20T10:49:50Z" level=info msg="/csi.v1.Node/NodeGetInfo: REP 0002: NodeId=80BAFB1F-C8D1-5A5A-ADE2-12CC0767612D, MaxVolumesPerNode=0, AccessibleTopology=segments: , XXX_NoUnkeyedLiteral={}, XXX_sizecache=0" [root@master-1-ytIuu2Xkibg1Q dell-csi-helm-installer]# proxy logs>>>>>>>> time="2023-02-20T10:45:44Z" level=info msg="Serving 10.42.0.11:60736 POST /proxy/refresh-token/" time="2023-02-20T10:45:44Z" level=info msg="Refreshing token!" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 POST /proxy/refresh-token/" time="2023-02-20T10:49:35Z" level=info msg="Refreshing token!" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 POST /proxy/refresh-token/" time="2023-02-20T10:49:35Z" level=info msg="Refreshing token!" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 POST /proxy/refresh-token/" time="2023-02-20T10:49:35Z" level=info msg="Refreshing token!" time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:35Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/version/" time="2023-02-20T10:49:35Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:35Z" level=info msg="Serving 10.42.0.11:33474 GET /api/types/System/instances/" time="2023-02-20T10:49:35Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/version/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/types/System/instances/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/instances/System::1a99af710210af0f/relationships/Sdc/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 POST /api/instances/System::1a99af710210af0f/action/approveSdc/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=debug msg="Asking OPA..." time="2023-02-20T10:49:49Z" level=debug msg="OPA Response" opa_response="{\"result\":{\"allow\":true,\"claims\":{\"aud\":\"karavi\",\"exp\":1676890205,\"group\":\"HR\",\"iss\":\"com.dell.karavi\",\"roles\":\"HRRole\",\"sub\":\"karavi-tenant\"},\"deny\":[],\"response\":{\"allowed\":true}}}\n" time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/login/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/version/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/types/System/instances/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 GET /api/instances/System::1a99af710210af0f/relationships/Sdc/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=info msg="Serving 10.42.0.11:33474 POST /api/instances/System::1a99af710210af0f/action/approveSdc/" time="2023-02-20T10:49:49Z" level=debug msg="Serving request" endpoint="https://10.247.13.206" system_id=1a99af710210af0f time="2023-02-20T10:49:49Z" level=debug msg="Asking OPA..." time="2023-02-20T10:49:49Z" level=debug msg="OPA Response" opa_response="{\"result\":{\"allow\":true,\"claims\":{\"aud\":\"karavi\",\"exp\":1676890205,\"group\":\"HR\",\"iss\":\"com.dell.karavi\",\"roles\":\"HRRole\",\"sub\":\"karavi-tenant\"},\"deny\":[],\"response\":{\"allowed\":true}}}\n" [root@LGLW9242 ~]#