diff --git a/pkg/rolling/restart_handler.go b/pkg/rolling/restart_handler.go index 8ce3dee..0ab5a08 100644 --- a/pkg/rolling/restart_handler.go +++ b/pkg/rolling/restart_handler.go @@ -89,9 +89,9 @@ func (rh *restartHandler) run() { } } -func (rh *restartHandler) stop(shouldWaitForDelay bool) { +func (rh *restartHandler) stop(waitForDelay bool) { close(rh.queue) - if shouldWaitForDelay { + if waitForDelay { rh.wg.Wait() } close(rh.done) diff --git a/pkg/rolling/rolling.go b/pkg/rolling/rolling.go index 1f89a51..4606e64 100644 --- a/pkg/rolling/rolling.go +++ b/pkg/rolling/rolling.go @@ -372,9 +372,9 @@ func (r *Rolling) processActionGroupStates(actions []*Ydb_Maintenance.ActionGrou r.state.unreportedButFinishedActionIds = []string{} restartCompleted := len(actions) == len(result.ActionStatuses) - shouldWaitForDelay := !restartCompleted + waitForDelay := !restartCompleted - restartHandler.stop(shouldWaitForDelay) + restartHandler.stop(waitForDelay) return restartCompleted } diff --git a/sammyvimes.txt b/sammyvimes.txt new file mode 100644 index 0000000..7a3d6e8 --- /dev/null +++ b/sammyvimes.txt @@ -0,0 +1,4 @@ +2024-12-25T18:13:51.105+0100 debug Current logging level enabled: debug +2024-12-25T18:13:51.105+0100 debug Authentication type not specified, implicitly assuming type: iam-token +2024-12-25T18:13:51.105+0100 info Start rolling restart +2024-12-25T18:13:51.105+0100 debug Invoke ListClusterNodes method diff --git a/sammyvimes2.txt b/sammyvimes2.txt new file mode 100644 index 0000000..874a6fd --- /dev/null +++ b/sammyvimes2.txt @@ -0,0 +1,2234 @@ +2024-12-25T18:23:03.432+0100 debug Current logging level enabled: debug +2024-12-25T18:23:03.432+0100 debug Authentication type not specified, implicitly assuming type: iam-token +2024-12-25T18:23:03.432+0100 info Start rolling restart +2024-12-25T18:23:03.432+0100 debug Invoke ListClusterNodes method +2024-12-25T18:23:04.802+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:04.802+0100 debug Invoke ListDatabases method +2024-12-25T18:23:05.724+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:05.724+0100 debug Invoke WhoAmI method +2024-12-25T18:23:06.348+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:06.348+0100 debug WhoAmI response: serviceaccount-e0tv2pwt7nw675626d@as +2024-12-25T18:23:06.348+0100 debug Will cleanup all previous maintenance tasks... +2024-12-25T18:23:06.348+0100 debug Invoke ListMaintenanceTasks method +2024-12-25T18:23:07.362+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:07.362+0100 debug Invoke GetMaintenanceTask method +2024-12-25T18:23:08.056+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:08.056+0100 debug Invoke DropMaintenanceTask method +2024-12-25T18:23:09.304+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:09.848+0100 debug Storage K8s restarter selected following nodes for restart: [node_id:5 host:"man0-0306.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-B" module:"DC=MAN-B/" rack:"DC=MAN-B/B-MAN-3#3D14" unit:"DC=MAN-B/R=B-MAN-3#3D14/108071404"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735065325 nanos:997000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:8 host:"man0-0304.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-C" module:"DC=MAN-C/" rack:"DC=MAN-C/B-MAN-3#3C16" unit:"DC=MAN-C/R=B-MAN-3#3C16/108270939"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735063888 nanos:884000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:9 host:"man0-0309.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-C" module:"DC=MAN-C/" rack:"DC=MAN-C/B-MAN-3#3C17" unit:"DC=MAN-C/R=B-MAN-3#3C17/108270938"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735147251 nanos:704000000} version:"experiments.f8fce6c" node_id:7 host:"man0-0305.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-C" module:"DC=MAN-C/" rack:"DC=MAN-C/B-MAN-3#3C15" unit:"DC=MAN-C/R=B-MAN-3#3C15/108071403"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735063733 nanos:538000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:6 host:"man0-0307.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-B" module:"DC=MAN-B/" rack:"DC=MAN-B/B-MAN-3#3D13" unit:"DC=MAN-B/R=B-MAN-3#3D13/108075384"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735147318 nanos:532000000} version:"experiments.f8fce6c" node_id:3 host:"man0-0185.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-A" module:"DC=MAN-A/" rack:"DC=MAN-A/B-MAN-3#3D10" unit:"DC=MAN-A/R=B-MAN-3#3D10/108075381"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735063747 nanos:994000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:4 host:"man0-0308.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-B" module:"DC=MAN-B/" rack:"DC=MAN-B/B-MAN-3#3D12" unit:"DC=MAN-B/R=B-MAN-3#3D12/108044251"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735065326 nanos:940000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:2 host:"man0-0287.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-A" module:"DC=MAN-A/" rack:"DC=MAN-A/B-MAN-3#3D11" unit:"DC=MAN-A/R=B-MAN-3#3D11/108075385"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735063905 nanos:314000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b" node_id:1 host:"man0-0310.ydb-dev.ik8s.nebiuscloud.net" port:19001 location:{data_center:"MAN-A" module:"DC=MAN-A/" rack:"DC=MAN-A/B-MAN-3#3D9" unit:"DC=MAN-A/R=B-MAN-3#3D9/108061102"} state:ITEM_STATE_UP storage:{} start_time:{seconds:1735063992 nanos:895000000} version:"24.3.11-hotfix.7.stream-nb-24-3.1.029c82b"] +seconds:1500 +2024-12-25T18:23:09.848+0100 debug Invoke CreateMaintenanceTask method +2024-12-25T18:23:11.427+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:11.427+0100 info Maintenance task rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba, processing loop started +2024-12-25T18:23:11.427+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba + Lock on node 8 PERFORMED, until: 2024-12-25 17:48:11 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 9 PENDING, ACTION_REASON_GENERIC + Lock on node 7 PENDING, ACTION_REASON_GENERIC + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:23:11.428+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:8} duration:{seconds:1500}}} action_uid:{task_uid:"rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba" action_id:"serviceaccount-e0tv2pwt7nw675626d@as-p-287"} status:ACTION_STATUS_PERFORMED reason:ACTION_REASON_OK deadline:{seconds:1735148891 nanos:386263000}} action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:9} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_GENERIC reason_details:"You cannot get two or more disks from the same group at the same time without specifying the PartialPermissionAllowed parameter"} action_states:{action:{lock_action:{scope:{node_id:7} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_GENERIC reason_details:"You cannot get two or more disks from the same group at the same time without specifying the PartialPermissionAllowed parameter"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) has temporary lock, VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request, Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) has temporary lock. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) has temporary lock, VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) has temporary lock, VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request, Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) has temporary lock. Down: VDisk [820001b5:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down, VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down"}] +2024-12-25T18:23:11.428+0100 info 1 ActionGroupStates moved to PERFORMED, will restart now... +2024-12-25T18:23:11.428+0100 debug Restart node with id: 8 +2024-12-25T18:23:11.428+0100 debug Drain node with id: 8 +2024-12-25T18:23:11.428+0100 info DRAINING NOT IMPLEMENTED YET +2024-12-25T18:23:11.428+0100 debug Restart node with id: 8 +2024-12-25T18:23:11.428+0100 info Restarting pod s3-dev-storage-8 on the man0-0304.ydb-dev.ik8s.nebiuscloud.net node +2024-12-25T18:23:11.627+0100 debug Pod s3-dev-storage-8 id: 34291020-87fd-4d5e-b229-55c11453d937 +2024-12-25T18:23:11.759+0100 debug Invoke CompleteAction method +2024-12-25T18:23:12.361+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:12.361+0100 debug Manage action result: + Completed action id: serviceaccount-e0tv2pwt7nw675626d@as-p-287, status: SUCCESS +2024-12-25T18:23:12.361+0100 info Wait next 10s delay. Total node progress: 1 out of 9 +2024-12-25T18:23:22.370+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:23:22.370+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:23:24.973+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:24.974+0100 debug Invoke ListClusterNodes method +2024-12-25T18:23:26.486+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:26.487+0100 warn No incompatible versions have been detected. However, 3 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:23:26.487+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba + Lock on node 9 PERFORMED, until: 2024-12-25 17:48:24 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 7 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + Lock on node 6 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + Lock on node 3 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + Lock on node 4 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + Lock on node 2 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + Lock on node 1 PENDING, ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED + +2024-12-25T18:23:26.487+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:9} duration:{seconds:1500}}} action_uid:{task_uid:"rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba" action_id:"serviceaccount-e0tv2pwt7nw675626d@as-p-288"} status:ACTION_STATUS_PERFORMED reason:ACTION_REASON_OK deadline:{seconds:1735148904 nanos:912780000}} action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0304.ydb-dev.ik8s.nebiuscloud.net:19001 (8) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:7} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '7' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '6' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '3' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '4' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '2' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TENANT_DISABLED_NODES_LIMIT_REACHED reason_details:"Cannot lock node '1' of tenant '/s3-dev': too many unavailable nodes. Locked: 1, down: 1, total: 9, limit: 30%"}] +2024-12-25T18:23:26.487+0100 info 1 ActionGroupStates moved to PERFORMED, will restart now... +2024-12-25T18:23:26.487+0100 debug Restart node with id: 9 +2024-12-25T18:23:26.487+0100 debug Drain node with id: 9 +2024-12-25T18:23:26.487+0100 info DRAINING NOT IMPLEMENTED YET +2024-12-25T18:23:26.487+0100 debug Restart node with id: 9 +2024-12-25T18:23:26.487+0100 info Restarting pod s3-dev-storage-0 on the man0-0309.ydb-dev.ik8s.nebiuscloud.net node +2024-12-25T18:23:26.681+0100 debug Pod s3-dev-storage-0 id: d7883e08-c0c3-430c-8ab1-dd08ac8228ec +2024-12-25T18:23:26.804+0100 debug Invoke CompleteAction method +2024-12-25T18:23:27.113+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:27.113+0100 debug Retrying after 1 seconds... + +2024-12-25T18:23:28.370+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:28.370+0100 debug Retrying after 2 seconds... + +2024-12-25T18:23:30.634+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:30.634+0100 debug Retrying after 4 seconds... + +2024-12-25T18:23:35.006+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:35.007+0100 debug Retrying after 8 seconds... + +2024-12-25T18:23:44.638+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:23:44.638+0100 debug Manage action result: + Completed action id: serviceaccount-e0tv2pwt7nw675626d@as-p-288, status: SUCCESS +2024-12-25T18:23:44.638+0100 info Wait next 10s delay. Total node progress: 2 out of 9 +2024-12-25T18:23:54.643+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:23:54.643+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:23:54.980+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:54.980+0100 debug Retrying after 1 seconds... + +2024-12-25T18:23:56.121+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:56.122+0100 debug Retrying after 2 seconds... + +2024-12-25T18:23:58.320+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:23:58.320+0100 debug Retrying after 4 seconds... + +2024-12-25T18:24:02.653+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:02.654+0100 debug Retrying after 8 seconds... + +2024-12-25T18:24:10.946+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:10.947+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:10.947+0100 info Wait next 10s delay. Total node progress: 2 out of 9 +2024-12-25T18:24:20.955+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:24:20.955+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:24:21.162+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:21.162+0100 debug Retrying after 1 seconds... + +2024-12-25T18:24:22.328+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:22.328+0100 debug Retrying after 2 seconds... + +2024-12-25T18:24:24.760+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:24.760+0100 debug Retrying after 4 seconds... + +2024-12-25T18:24:29.306+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:29.306+0100 debug Retrying after 8 seconds... + +2024-12-25T18:24:37.687+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:37.687+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: authentication handshake failed: EOF" +2024-12-25T18:24:37.687+0100 info Wait next 10s delay. Total node progress: 2 out of 9 +2024-12-25T18:24:47.691+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:24:47.691+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:24:47.691+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:24:47.692+0100 debug Retrying after 1 seconds... + +2024-12-25T18:24:48.693+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:24:48.693+0100 debug Retrying after 2 seconds... + +2024-12-25T18:24:50.694+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:24:50.694+0100 debug Retrying after 4 seconds... + +2024-12-25T18:24:54.697+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:24:54.697+0100 debug Retrying after 8 seconds... + +2024-12-25T18:25:02.704+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:25:02.704+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:25:02.704+0100 info Wait next 10s delay. Total node progress: 2 out of 9 +2024-12-25T18:25:12.713+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:25:12.713+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:25:12.714+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:25:12.714+0100 debug Retrying after 1 seconds... + +2024-12-25T18:25:13.715+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:25:13.715+0100 debug Retrying after 2 seconds... + +2024-12-25T18:25:17.103+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:17.103+0100 debug Invoke ListClusterNodes method +2024-12-25T18:25:18.950+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:18.951+0100 warn No incompatible versions have been detected. However, 3 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:25:18.951+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba + Lock on node 7 PERFORMED, until: 2024-12-25 17:50:17 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_GENERIC + +2024-12-25T18:25:18.951+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:7} duration:{seconds:1500}}} action_uid:{task_uid:"rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba" action_id:"serviceaccount-e0tv2pwt7nw675626d@as-p-289"} status:ACTION_STATUS_PERFORMED reason:ACTION_REASON_OK deadline:{seconds:1735149017 nanos:53589000}} action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: PDisk 8:27 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) has temporary lock. Down: PDisk 8:27 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) has temporary lock. Down: PDisk 8:27 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) has temporary lock. Down: PDisk 8:27 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) has temporary lock. Down: PDisk 8:27 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_GENERIC reason_details:"You cannot get two or more disks from the same group at the same time without specifying the PartialPermissionAllowed parameter"}] +2024-12-25T18:25:18.951+0100 info 1 ActionGroupStates moved to PERFORMED, will restart now... +2024-12-25T18:25:18.951+0100 debug Restart node with id: 7 +2024-12-25T18:25:18.951+0100 debug Drain node with id: 7 +2024-12-25T18:25:18.951+0100 info DRAINING NOT IMPLEMENTED YET +2024-12-25T18:25:18.951+0100 debug Restart node with id: 7 +2024-12-25T18:25:18.951+0100 info Restarting pod s3-dev-storage-5 on the man0-0305.ydb-dev.ik8s.nebiuscloud.net node +2024-12-25T18:25:19.562+0100 debug Pod s3-dev-storage-5 id: a0037812-bad5-4358-8159-fb2d2e215810 +2024-12-25T18:25:19.782+0100 debug Invoke CompleteAction method +2024-12-25T18:25:20.426+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:20.426+0100 debug Manage action result: + Completed action id: serviceaccount-e0tv2pwt7nw675626d@as-p-289, status: SUCCESS +2024-12-25T18:25:20.426+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:25:30.435+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:25:30.435+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:25:31.927+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:31.927+0100 debug Invoke ListClusterNodes method +2024-12-25T18:25:33.381+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:33.383+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:25:33.383+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:30:31 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:25:33.383+0100 debug Task has retry after attribute: 2024-12-25 17:30:31 +2024-12-25T18:25:33.383+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:25:33.383+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:2:1:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:25:33.383+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:25:43.387+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:25:43.387+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:25:45.301+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:45.301+0100 debug Invoke ListClusterNodes method +2024-12-25T18:25:46.491+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:46.491+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:25:46.491+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:30:45 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:25:46.491+0100 debug Task has retry after attribute: 2024-12-25 17:30:45 +2024-12-25T18:25:46.492+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:25:46.492+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:25:46.492+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:25:56.495+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:25:56.495+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:25:58.271+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:58.272+0100 debug Invoke ListClusterNodes method +2024-12-25T18:25:59.920+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:25:59.921+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:25:59.921+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:30:58 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:25:59.921+0100 debug Task has retry after attribute: 2024-12-25 17:30:58 +2024-12-25T18:25:59.921+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:25:59.921+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, PDisk 8:1021 (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:25:59.921+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:26:09.925+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:26:09.925+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:26:11.346+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:11.346+0100 debug Invoke ListClusterNodes method +2024-12-25T18:26:12.812+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:12.813+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:26:12.813+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:31:11 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:26:12.813+0100 debug Task has retry after attribute: 2024-12-25 17:31:11 +2024-12-25T18:26:12.813+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:26:12.813+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:26:12.813+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:26:22.814+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:26:22.814+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:26:24.383+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:24.383+0100 debug Invoke ListClusterNodes method +2024-12-25T18:26:26.007+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:26.008+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:26:26.008+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:31:24 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:26:26.008+0100 debug Task has retry after attribute: 2024-12-25 17:31:24 +2024-12-25T18:26:26.008+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:26:26.008+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:26:26.008+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:26:36.012+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:26:36.012+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:26:37.411+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:37.411+0100 debug Invoke ListClusterNodes method +2024-12-25T18:26:39.026+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:39.027+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:26:39.028+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:31:37 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:26:39.028+0100 debug Task has retry after attribute: 2024-12-25 17:31:37 +2024-12-25T18:26:39.028+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:26:39.028+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:26:39.028+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:26:49.033+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:26:49.033+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:26:50.444+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:50.444+0100 debug Invoke ListClusterNodes method +2024-12-25T18:26:51.930+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:26:51.931+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:26:51.931+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:31:50 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:26:51.931+0100 debug Task has retry after attribute: 2024-12-25 17:31:50 +2024-12-25T18:26:51.931+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:26:51.931+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:26:51.931+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:27:01.932+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:27:01.932+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:27:03.571+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:03.571+0100 debug Invoke ListClusterNodes method +2024-12-25T18:27:05.210+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:05.211+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:27:05.211+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:32:03 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:27:05.211+0100 debug Task has retry after attribute: 2024-12-25 17:32:03 +2024-12-25T18:27:05.211+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:27:05.211+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:27:05.211+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:27:15.215+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:27:15.215+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:27:16.600+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:16.600+0100 debug Invoke ListClusterNodes method +2024-12-25T18:27:18.097+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:18.098+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:27:18.098+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:32:16 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:27:18.098+0100 debug Task has retry after attribute: 2024-12-25 17:32:16 +2024-12-25T18:27:18.098+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:27:18.098+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:27:18.098+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:27:28.103+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:27:28.103+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:27:29.619+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:29.619+0100 debug Invoke ListClusterNodes method +2024-12-25T18:27:31.781+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:31.782+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:27:31.782+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:32:29 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:27:31.782+0100 debug Task has retry after attribute: 2024-12-25 17:32:29 +2024-12-25T18:27:31.782+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:27:31.782+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:27:31.782+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:27:41.787+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:27:41.787+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:27:43.417+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:43.417+0100 debug Invoke ListClusterNodes method +2024-12-25T18:27:45.173+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:45.174+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:27:45.174+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:32:43 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:27:45.174+0100 debug Task has retry after attribute: 2024-12-25 17:32:43 +2024-12-25T18:27:45.174+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, PDisk 7:1021 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:27:45.174+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, PDisk 7:1021 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:27:45.174+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:27:55.184+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:27:55.184+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:27:56.559+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:56.559+0100 debug Invoke ListClusterNodes method +2024-12-25T18:27:58.193+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:27:58.194+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:27:58.194+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:32:56 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:27:58.194+0100 debug Task has retry after attribute: 2024-12-25 17:32:56 +2024-12-25T18:27:58.194+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:27:58.194+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:27:58.194+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:28:08.199+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:28:08.199+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:28:09.551+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:09.551+0100 debug Invoke ListClusterNodes method +2024-12-25T18:28:10.880+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:10.881+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:28:10.881+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:33:09 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:28:10.881+0100 debug Task has retry after attribute: 2024-12-25 17:33:09 +2024-12-25T18:28:10.881+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:28:10.881+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:28:10.881+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:28:20.887+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:28:20.887+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:28:22.369+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:22.369+0100 debug Invoke ListClusterNodes method +2024-12-25T18:28:24.092+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:24.093+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:28:24.093+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:33:22 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:28:24.093+0100 debug Task has retry after attribute: 2024-12-25 17:33:22 +2024-12-25T18:28:24.093+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:28:24.093+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:0:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR10) is down, VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:28:24.093+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:28:34.097+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:28:34.097+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:28:35.472+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:35.472+0100 debug Invoke ListClusterNodes method +2024-12-25T18:28:37.018+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:37.019+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:28:37.019+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:33:35 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:28:37.019+0100 debug Task has retry after attribute: 2024-12-25 17:33:35 +2024-12-25T18:28:37.019+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:28:37.019+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:28:37.019+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:28:47.023+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:28:47.023+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:28:48.366+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:48.367+0100 debug Invoke ListClusterNodes method +2024-12-25T18:28:49.899+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:28:49.900+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:28:49.901+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:33:48 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:28:49.901+0100 debug Task has retry after attribute: 2024-12-25 17:33:48 +2024-12-25T18:28:49.901+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:28:49.901+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:28:49.901+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:28:59.901+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:28:59.901+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:29:01.207+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:29:01.207+0100 debug Invoke ListClusterNodes method +2024-12-25T18:29:03.426+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:29:03.427+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:29:03.427+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:34:01 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:29:03.427+0100 debug Task has retry after attribute: 2024-12-25 17:34:01 +2024-12-25T18:29:03.427+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:29:03.427+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:29:03.427+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:29:13.432+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:29:13.432+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:29:14.983+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:29:14.983+0100 debug Invoke ListClusterNodes method +2024-12-25T18:29:16.276+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:29:16.277+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:29:16.277+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:34:14 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:29:16.277+0100 debug Task has retry after attribute: 2024-12-25 17:34:14 +2024-12-25T18:29:16.277+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down"}] +2024-12-25T18:29:16.277+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038535': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001c7:2:2:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR12) is locked by this request. Down: VDisk [820001c7:2:0:2:0] (man0-0304.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is down, VDisk [820001c7:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR24) is down, PDisk 6:1015 (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR02) is down + +2024-12-25T18:29:16.277+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:29:26.282+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:29:26.282+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:29:26.283+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:26.283+0100 debug Retrying after 1 seconds... + +2024-12-25T18:29:27.283+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:27.283+0100 debug Retrying after 2 seconds... + +2024-12-25T18:29:29.284+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:29.284+0100 debug Retrying after 4 seconds... + +2024-12-25T18:29:33.285+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:33.285+0100 debug Retrying after 8 seconds... + +2024-12-25T18:29:41.291+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:41.291+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:41.291+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:29:51.299+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:29:51.299+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:29:51.300+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:51.300+0100 debug Retrying after 1 seconds... + +2024-12-25T18:29:52.301+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:52.301+0100 debug Retrying after 2 seconds... + +2024-12-25T18:29:54.302+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:54.302+0100 debug Retrying after 4 seconds... + +2024-12-25T18:29:58.305+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:29:58.305+0100 debug Retrying after 8 seconds... + +2024-12-25T18:30:06.309+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:06.309+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:06.309+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:30:16.310+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:30:16.310+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:30:16.310+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:16.311+0100 debug Retrying after 1 seconds... + +2024-12-25T18:30:17.312+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:17.312+0100 debug Retrying after 2 seconds... + +2024-12-25T18:30:19.314+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:19.314+0100 debug Retrying after 4 seconds... + +2024-12-25T18:30:23.316+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:23.316+0100 debug Retrying after 8 seconds... + +2024-12-25T18:30:31.323+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:31.323+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:31.323+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:30:41.331+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:30:41.331+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:30:41.332+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:41.332+0100 debug Retrying after 1 seconds... + +2024-12-25T18:30:42.333+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:42.333+0100 debug Retrying after 2 seconds... + +2024-12-25T18:30:44.334+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:44.334+0100 debug Retrying after 4 seconds... + +2024-12-25T18:30:48.337+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:48.337+0100 debug Retrying after 8 seconds... + +2024-12-25T18:30:56.344+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:56.344+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:30:56.344+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:31:06.353+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:31:06.353+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:31:06.353+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:06.353+0100 debug Retrying after 1 seconds... + +2024-12-25T18:31:07.354+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:07.354+0100 debug Retrying after 2 seconds... + +2024-12-25T18:31:09.355+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:09.355+0100 debug Retrying after 4 seconds... + +2024-12-25T18:31:13.358+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:13.358+0100 debug Retrying after 8 seconds... + +2024-12-25T18:31:21.365+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:21.365+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:21.365+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:31:31.373+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:31:31.373+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:31:31.374+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:31.374+0100 debug Retrying after 1 seconds... + +2024-12-25T18:31:32.375+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:32.375+0100 debug Retrying after 2 seconds... + +2024-12-25T18:31:34.376+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:34.376+0100 debug Retrying after 4 seconds... + +2024-12-25T18:31:38.379+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:38.379+0100 debug Retrying after 8 seconds... + +2024-12-25T18:31:46.381+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:46.381+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:46.381+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:31:56.381+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:31:56.381+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:31:56.382+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:56.382+0100 debug Retrying after 1 seconds... + +2024-12-25T18:31:57.382+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:57.382+0100 debug Retrying after 2 seconds... + +2024-12-25T18:31:59.383+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:31:59.383+0100 debug Retrying after 4 seconds... + +2024-12-25T18:32:03.386+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:03.386+0100 debug Retrying after 8 seconds... + +2024-12-25T18:32:11.393+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:11.393+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:11.393+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:32:21.401+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:32:21.401+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:32:21.402+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:21.402+0100 debug Retrying after 1 seconds... + +2024-12-25T18:32:22.404+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:22.404+0100 debug Retrying after 2 seconds... + +2024-12-25T18:32:24.405+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:24.405+0100 debug Retrying after 4 seconds... + +2024-12-25T18:32:28.408+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:28.408+0100 debug Retrying after 8 seconds... + +2024-12-25T18:32:36.415+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:36.415+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:36.415+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:32:46.425+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:32:46.425+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:32:46.425+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:46.425+0100 debug Retrying after 1 seconds... + +2024-12-25T18:32:47.426+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:47.426+0100 debug Retrying after 2 seconds... + +2024-12-25T18:32:49.427+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:49.427+0100 debug Retrying after 4 seconds... + +2024-12-25T18:32:53.430+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:32:53.430+0100 debug Retrying after 8 seconds... + +2024-12-25T18:33:01.437+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:33:01.437+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:33:01.437+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:33:11.445+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:33:11.445+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:33:11.446+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:33:11.446+0100 debug Retrying after 1 seconds... + +2024-12-25T18:33:12.448+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:33:12.448+0100 debug Retrying after 2 seconds... + +2024-12-25T18:33:14.449+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:33:14.449+0100 debug Retrying after 4 seconds... + +2024-12-25T18:33:19.724+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:19.724+0100 debug Invoke ListClusterNodes method +2024-12-25T18:33:21.032+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:21.033+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:33:21.033+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:38:19 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:33:21.033+0100 debug Task has retry after attribute: 2024-12-25 17:38:19 +2024-12-25T18:33:21.033+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:33:21.033+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:33:21.033+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:33:31.037+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:33:31.037+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:33:32.900+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:32.900+0100 debug Invoke ListClusterNodes method +2024-12-25T18:33:34.424+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:34.425+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:33:34.425+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:38:32 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:33:34.425+0100 debug Task has retry after attribute: 2024-12-25 17:38:32 +2024-12-25T18:33:34.425+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:33:34.425+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:33:34.425+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:33:44.429+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:33:44.430+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:33:46.145+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:46.145+0100 debug Invoke ListClusterNodes method +2024-12-25T18:33:47.679+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:47.680+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:33:47.680+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:38:45 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:33:47.680+0100 debug Task has retry after attribute: 2024-12-25 17:38:45 +2024-12-25T18:33:47.680+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:33:47.680+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:33:47.680+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:33:57.685+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:33:57.685+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:33:59.015+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:33:59.016+0100 debug Invoke ListClusterNodes method +2024-12-25T18:34:00.578+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:00.580+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:34:00.580+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:38:58 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:34:00.580+0100 debug Task has retry after attribute: 2024-12-25 17:38:58 +2024-12-25T18:34:00.580+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:34:00.580+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:34:00.580+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:34:10.584+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:34:10.584+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:34:11.742+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:11.742+0100 debug Invoke ListClusterNodes method +2024-12-25T18:34:12.974+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:12.975+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:34:12.975+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:39:11 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:34:12.975+0100 debug Task has retry after attribute: 2024-12-25 17:39:11 +2024-12-25T18:34:12.975+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:34:12.975+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:34:12.975+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:34:22.978+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:34:22.978+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:34:24.541+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:24.541+0100 debug Invoke ListClusterNodes method +2024-12-25T18:34:26.140+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:26.141+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:34:26.141+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:39:24 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:34:26.141+0100 debug Task has retry after attribute: 2024-12-25 17:39:24 +2024-12-25T18:34:26.141+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:34:26.141+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:34:26.141+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:34:36.146+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:34:36.146+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:34:37.405+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:37.405+0100 debug Invoke ListClusterNodes method +2024-12-25T18:34:38.673+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:38.674+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:34:38.674+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:39:37 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:34:38.674+0100 debug Task has retry after attribute: 2024-12-25 17:39:37 +2024-12-25T18:34:38.674+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:34:38.674+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:34:38.674+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:34:48.678+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:34:48.678+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:34:50.317+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:50.318+0100 debug Invoke ListClusterNodes method +2024-12-25T18:34:51.733+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:34:51.733+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:34:51.733+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:39:50 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:34:51.733+0100 debug Task has retry after attribute: 2024-12-25 17:39:50 +2024-12-25T18:34:51.733+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:34:51.733+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:34:51.733+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:35:01.734+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:35:01.734+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:35:03.044+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:03.044+0100 debug Invoke ListClusterNodes method +2024-12-25T18:35:04.770+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:04.771+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:35:04.771+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:40:02 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:35:04.771+0100 debug Task has retry after attribute: 2024-12-25 17:40:02 +2024-12-25T18:35:04.771+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:35:04.771+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:35:04.771+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:35:14.776+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:35:14.776+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:35:16.252+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:16.252+0100 debug Invoke ListClusterNodes method +2024-12-25T18:35:17.693+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:17.694+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:35:17.694+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:40:16 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:35:17.694+0100 debug Task has retry after attribute: 2024-12-25 17:40:16 +2024-12-25T18:35:17.694+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:35:17.694+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:35:17.694+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:35:27.699+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:35:27.699+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:35:29.120+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:29.120+0100 debug Invoke ListClusterNodes method +2024-12-25T18:35:30.484+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:30.485+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:35:30.485+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:40:29 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:35:30.485+0100 debug Task has retry after attribute: 2024-12-25 17:40:29 +2024-12-25T18:35:30.485+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down"}] +2024-12-25T18:35:30.485+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038517': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001b5:2:2:1:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR05) is locked by this request. Down: VDisk [820001b5:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR09) is down, VDisk [820001b5:2:0:1:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR26) is down + +2024-12-25T18:35:30.485+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:35:40.490+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:35:40.490+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:35:41.753+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:41.753+0100 debug Invoke ListClusterNodes method +2024-12-25T18:35:42.879+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:42.881+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:35:42.881+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:40:41 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 1 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:35:42.881+0100 debug Task has retry after attribute: 2024-12-25 17:40:41 +2024-12-25T18:35:42.881+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '2181038542': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001ce:2:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is locked by this request. Down: VDisk [820001ce:2:2:0:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR16) is down, VDisk [820001ce:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR14) is down"}] +2024-12-25T18:35:42.881+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '2181038542': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [820001ce:2:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR06) is locked by this request. Down: VDisk [820001ce:2:2:0:0] (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR16) is down, VDisk [820001ce:2:1:0:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/ROTKIKIMR14) is down + +2024-12-25T18:35:42.881+0100 info Wait next 10s delay. Total node progress: 3 out of 9 +2024-12-25T18:35:52.884+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:35:52.884+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:35:54.343+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:54.343+0100 debug Invoke ListClusterNodes method +2024-12-25T18:35:55.880+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:55.880+0100 warn No incompatible versions have been detected. However, 4 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:35:55.880+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba + Lock on node 1 PERFORMED, until: 2024-12-25 18:00:54 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:35:55.881+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:1} duration:{seconds:1500}}} action_uid:{task_uid:"rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba" action_id:"serviceaccount-e0tv2pwt7nw675626d@as-p-290"} status:ACTION_STATUS_PERFORMED reason:ACTION_REASON_OK deadline:{seconds:1735149654 nanos:239519000}} action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:35:55.881+0100 info 1 ActionGroupStates moved to PERFORMED, will restart now... +2024-12-25T18:35:55.881+0100 debug Restart node with id: 1 +2024-12-25T18:35:55.881+0100 debug Drain node with id: 1 +2024-12-25T18:35:55.881+0100 info DRAINING NOT IMPLEMENTED YET +2024-12-25T18:35:55.881+0100 debug Restart node with id: 1 +2024-12-25T18:35:55.881+0100 info Restarting pod s3-dev-storage-4 on the man0-0310.ydb-dev.ik8s.nebiuscloud.net node +2024-12-25T18:35:57.255+0100 debug Pod s3-dev-storage-4 id: e2f46ceb-cbd2-4ed3-9a11-8a62b1a28228 +2024-12-25T18:35:57.629+0100 debug Invoke CompleteAction method +2024-12-25T18:35:58.471+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:35:58.471+0100 debug Manage action result: + Completed action id: serviceaccount-e0tv2pwt7nw675626d@as-p-290, status: SUCCESS +2024-12-25T18:35:58.471+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:36:08.480+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:36:08.480+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:36:10.630+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:10.630+0100 debug Invoke ListClusterNodes method +2024-12-25T18:36:12.064+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:12.066+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:36:12.066+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:41:10 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:36:12.066+0100 debug Task has retry after attribute: 2024-12-25 17:41:10 +2024-12-25T18:36:12.066+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:36:12.066+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:36:12.066+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:36:22.070+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:36:22.070+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:36:23.514+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:23.514+0100 debug Invoke ListClusterNodes method +2024-12-25T18:36:25.269+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:25.270+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:36:25.270+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:41:23 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:36:25.270+0100 debug Task has retry after attribute: 2024-12-25 17:41:23 +2024-12-25T18:36:25.270+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:36:25.270+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:36:25.270+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:36:35.270+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:36:35.270+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:36:36.539+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:36.540+0100 debug Invoke ListClusterNodes method +2024-12-25T18:36:38.279+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:38.280+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:36:38.280+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:41:36 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:36:38.280+0100 debug Task has retry after attribute: 2024-12-25 17:41:36 +2024-12-25T18:36:38.280+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:36:38.280+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:36:38.280+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:36:48.285+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:36:48.285+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:36:49.436+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:49.436+0100 debug Invoke ListClusterNodes method +2024-12-25T18:36:51.175+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:36:51.176+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:36:51.176+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:41:49 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:36:51.176+0100 debug Task has retry after attribute: 2024-12-25 17:41:49 +2024-12-25T18:36:51.176+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:36:51.176+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:36:51.176+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:37:01.177+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:37:01.177+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:37:02.869+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:02.870+0100 debug Invoke ListClusterNodes method +2024-12-25T18:37:04.387+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:04.388+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:37:04.388+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:42:02 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:37:04.388+0100 debug Task has retry after attribute: 2024-12-25 17:42:02 +2024-12-25T18:37:04.388+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:37:04.388+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:37:04.388+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:37:14.392+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:37:14.392+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:37:15.856+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:15.857+0100 debug Invoke ListClusterNodes method +2024-12-25T18:37:17.291+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:17.292+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:37:17.292+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:42:15 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:37:17.292+0100 debug Task has retry after attribute: 2024-12-25 17:42:15 +2024-12-25T18:37:17.292+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:37:17.292+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:37:17.292+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:37:27.297+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:37:27.297+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:37:27.297+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:37:27.298+0100 debug Retrying after 1 seconds... + +2024-12-25T18:37:28.298+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:37:28.298+0100 debug Retrying after 2 seconds... + +2024-12-25T18:37:30.300+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:37:30.300+0100 debug Retrying after 4 seconds... + +2024-12-25T18:37:34.303+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:37:34.303+0100 debug Retrying after 8 seconds... + +2024-12-25T18:37:43.844+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:43.844+0100 debug Invoke ListClusterNodes method +2024-12-25T18:37:45.451+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:45.452+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:37:45.452+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:42:43 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:37:45.452+0100 debug Task has retry after attribute: 2024-12-25 17:42:43 +2024-12-25T18:37:45.452+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:37:45.452+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:37:45.452+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:37:55.457+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:37:55.457+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:37:56.541+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:56.541+0100 debug Invoke ListClusterNodes method +2024-12-25T18:37:58.465+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:37:58.466+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:37:58.466+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:42:56 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:37:58.466+0100 debug Task has retry after attribute: 2024-12-25 17:42:56 +2024-12-25T18:37:58.466+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:37:58.466+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:37:58.466+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:38:08.471+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:38:08.471+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:38:09.927+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:09.928+0100 debug Invoke ListClusterNodes method +2024-12-25T18:38:11.579+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:11.580+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:38:11.580+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:43:09 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:38:11.580+0100 debug Task has retry after attribute: 2024-12-25 17:43:09 +2024-12-25T18:38:11.580+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:38:11.580+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:38:11.580+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:38:21.584+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:38:21.584+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:38:23.245+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:23.246+0100 debug Invoke ListClusterNodes method +2024-12-25T18:38:24.764+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:24.765+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:38:24.765+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:43:23 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:38:24.765+0100 debug Task has retry after attribute: 2024-12-25 17:43:23 +2024-12-25T18:38:24.765+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:38:24.765+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: Host man0-0310.ydb-dev.ik8s.nebiuscloud.net:19001 (1) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:38:24.765+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:38:34.769+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:38:34.769+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:38:36.041+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:36.041+0100 debug Invoke ListClusterNodes method +2024-12-25T18:38:37.404+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:37.405+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:38:37.405+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:43:35 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:38:37.405+0100 debug Task has retry after attribute: 2024-12-25 17:43:35 +2024-12-25T18:38:37.405+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:38:37.405+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:38:37.405+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:38:47.409+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:38:47.409+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:38:48.744+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:48.744+0100 debug Invoke ListClusterNodes method +2024-12-25T18:38:50.024+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:38:50.024+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:38:50.024+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:43:48 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:38:50.024+0100 debug Task has retry after attribute: 2024-12-25 17:43:48 +2024-12-25T18:38:50.024+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:38:50.024+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:38:50.024+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:39:00.028+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:39:00.028+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:39:01.287+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:01.287+0100 debug Invoke ListClusterNodes method +2024-12-25T18:39:02.781+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:02.781+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:39:02.781+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:44:01 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:39:02.781+0100 debug Task has retry after attribute: 2024-12-25 17:44:01 +2024-12-25T18:39:02.782+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:39:02.782+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:39:02.782+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:39:12.786+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:39:12.786+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:39:14.111+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:14.112+0100 debug Invoke ListClusterNodes method +2024-12-25T18:39:16.180+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:16.181+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:39:16.181+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:44:14 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:39:16.181+0100 debug Task has retry after attribute: 2024-12-25 17:44:14 +2024-12-25T18:39:16.181+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:39:16.181+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:39:16.181+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:39:26.185+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:39:26.185+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:39:27.550+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:27.550+0100 debug Invoke ListClusterNodes method +2024-12-25T18:39:29.693+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:29.694+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:39:29.694+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:44:27 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:39:29.694+0100 debug Task has retry after attribute: 2024-12-25 17:44:27 +2024-12-25T18:39:29.694+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:39:29.694+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:39:29.694+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:39:39.699+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:39:39.699+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:39:40.918+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:40.919+0100 debug Invoke ListClusterNodes method +2024-12-25T18:39:42.290+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:39:42.291+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:39:42.292+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:44:40 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:39:42.292+0100 debug Task has retry after attribute: 2024-12-25 17:44:40 +2024-12-25T18:39:42.292+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:39:42.292+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down + +2024-12-25T18:39:42.292+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:39:52.297+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:39:52.297+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:39:52.297+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:39:52.297+0100 debug Retrying after 1 seconds... + +2024-12-25T18:39:53.299+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:39:53.299+0100 debug Retrying after 2 seconds... + +2024-12-25T18:39:55.299+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:39:55.299+0100 debug Retrying after 4 seconds... + +2024-12-25T18:39:59.302+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:39:59.302+0100 debug Retrying after 8 seconds... + +2024-12-25T18:40:07.309+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:07.309+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:07.309+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:40:17.309+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:40:17.309+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:40:17.310+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:17.310+0100 debug Retrying after 1 seconds... + +2024-12-25T18:40:18.312+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:18.312+0100 debug Retrying after 2 seconds... + +2024-12-25T18:40:20.312+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:20.312+0100 debug Retrying after 4 seconds... + +2024-12-25T18:40:24.315+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:24.315+0100 debug Retrying after 8 seconds... + +2024-12-25T18:40:32.322+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:32.322+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:32.322+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:40:42.331+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:40:42.331+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:40:42.332+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:42.332+0100 debug Retrying after 1 seconds... + +2024-12-25T18:40:43.332+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:43.332+0100 debug Retrying after 2 seconds... + +2024-12-25T18:40:45.333+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:45.333+0100 debug Retrying after 4 seconds... + +2024-12-25T18:40:49.336+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:49.336+0100 debug Retrying after 8 seconds... + +2024-12-25T18:40:57.344+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:57.344+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:40:57.344+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:41:07.352+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:41:07.352+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:41:07.353+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:07.353+0100 debug Retrying after 1 seconds... + +2024-12-25T18:41:08.355+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:08.355+0100 debug Retrying after 2 seconds... + +2024-12-25T18:41:10.355+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:10.355+0100 debug Retrying after 4 seconds... + +2024-12-25T18:41:14.358+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:14.358+0100 debug Retrying after 8 seconds... + +2024-12-25T18:41:22.365+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:22.365+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:22.365+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:41:32.373+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:41:32.373+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:41:32.374+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:32.374+0100 debug Retrying after 1 seconds... + +2024-12-25T18:41:33.375+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:33.375+0100 debug Retrying after 2 seconds... + +2024-12-25T18:41:35.376+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:35.376+0100 debug Retrying after 4 seconds... + +2024-12-25T18:41:39.380+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:39.380+0100 debug Retrying after 8 seconds... + +2024-12-25T18:41:47.381+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:47.381+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:47.381+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:41:57.382+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:41:57.382+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:41:57.382+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:57.382+0100 debug Retrying after 1 seconds... + +2024-12-25T18:41:58.383+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:41:58.383+0100 debug Retrying after 2 seconds... + +2024-12-25T18:42:00.384+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:00.384+0100 debug Retrying after 4 seconds... + +2024-12-25T18:42:04.387+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:04.387+0100 debug Retrying after 8 seconds... + +2024-12-25T18:42:12.393+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:12.393+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:12.393+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:42:22.402+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:42:22.402+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:42:22.402+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:22.402+0100 debug Retrying after 1 seconds... + +2024-12-25T18:42:23.403+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:23.403+0100 debug Retrying after 2 seconds... + +2024-12-25T18:42:25.404+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:25.404+0100 debug Retrying after 4 seconds... + +2024-12-25T18:42:29.407+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:29.407+0100 debug Retrying after 8 seconds... + +2024-12-25T18:42:37.414+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:37.414+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:37.414+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:42:47.423+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:42:47.423+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:42:47.424+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:47.424+0100 debug Retrying after 1 seconds... + +2024-12-25T18:42:48.426+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:48.426+0100 debug Retrying after 2 seconds... + +2024-12-25T18:42:50.427+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:50.427+0100 debug Retrying after 4 seconds... + +2024-12-25T18:42:54.429+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:42:54.429+0100 debug Retrying after 8 seconds... + +2024-12-25T18:43:03.815+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:03.815+0100 debug Invoke ListClusterNodes method +2024-12-25T18:43:05.025+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:05.025+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:43:05.025+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:48:03 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:43:05.025+0100 debug Task has retry after attribute: 2024-12-25 17:48:03 +2024-12-25T18:43:05.025+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:43:05.026+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:43:05.026+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:43:15.029+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:43:15.029+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:43:16.603+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:16.604+0100 debug Invoke ListClusterNodes method +2024-12-25T18:43:18.258+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:18.259+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:43:18.259+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:48:16 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:43:18.259+0100 debug Task has retry after attribute: 2024-12-25 17:48:16 +2024-12-25T18:43:18.259+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:43:18.259+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:43:18.259+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:43:28.265+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:43:28.265+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:43:29.442+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:29.442+0100 debug Invoke ListClusterNodes method +2024-12-25T18:43:30.677+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:30.678+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:43:30.678+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:48:29 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:43:30.678+0100 debug Task has retry after attribute: 2024-12-25 17:48:29 +2024-12-25T18:43:30.678+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:43:30.678+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down + +2024-12-25T18:43:30.678+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:43:40.683+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:43:40.683+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:43:42.271+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:42.271+0100 debug Invoke ListClusterNodes method +2024-12-25T18:43:43.521+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:43.522+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:43:43.522+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:48:42 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:43:43.522+0100 debug Task has retry after attribute: 2024-12-25 17:48:42 +2024-12-25T18:43:43.522+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:43:43.522+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down + +2024-12-25T18:43:43.522+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:43:53.527+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:43:53.527+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:43:54.663+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:54.663+0100 debug Invoke ListClusterNodes method +2024-12-25T18:43:56.544+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:43:56.546+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:43:56.546+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:48:54 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:43:56.546+0100 debug Task has retry after attribute: 2024-12-25 17:48:54 +2024-12-25T18:43:56.546+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:43:56.546+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down + +2024-12-25T18:43:56.546+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:44:06.551+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:44:06.551+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:44:07.919+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:07.920+0100 debug Invoke ListClusterNodes method +2024-12-25T18:44:09.574+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:09.575+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:44:09.575+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:49:07 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:44:09.575+0100 debug Task has retry after attribute: 2024-12-25 17:49:07 +2024-12-25T18:44:09.575+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down"}] +2024-12-25T18:44:09.575+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, Host man0-0305.ydb-dev.ik8s.nebiuscloud.net:19001 (7) is down + +2024-12-25T18:44:09.575+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:44:19.579+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:44:19.579+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:44:21.671+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:21.671+0100 debug Invoke ListClusterNodes method +2024-12-25T18:44:23.155+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:23.156+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:44:23.156+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:49:21 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:44:23.156+0100 debug Task has retry after attribute: 2024-12-25 17:49:21 +2024-12-25T18:44:23.156+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:44:23.156+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:44:23.156+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:44:33.160+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:44:33.160+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:44:34.439+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:34.439+0100 debug Invoke ListClusterNodes method +2024-12-25T18:44:35.876+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:35.877+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:44:35.877+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:49:34 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:44:35.877+0100 debug Task has retry after attribute: 2024-12-25 17:49:34 +2024-12-25T18:44:35.877+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:44:35.877+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:44:35.877+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:44:45.881+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:44:45.881+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:44:47.450+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:47.450+0100 debug Invoke ListClusterNodes method +2024-12-25T18:44:48.781+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:44:48.782+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:44:48.782+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:49:47 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:44:48.782+0100 debug Task has retry after attribute: 2024-12-25 17:49:47 +2024-12-25T18:44:48.782+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:44:48.782+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:44:48.782+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:44:58.784+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:44:58.784+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:45:00.455+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:45:00.456+0100 debug Invoke ListClusterNodes method +2024-12-25T18:45:01.934+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:45:01.934+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:45:01.934+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:50:00 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:45:01.934+0100 debug Task has retry after attribute: 2024-12-25 17:50:00 +2024-12-25T18:45:01.934+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:45:01.934+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:45:01.934+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:45:11.939+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:45:11.939+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:45:13.562+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:45:13.562+0100 debug Invoke ListClusterNodes method +2024-12-25T18:45:15.106+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:45:15.107+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:45:15.107+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:50:13 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:45:15.107+0100 debug Task has retry after attribute: 2024-12-25 17:50:13 +2024-12-25T18:45:15.107+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:45:15.107+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:45:15.107+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:45:25.111+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:45:25.111+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:45:25.112+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:25.112+0100 debug Retrying after 1 seconds... + +2024-12-25T18:45:26.113+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:26.113+0100 debug Retrying after 2 seconds... + +2024-12-25T18:45:28.114+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:28.114+0100 debug Retrying after 4 seconds... + +2024-12-25T18:45:32.117+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:32.117+0100 debug Retrying after 8 seconds... + +2024-12-25T18:45:40.124+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:40.124+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:40.124+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:45:50.133+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:45:50.133+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:45:50.133+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:50.133+0100 debug Retrying after 1 seconds... + +2024-12-25T18:45:51.134+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:51.134+0100 debug Retrying after 2 seconds... + +2024-12-25T18:45:53.135+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:53.135+0100 debug Retrying after 4 seconds... + +2024-12-25T18:45:57.138+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:45:57.138+0100 debug Retrying after 8 seconds... + +2024-12-25T18:46:05.145+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:05.145+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:05.145+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:46:15.153+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:46:15.153+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:46:15.154+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:15.154+0100 debug Retrying after 1 seconds... + +2024-12-25T18:46:16.155+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:16.155+0100 debug Retrying after 2 seconds... + +2024-12-25T18:46:18.156+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:18.156+0100 debug Retrying after 4 seconds... + +2024-12-25T18:46:22.159+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:22.159+0100 debug Retrying after 8 seconds... + +2024-12-25T18:46:30.167+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:30.167+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:30.167+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:46:40.177+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:46:40.177+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:46:40.178+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:40.178+0100 debug Retrying after 1 seconds... + +2024-12-25T18:46:41.178+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:41.178+0100 debug Retrying after 2 seconds... + +2024-12-25T18:46:43.179+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:43.179+0100 debug Retrying after 4 seconds... + +2024-12-25T18:46:47.182+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:47.182+0100 debug Retrying after 8 seconds... + +2024-12-25T18:46:55.189+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:55.189+0100 warn Failed to refresh maintenance task: number of retries exceeded: 5. Last error:rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:46:55.189+0100 info Wait next 10s delay. Total node progress: 4 out of 9 +2024-12-25T18:47:05.191+0100 info Refresh maintenance task with id: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +2024-12-25T18:47:05.191+0100 debug Invoke RefreshMaintenanceTask method +2024-12-25T18:47:05.191+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:47:05.191+0100 debug Retrying after 1 seconds... + +2024-12-25T18:47:06.192+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:47:06.192+0100 debug Retrying after 2 seconds... + +2024-12-25T18:47:08.193+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:47:08.193+0100 debug Retrying after 4 seconds... + +2024-12-25T18:47:12.196+0100 debug Invocation error: rpc error: code = Unavailable desc = connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2135: connect: connection refused" +2024-12-25T18:47:12.196+0100 debug Retrying after 8 seconds... + +2024-12-25T18:47:22.176+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:47:22.177+0100 debug Invoke ListClusterNodes method +2024-12-25T18:47:23.612+0100 debug Invocation result: +Operation status: SUCCESS +2024-12-25T18:47:23.613+0100 warn No incompatible versions have been detected. However, 5 nodes reported unparsable versions. + You may have introduced incompatible versions to the cluster +2024-12-25T18:47:23.613+0100 debug Maintenance task result: +Uid: rolling-restart-c98e4c33-c897-4470-a99c-830485cacfba +Retry after: 2024-12-25 17:52:22 + Lock on node 5 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 6 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 3 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 4 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + Lock on node 2 PENDING, ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS + +2024-12-25T18:47:23.613+0100 debug Task has retry after attribute: 2024-12-25 17:52:22 +2024-12-25T18:47:23.613+0100 debug Unfiltered ActionGroupStates: [action_states:{action:{lock_action:{scope:{node_id:5} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:6} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:3} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:4} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"} action_states:{action:{lock_action:{scope:{node_id:2} duration:{seconds:1500}}} status:ACTION_STATUS_PENDING reason:ACTION_REASON_TOO_MANY_UNAVAILABLE_VDISKS reason_details:"Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down"}] +2024-12-25T18:47:23.613+0100 info No actions can be taken yet, CMS didn't move any actions to PERFORMED because of: Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:1:0] (man0-0306.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:2:0] (man0-0307.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:2:0] (man0-0185.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Number of data centers with unavailable vdisks: 3. Locked: VDisk [0:1:1:0:0] (man0-0308.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down +Issue in affected group with id '0': too many unavailable vdisks. Locked: VDisk [0:1:0:1:0] (man0-0287.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is locked by this request. Down: VDisk [0:1:0:0:0] (man0-0310.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down, VDisk [0:1:2:2:0] (man0-0309.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR02) is down, PDisk 7:27 (man0-0305.ydb-dev.ik8s.nebiuscloud.net:/dev/disk/by-partlabel/NVMEKIKIMR01) is down + +2024-12-25T18:47:23.613+0100 info Wait next 10s delay. Total node progress: 4 out of 9