<fix>[storage]: when the snapshot group creation fails, delete the successfully created snapshots

taogan21 · taogan21 · commit 632d824a52db · 2025-11-13T09:35:44.000+08:00
Resolves: ZSV-9792

Change-Id: I6b65736e646e7163777a7872667077756771726d
diff --git a/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java b/plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageFactory.java
@@ -742,6 +742,7 @@ public void beforeTakeLiveSnapshotsOnVolumes(CreateVolumesSnapshotOverlayInnerMs
         logger.info(String.format("take snapshots for volumes[%s] on %s",
                 msg.getLockedVolumeUuids(), getClass().getCanonicalName()));
 
+        List<VolumeSnapshotInventory> inventories = Collections.synchronizedList(new ArrayList<>());
         ErrorCodeList errList = new ErrorCodeList();
         new While<>(cephStructs).all((struct, whileCompletion) -> {
             VolumeSnapshotVO vo = Q.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.uuid, struct.getResourceUuid()).find();
@@ -779,6 +780,7 @@ public void run(MessageReply reply) {
                     dbf.update(vo);
 
                     struct.getVolumeSnapshotStruct().setCurrent(treply.getInventory());
+                    inventories.add(treply.getInventory());
                     whileCompletion.done();
                 }
             });
@@ -787,6 +789,17 @@ public void run(MessageReply reply) {
             public void done(ErrorCodeList errorCodeList) {
                 if (!errList.getCauses().isEmpty()) {
                     completion.fail(errList.getCauses().get(0));
+
+                    inventories.forEach(snapshot -> {
+                        VolumeSnapshotDeletionMsg msg = new VolumeSnapshotDeletionMsg();
+                        msg.setSnapshotUuid(snapshot.getUuid());
+                        msg.setTreeUuid(snapshot.getTreeUuid());
+                        msg.setVolumeUuid(snapshot.getVolumeUuid());
+                        msg.setScope(DeleteVolumeSnapshotScope.Single.toString());
+                        msg.setDirection(DeleteVolumeSnapshotDirection.Commit.toString());
+                        bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, snapshot.getUuid());
+                        bus.send(msg);
+                    });
                     return;
                 }
                 completion.success();
diff --git a/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java b/storage/src/main/java/org/zstack/storage/addon/primary/ExternalPrimaryStorageFactory.java
@@ -537,6 +537,7 @@ public void beforeTakeLiveSnapshotsOnVolumes(CreateVolumesSnapshotOverlayInnerMs
         logger.info(String.format("take snapshots for volumes[%s] on %s",
                 msg.getLockedVolumeUuids(), getClass().getCanonicalName()));
 
+        List<VolumeSnapshotInventory> inventories = Collections.synchronizedList(new ArrayList<>());
         ErrorCodeList errList = new ErrorCodeList();
         new While<>(storageSnapshots).all((struct, whileCompletion) -> {
             VolumeSnapshotVO vo = Q.New(VolumeSnapshotVO.class).eq(VolumeSnapshotVO_.uuid, struct.getResourceUuid()).find();
@@ -574,6 +575,7 @@ public void run(MessageReply reply) {
                     dbf.update(vo);
 
                     struct.getVolumeSnapshotStruct().setCurrent(treply.getInventory());
+                    inventories.add(treply.getInventory());
                     whileCompletion.done();
                 }
             });
@@ -582,6 +584,18 @@ public void run(MessageReply reply) {
             public void done(ErrorCodeList errorCodeList) {
                 if (!errList.getCauses().isEmpty()) {
                     completion.fail(errList.getCauses().get(0));
+
+                    inventories.forEach(snapshot -> {
+                        VolumeSnapshotDeletionMsg msg = new VolumeSnapshotDeletionMsg();
+                        msg.setSnapshotUuid(snapshot.getUuid());
+                        msg.setTreeUuid(snapshot.getTreeUuid());
+                        msg.setVolumeUuid(snapshot.getVolumeUuid());
+                        msg.setScope(DeleteVolumeSnapshotScope.Single.toString());
+                        msg.setDirection(DeleteVolumeSnapshotDirection.Commit.toString());
+                        bus.makeTargetServiceIdByResourceUuid(msg, VolumeSnapshotConstant.SERVICE_ID, snapshot.getUuid());
+                        bus.send(msg);
+                    });
+
                     return;
                 }
                 completion.success();
diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotApiInterceptor.java
@@ -31,6 +31,7 @@
 import javax.persistence.Tuple;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 
@@ -74,6 +75,8 @@ public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionExcepti
             validate((APIBatchDeleteVolumeSnapshotMsg) msg);
         } else if (msg instanceof APIRevertVmFromSnapshotGroupMsg) {
             validate((APIRevertVmFromSnapshotGroupMsg) msg);
+        } else if (msg instanceof APIDeleteVolumeSnapshotGroupMsg) {
+            validate((APIDeleteVolumeSnapshotGroupMsg) msg);
         }
 
         setServiceId(msg);
@@ -217,4 +220,29 @@ private void validate(APIBatchDeleteVolumeSnapshotMsg msg) {
             throw new ApiMessageInterceptionException(operr("can not find volume uuid for snapshosts[uuid: %s]", msg.getUuids()));
         }
     }
+
+    private void validate(APIDeleteVolumeSnapshotGroupMsg msg) {
+        VolumeSnapshotGroupVO groupVO = dbf.findByUuid(msg.getUuid(), VolumeSnapshotGroupVO.class);
+        // 获取当前虚拟机所有内存快照
+        // 检测内存快照是否完整
+        // 1 完整 允许删除
+        // 2 不完整 不允许删除
+        List<VolumeSnapshotGroupVO> groups = Q.New(VolumeSnapshotGroupVO.class)
+                .eq(VolumeSnapshotGroupVO_.vmInstanceUuid, groupVO.getVmInstanceUuid())
+                .orderByAsc(VolumeSnapshotGroupVO_.createDate)
+                .list();
+
+        final String[] ungroupUuid = new String[1];
+        groups.forEach(group -> {
+            Set<VolumeSnapshotGroupRefVO> volumeSnapshotRefs = group.getVolumeSnapshotRefs();
+            volumeSnapshotRefs.forEach(ref -> {
+                if (ref.isSnapshotDeleted()) {
+                    ungroupUuid[0] = group.getUuid();
+                }
+            });
+        });
+        if (ungroupUuid[0] != null) {
+            throw new ApiMessageInterceptionException(argerr("volume snapshot group[uuid:%s] is not complete, cannot delete volume snapshot group", ungroupUuid[0]));
+        }
+    }
 }
diff --git a/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java b/storage/src/main/java/org/zstack/storage/snapshot/VolumeSnapshotTreeBase.java
@@ -2088,21 +2088,37 @@ private void ungroupAfterDeleted(List<VolumeSnapshotInventory> snapshots) {
         List<String> uuids = snapshots.stream().map(VolumeSnapshotInventory::getUuid).collect(Collectors.toList());
         SQL.New(VolumeSnapshotGroupRefVO.class).in(VolumeSnapshotGroupRefVO_.volumeSnapshotUuid, uuids)
                 .set(VolumeSnapshotGroupRefVO_.snapshotDeleted, true).update();
-        if (currentRoot.getVolumeType().equals(VolumeType.Root.toString())) {
-            List<String> groupUuids = new ArrayList<>();
-            for (VolumeSnapshotInventory snapshot : snapshots) {
-                String groupUuid = snapshot.getGroupUuid();
-                if (groupUuid != null) {
-                    logger.debug(String.format("root volume snapshot[uuid:%s, name:%s] has been deleted, " +
-                            "ungroup snapshot group[uuid:%s]", snapshot.getUuid(), snapshot.getName(), groupUuid));
-                    groupUuids.add(groupUuid);
-                }
 
-            }
+        List<String> groupUuids = snapshots.stream().map(VolumeSnapshotInventory::getGroupUuid).filter(Objects::nonNull).collect(Collectors.toList());
+        if (groupUuids.isEmpty()) {
+            return;
+        }
+
+        List<VolumeSnapshotGroupVO> groupVOs = Q.New(VolumeSnapshotGroupVO.class).in(VolumeSnapshotGroupVO_.uuid, groupUuids).list();
+        groupVOs.forEach(groupVO -> {
+            new RunInQueue(String.format("ungroup-volumeSnapshotGroup-%s", groupVO.getUuid()), thdf, 1)
+                    .name("ungroup-volumeSnapshotGroup-in-queue").asyncBackup(null)
+                    .run(chain -> ungroupAfterDeleted(groupVO, new NoErrorCompletion(chain) {
+                                @Override
+                                public void done() {
+                                    chain.next();
+                                }
+                            })
+                    );
+        });
+    }
 
-            groupUuids.forEach(groupUuid -> vidm.deleteArchiveVmInstanceResourceMetadataGroup(groupUuid));
-            dbf.removeByPrimaryKeys(groupUuids, VolumeSnapshotGroupVO.class);
+    private void ungroupAfterDeleted(VolumeSnapshotGroupVO groupVO, NoErrorCompletion completion) {
+        if (!groupVO.getVolumeSnapshotRefs().stream().allMatch(VolumeSnapshotGroupRefVO::isSnapshotDeleted)) {
+            logger.debug(String.format("skipping ungroup operation for snapshot group[uuid:%s]: " +
+                    "no group meet deletion criteria (due to remaining volume snapshots).", groupVO.getUuid()));
+            completion.done();
+            return;
         }
+        logger.debug(String.format("snapshot group[uuid:%s] all volume snapshot has been deleted, delete snapshot group", groupVO.getUuid()));
+        vidm.deleteArchiveVmInstanceResourceMetadataGroup(groupVO.getUuid());
+        dbf.removeByPrimaryKey(groupVO.getUuid(), VolumeSnapshotGroupVO.class);
+        completion.done();
     }
 
     private List<VolumeSnapshotBackupStorageDeletionMsg> makeVolumeSnapshotBackupStorageDeletionMsg(List<String> bsUuids) {
