<feature>[vm]: support DELETE /vm-instances/metadata bulk cleanup

conf/serviceConfig/vmInstance.xml

@@ -267,6 +267,9 @@
     <message>
         <name>org.zstack.header.vm.APICleanupVmInstanceMetadataMsg</name>
     </message>
+    <message>
+        <name>org.zstack.header.vm.APICleanupAllVmInstanceMetadataMsg</name>
+    </message>
     <message>
         <name>org.zstack.header.vm.APIRegisterVmInstanceFromMetadataMsg</name>
     </message>

header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageMsg.java

@@ -0,0 +1,25 @@
+package org.zstack.header.storage.primary;
+
+import org.zstack.header.message.NeedReplyMessage;
+
+public class CleanupAllVmMetadataOnPrimaryStorageMsg extends NeedReplyMessage implements PrimaryStorageMessage {
+    private String primaryStorageUuid;
+    private String metadataDir;
+
+    @Override
+    public String getPrimaryStorageUuid() {
+        return primaryStorageUuid;
+    }
+
+    public void setPrimaryStorageUuid(String primaryStorageUuid) {
+        this.primaryStorageUuid = primaryStorageUuid;
+    }
+
+    public String getMetadataDir() {
+        return metadataDir;
+    }
+
+    public void setMetadataDir(String metadataDir) {
+        this.metadataDir = metadataDir;
+    }
+}

header/src/main/java/org/zstack/header/storage/primary/CleanupAllVmMetadataOnPrimaryStorageReply.java

@@ -0,0 +1,6 @@
+package org.zstack.header.storage.primary;
+
+import org.zstack.header.message.MessageReply;
+
+public class CleanupAllVmMetadataOnPrimaryStorageReply extends MessageReply {
+}

header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEvent.java

@@ -0,0 +1,33 @@
+package org.zstack.header.vm;
+
+import org.zstack.header.message.APIEvent;
+import org.zstack.header.rest.RestResponse;
+
+import java.util.List;
+
+@RestResponse(fieldsTo = {"all"})
+public class APICleanupAllVmInstanceMetadataEvent extends APIEvent {
+    private List<String> failedPrimaryStorageUuids;
+
+    public APICleanupAllVmInstanceMetadataEvent() {
+        super(null);
+    }
+
+    public APICleanupAllVmInstanceMetadataEvent(String apiId) {
+        super(apiId);
+    }
+
+    public List<String> getFailedPrimaryStorageUuids() {
+        return failedPrimaryStorageUuids;
+    }
+
+    public void setFailedPrimaryStorageUuids(List<String> failedPrimaryStorageUuids) {
+        this.failedPrimaryStorageUuids = failedPrimaryStorageUuids;
+    }
+
+    public static APICleanupAllVmInstanceMetadataEvent __example__() {
+        APICleanupAllVmInstanceMetadataEvent evt = new APICleanupAllVmInstanceMetadataEvent();
+        evt.failedPrimaryStorageUuids = java.util.Collections.emptyList();
+        return evt;
+    }
+}

header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataEventDoc_zh_cn.groovy

@@ -0,0 +1,29 @@
+package org.zstack.header.vm
+
+import org.zstack.header.errorcode.ErrorCode
+
+doc {
+
+	title "清理全部云主机元数据返回"
+
+	field {
+		name "failedPrimaryStorageUuids"
+		desc "清理失败的主存储UUID列表；具体失败原因汇总见 error 字段，逐条详情见 mn / agent 日志"
+		type "List"
+		since "5.0.0"
+	}
+	field {
+		name "success"
+		desc "操作是否成功；任一主存储清理失败则为 false"
+		type "boolean"
+		since "5.0.0"
+	}
+	ref {
+		name "error"
+		path "org.zstack.header.vm.APICleanupAllVmInstanceMetadataEvent.error"
+		desc "错误码；success=false 时聚合所有失败主存储的失败原因"
+		type "ErrorCode"
+		since "5.0.0"
+		clz ErrorCode.class
+	}
+}

header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsg.java

@@ -0,0 +1,34 @@
+package org.zstack.header.vm;
+
+import org.springframework.http.HttpMethod;
+import org.zstack.header.message.APIMessage;
+import org.zstack.header.message.APIParam;
+import org.zstack.header.rest.RestRequest;
+import org.zstack.header.storage.primary.PrimaryStorageVO;
+
+import java.util.List;
+
+@RestRequest(
+        path = "/vm-instances/metadata",
+        method = HttpMethod.DELETE,
+        responseClass = APICleanupAllVmInstanceMetadataEvent.class,
+        isAction = true
+)
+public class APICleanupAllVmInstanceMetadataMsg extends APIMessage {
+    @APIParam(resourceType = PrimaryStorageVO.class, required = false)
+    private List<String> primaryStorageUuids;
+
+    public List<String> getPrimaryStorageUuids() {
+        return primaryStorageUuids;
+    }
+
+    public void setPrimaryStorageUuids(List<String> primaryStorageUuids) {
+        this.primaryStorageUuids = primaryStorageUuids;
+    }
+
+    public static APICleanupAllVmInstanceMetadataMsg __example__() {
+        APICleanupAllVmInstanceMetadataMsg msg = new APICleanupAllVmInstanceMetadataMsg();
+        msg.primaryStorageUuids = java.util.Arrays.asList(uuid(), uuid());
+        return msg;
+    }
+}

header/src/main/java/org/zstack/header/vm/APICleanupAllVmInstanceMetadataMsgDoc_zh_cn.groovy

@@ -0,0 +1,58 @@
+package org.zstack.header.vm
+
+import org.zstack.header.vm.APICleanupAllVmInstanceMetadataEvent
+
+doc {
+	title "清理全部云主机元数据"
+
+	category "云主机"
+
+	desc """清理一个或多个主存储上保存的全部云主机元数据文件，仅管理员可调用。当 primaryStorageUuids 为空（未传或传空列表）时，将清理系统中所有 Enabled+Connected 且支持云主机元数据的主存储；否则仅清理列表中指定的主存储。"""
+
+	rest {
+		request {
+			url "DELETE /v1/vm-instances/metadata"
+
+			header (Authorization: 'OAuth the-session-uuid')
+
+			clz APICleanupAllVmInstanceMetadataMsg.class
+
+			desc """"""
+
+			params {
+
+				column {
+					name "primaryStorageUuids"
+					enclosedIn "cleanupAllVmInstanceMetadata"
+					desc "需要清理云主机元数据的主存储UUID列表；为空时清理所有 Enabled+Connected 主存储上的元数据"
+					location "body"
+					type "List"
+					optional true
+					since "5.0.0"
+				}
+				column {
+					name "systemTags"
+					enclosedIn ""
+					desc "系统标签"
+					location "query"
+					type "List"
+					optional true
+					since "5.0.0"
+				}
+				column {
+					name "userTags"
+					enclosedIn ""
+					desc "用户标签"
+					location "query"
+					type "List"
+					optional true
+					since "5.0.0"
+				}
+			}
+		}
+
+		response {
+			clz APICleanupAllVmInstanceMetadataEvent.class
+		}
+	}
+}

plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java

@@ -72,6 +72,8 @@
 import static org.zstack.utils.CollectionUtils.toMap;
 import static org.zstack.utils.CollectionUtils.transformAndRemoveNull;
 
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * Created by frank on 6/30/2015.
  */
@@ -907,6 +909,8 @@ public void handleLocalMessage(Message msg) {
             handle((CommitVolumeSnapshotOnPrimaryStorageMsg) msg);
         } else if (msg instanceof PullVolumeSnapshotOnPrimaryStorageMsg) {
             handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg);
+        } else if (msg instanceof CleanupAllVmMetadataOnPrimaryStorageMsg) {
+            handle((CleanupAllVmMetadataOnPrimaryStorageMsg) msg);
         } else {
             super.handleLocalMessage(msg);
         }
@@ -3625,4 +3629,64 @@ public String getName() {
             }
         });
     }
+
+    @Override
+    protected void handle(final CleanupAllVmMetadataOnPrimaryStorageMsg msg) {
+        CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply();
+
+        List<String> connectedHostUuids = getConnectedLocalStorageHostUuids();
+        if (connectedHostUuids.isEmpty()) {
+            logger.warn(String.format("[MetadataCleanup] cleanAll: no connected host found for local ps[uuid:%s]", self.getUuid()));
+            bus.reply(msg, reply);
+            return;
+        }
+
+        new While<>(connectedHostUuids).all((hostUuid, com) -> {
+            final LocalStorageHypervisorBackend bkd;
+            try {
+                LocalStorageHypervisorFactory f = getHypervisorBackendFactoryByHostUuid(hostUuid);
+                bkd = f.getHypervisorBackend(self);
+            } catch (Exception e) {
+                logger.warn(String.format("[MetadataCleanup] cleanAll: failed to prepare backend for host[uuid:%s] on ps[uuid:%s]: %s",
+                        hostUuid, self.getUuid(), e.getMessage()));
+                com.addError(operr("host[uuid:%s] backend prepare failed: %s", hostUuid, e.getMessage()));
+                com.done();
+                return;
+            }
+            bkd.handle(msg, hostUuid, new ReturnValueCompletion<CleanupAllVmMetadataOnPrimaryStorageReply>(com) {
+                @Override
+                public void success(CleanupAllVmMetadataOnPrimaryStorageReply returnValue) {
+                    com.done();
+                }
+
+                @Override
+                public void fail(ErrorCode errorCode) {
+                    logger.warn(String.format("[MetadataCleanup] cleanAll: failed on host[uuid:%s] on ps[uuid:%s]: %s",
+                            hostUuid, self.getUuid(), errorCode));
+                    com.addError(operr("host[uuid:%s]: %s", hostUuid, errorCode.getDescription()));
+                    com.done();
+                }
+            });
+        }).run(new WhileDoneCompletion(msg) {
+            @Override
+            public void done(ErrorCodeList errorCodeList) {
+                if (!errorCodeList.getCauses().isEmpty()) {
+                    reply.setError(operr("local primary storage[uuid:%s] cleanAll failed on %d/%d host(s): %s",
+                            self.getUuid(), errorCodeList.getCauses().size(), connectedHostUuids.size(), errorCodeList));
+                }
+                bus.reply(msg, reply);
+            }
+        });
+    }
+
+    private List<String> getConnectedLocalStorageHostUuids() {
+        return SQL.New(
+                        "select h.hostUuid from LocalStorageHostRefVO h, HostVO host" +
+                                " where h.primaryStorageUuid = :psUuid" +
+                                " and h.hostUuid = host.uuid" +
+                                " and host.status = :hstatus", String.class)
+                .param("psUuid", self.getUuid())
+                .param("hstatus", HostStatus.Connected)
+                .list();
+    }
 }

plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageHypervisorBackend.java

@@ -132,5 +132,7 @@ public LocalStorageHypervisorBackend(PrimaryStorageVO self) {
 
     abstract void handle(CleanupVmInstanceMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion<CleanupVmInstanceMetadataOnPrimaryStorageReply> completion);
 
+    abstract void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion<CleanupAllVmMetadataOnPrimaryStorageReply> completion);
+
     abstract void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion<RebaseVolumeBackingFileOnPrimaryStorageReply> completion);
 }

plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageKvmBackend.java

@@ -950,6 +950,13 @@ public static class CleanupVmMetadataCmd extends AgentCommand {
     public static class CleanupVmMetadataRsp extends AgentResponse {
     }
 
+    public static class CleanupAllVmMetadataCmd extends AgentCommand {
+        public String metadataDir;
+    }
+
+    public static class CleanupAllVmMetadataRsp extends AgentResponse {
+    }
+
     public static class PrefixRebaseBackingFilesCmd extends LocalStorageKvmBackend.AgentCommand {
         public List<String> filePaths;
         public String oldPrefix;
@@ -996,6 +1003,7 @@ public static class PrefixRebaseBackingFilesRsp extends LocalStorageKvmBackend.A
     public static final String GET_VM_INSTANCE_METADATA_PATH = "/localstorage/vm/metadata/get";
     public static final String SCAN_VM_METADATA_PATH = "/localstorage/vm/metadata/scan";
     public static final String CLEANUP_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanup";
+    public static final String CLEANUP_ALL_VM_METADATA_PATH = "/localstorage/vm/metadata/cleanup-all";
     public static final String PREFIX_REBASE_BACKING_FILES_PATH = "/localstorage/snapshot/prefixrebasebackingfiles";
 
     public LocalStorageKvmBackend() {
@@ -3955,6 +3963,25 @@ public void fail(ErrorCode errorCode) {
         });
     }
 
+    @Override
+    void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion<CleanupAllVmMetadataOnPrimaryStorageReply> completion) {
+        CleanupAllVmMetadataCmd cmd = new CleanupAllVmMetadataCmd();
+        cmd.metadataDir = msg.getMetadataDir();
+
+        httpCall(CLEANUP_ALL_VM_METADATA_PATH, hostUuid, cmd, CleanupAllVmMetadataRsp.class, new ReturnValueCompletion<CleanupAllVmMetadataRsp>(completion) {
+            @Override
+            public void success(CleanupAllVmMetadataRsp rsp) {
+                CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply();
+                completion.success(reply);
+            }
+
+            @Override
+            public void fail(ErrorCode errorCode) {
+                completion.fail(errorCode);
+            }
+        });
+    }
+
     @Override
     void handle(RebaseVolumeBackingFileOnPrimaryStorageMsg msg, String hostUuid, ReturnValueCompletion<RebaseVolumeBackingFileOnPrimaryStorageReply> completion) {
         PrefixRebaseBackingFilesCmd cmd = new PrefixRebaseBackingFilesCmd();

plugin/nfsPrimaryStorage/src/main/java/org/zstack/storage/primary/nfs/NfsPrimaryStorage.java

@@ -136,6 +136,8 @@ protected void handleLocalMessage(Message msg) {
             handle((PullVolumeSnapshotOnPrimaryStorageMsg) msg);
         } else if (msg instanceof RebaseVolumeBackingFileOnPrimaryStorageMsg) {
             handle((RebaseVolumeBackingFileOnPrimaryStorageMsg) msg);
+        } else if (msg instanceof CleanupAllVmMetadataOnPrimaryStorageMsg) {
+            handle((CleanupAllVmMetadataOnPrimaryStorageMsg) msg);
         } else {
             super.handleLocalMessage(msg);
         }
@@ -2089,4 +2091,42 @@ public void fail(ErrorCode errorCode) {
             }
         });
     }
+
+    @Override
+    protected void handle(CleanupAllVmMetadataOnPrimaryStorageMsg msg) {
+        CleanupAllVmMetadataOnPrimaryStorageReply reply = new CleanupAllVmMetadataOnPrimaryStorageReply();
+        List<HostInventory> connectedHosts = factory.getConnectedHostForOperation(getSelfInventory());
+        if (connectedHosts.isEmpty()) {
+            reply.setError(operr("no connected host found for NFS primary storage[uuid:%s]", self.getUuid()));
+            bus.reply(msg, reply);
+            return;
+        }
+        cleanupAllOnHostWithFallback(msg, reply, connectedHosts, 0);
+    }
+
+    private void cleanupAllOnHostWithFallback(CleanupAllVmMetadataOnPrimaryStorageMsg msg,
+                                              CleanupAllVmMetadataOnPrimaryStorageReply reply,
+                                              List<HostInventory> connectedHosts, int idx) {
+        if (idx >= connectedHosts.size()) {
+            reply.setError(operr("failed to cleanup all vm metadata on NFS primary storage[uuid:%s] after trying %d connected host(s)",
+                    self.getUuid(), connectedHosts.size()));
+            bus.reply(msg, reply);
+            return;
+        }
+        String hostUuid = connectedHosts.get(idx).getUuid();
+        final NfsPrimaryStorageBackend backend = getBackendByHostUuid(hostUuid);
+        backend.handle(msg, hostUuid, new ReturnValueCompletion<CleanupAllVmMetadataOnPrimaryStorageReply>(msg) {
+            @Override
+            public void success(CleanupAllVmMetadataOnPrimaryStorageReply r) {
+                bus.reply(msg, r);
+            }
+
+            @Override
+            public void fail(ErrorCode errorCode) {
+                logger.warn(String.format("[MetadataCleanup] cleanAll: NFS ps[uuid:%s] failed on host[uuid:%s]: %s; trying next connected host",
+                        self.getUuid(), hostUuid, errorCode));
+                cleanupAllOnHostWithFallback(msg, reply, connectedHosts, idx + 1);
+            }
+        });
+    }
 }