diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java index e445e50f82cb..e160227749db 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java @@ -22,4 +22,5 @@ public interface KubernetesClusterHelper extends Adapter { ControlledEntity findByUuid(String uuid); + ControlledEntity findByVmId(long vmId); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java index 0ef916ab9591..60bd81c7c5a5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java @@ -17,6 +17,7 @@ package com.cloud.kubernetes.cluster; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.utils.component.AdapterBase; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.framework.config.ConfigKey; @@ -24,18 +25,30 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; +import java.util.Objects; @Component public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable { @Inject private KubernetesClusterDao kubernetesClusterDao; + @Inject + private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Override public ControlledEntity findByUuid(String uuid) { return kubernetesClusterDao.findByUuid(uuid); } + @Override + public ControlledEntity findByVmId(long vmId) { + KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId); + if (Objects.isNull(clusterVmMapVO)) { + return null; + } + return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId()); + } + @Override public String getConfigComponentName() { return KubernetesClusterHelper.class.getSimpleName(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java index 688a611ac998..45c0b79485c0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java @@ -23,6 +23,8 @@ public interface KubernetesClusterVmMapDao extends GenericDao { public List listByClusterId(long clusterId); + + public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId); public List listByClusterIdAndVmIdsIn(long clusterId, List vmIds); int removeByClusterIdAndVmIdsIn(long clusterId, List vmIds); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java index b9f2ec917b2b..0d90a4cdaca9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -31,12 +31,17 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase implements KubernetesClusterVmMapDao { private final SearchBuilder clusterIdSearch; + private final SearchBuilder vmIdSearch; public KubernetesClusterVmMapDaoImpl() { clusterIdSearch = createSearchBuilder(); clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN); clusterIdSearch.done(); + + vmIdSearch = createSearchBuilder(); + vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ); + vmIdSearch.done(); } @Override @@ -47,6 +52,13 @@ public List listByClusterId(long clusterId) { return listBy(sc, filter); } + @Override + public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) { + SearchCriteria sc = vmIdSearch.create(); + sc.setParameters("vmId", vmId); + return findOneBy(sc); + } + @Override public List listByClusterIdAndVmIdsIn(long clusterId, List vmIds) { SearchCriteria sc = clusterIdSearch.create(); diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index 0986c367cf9f..571e68496d62 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -505,10 +505,12 @@ public boolean applyPFRules(Network network, List rules) thr if (!canHandle(network, Network.Service.PortForwarding)) { return false; } + boolean result = true; for (PortForwardingRule rule : rules) { IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId()); UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId()); - if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) { + if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) || + (vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) { continue; } NsxOpObject nsxObject = getNsxOpObject(network); @@ -523,8 +525,8 @@ public boolean applyPFRules(Network network, List rules) thr .setNetworkResourceId(nsxObject.getNetworkResourceId()) .setNetworkResourceName(nsxObject.getNetworkResourceName()) .setVpcResource(nsxObject.isVpcResource()) - .setVmId(vm.getId()) - .setVmIp(vm.getPrivateIpAddress()) + .setVmId(Objects.nonNull(vm) ? vm.getId() : 0) + .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null) .setPublicIp(publicIp.getAddress().addr()) .setPrivatePort(privatePort) .setPublicPort(publicPort) @@ -532,12 +534,12 @@ public boolean applyPFRules(Network network, List rules) thr .setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT)) .build(); if (rule.getState() == FirewallRule.State.Add) { - return nsxService.createPortForwardRule(networkRule); + result &= nsxService.createPortForwardRule(networkRule); } else if (rule.getState() == FirewallRule.State.Revoke) { - return nsxService.deletePortForwardRule(networkRule); + result &= nsxService.deletePortForwardRule(networkRule); } } - return true; + return result; } public Pair getVpcOrNetwork(Long vpcId, long networkId) { @@ -613,6 +615,7 @@ private NsxOpObject getNsxOpObject(Network network) { @Override public boolean applyLBRules(Network network, List rules) throws ResourceUnavailableException { + boolean result = true; for (LoadBalancingRule loadBalancingRule : rules) { if (loadBalancingRule.getState() == FirewallRule.State.Active) { continue; @@ -638,12 +641,12 @@ public boolean applyLBRules(Network network, List rules) thro .setAlgorithm(loadBalancingRule.getAlgorithm()) .build(); if (loadBalancingRule.getState() == FirewallRule.State.Add) { - return nsxService.createLbRule(networkRule); + result &= nsxService.createLbRule(networkRule); } else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) { - return nsxService.deleteLbRule(networkRule); + result &= nsxService.deleteLbRule(networkRule); } } - return true; + return result; } @Override diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 2927db638abc..844e03807570 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -51,6 +51,9 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper; +import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.element.NsxProviderVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject VMScheduleManager vmScheduleManager; + @Inject + NsxProviderDao nsxProviderDao; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private boolean _dailyOrHourly = false; private int capacityReleaseInterval; private ExecutorService _vmIpFetchThreadExecutor; + private List kubernetesClusterHelpers; private String _instance; @@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private static final int NUM_OF_2K_BLOCKS = 512; private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES; + public List getKubernetesClusterHelpers() { + return kubernetesClusterHelpers; + } + + public void setKubernetesClusterHelpers(final List kubernetesClusterHelpers) { + this.kubernetesClusterHelpers = kubernetesClusterHelpers; + } + @Inject private OrchestrationService _orchSrvc; @@ -2528,11 +2542,15 @@ private boolean cleanupVmResources(long vmId) { } // cleanup port forwarding rules - if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) { - s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge"); - } else { - success = false; - s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge"); + VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId); + NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId()); + if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) { + if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) { + s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge"); + } else { + success = false; + s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge"); + } } // cleanup load balancer rules diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 7227264e2297..9656d544fbb0 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -106,8 +106,9 @@ - - + + +