Thanks to visit codestin.com
Credit goes to github.com

Skip to content

NSX: Cleanup NSX resources during k8s cluster cleanup #8528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 19, 2024
Merged

NSX: Cleanup NSX resources during k8s cluster cleanup #8528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@
public interface KubernetesClusterHelper extends Adapter {

ControlledEntity findByUuid(String uuid);
ControlledEntity findByVmId(long vmId);
}
13 changes: 13 additions & 0 deletions ...netes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,38 @@
package com.cloud.kubernetes.cluster;

import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.utils.component.AdapterBase;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.springframework.stereotype.Component;

import javax.inject.Inject;
import java.util.Objects;

@Component
public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {

@Inject
private KubernetesClusterDao kubernetesClusterDao;
@Inject
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;

@Override
public ControlledEntity findByUuid(String uuid) {
return kubernetesClusterDao.findByUuid(uuid);
}

@Override
public ControlledEntity findByVmId(long vmId) {
KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
if (Objects.isNull(clusterVmMapVO)) {
return null;
}
return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
}

@Override
public String getConfigComponentName() {
return KubernetesClusterHelper.class.getSimpleName();
Expand Down
2 changes: 2 additions & 0 deletions ...tes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);

public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);

int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
Expand Down
12 changes: 12 additions & 0 deletions ...service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,17 @@
public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {

private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;

public KubernetesClusterVmMapDaoImpl() {
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
clusterIdSearch.done();

vmIdSearch = createSearchBuilder();
vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
vmIdSearch.done();
}

@Override
Expand All @@ -47,6 +52,13 @@ public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId) {
return listBy(sc, filter);
}

@Override
public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
sc.setParameters("vmId", vmId);
return findOneBy(sc);
}

@Override
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();
Expand Down
21 changes: 12 additions & 9 deletions plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -505,10 +505,12 @@ public boolean applyPFRules(Network network, List<PortForwardingRule> rules) thr
if (!canHandle(network, Network.Service.PortForwarding)) {
return false;
}
boolean result = true;
for (PortForwardingRule rule : rules) {
IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
(vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
continue;
}
NsxOpObject nsxObject = getNsxOpObject(network);
Expand All @@ -523,21 +525,21 @@ public boolean applyPFRules(Network network, List<PortForwardingRule> rules) thr
.setNetworkResourceId(nsxObject.getNetworkResourceId())
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
.setVmId(vm.getId())
.setVmIp(vm.getPrivateIpAddress())
.setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
.setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
.setPublicIp(publicIp.getAddress().addr())
.setPrivatePort(privatePort)
.setPublicPort(publicPort)
.setRuleId(rule.getId())
.setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
.build();
if (rule.getState() == FirewallRule.State.Add) {
return nsxService.createPortForwardRule(networkRule);
result &= nsxService.createPortForwardRule(networkRule);
} else if (rule.getState() == FirewallRule.State.Revoke) {
return nsxService.deletePortForwardRule(networkRule);
result &= nsxService.deletePortForwardRule(networkRule);
}
}
return true;
return result;
}

public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
Expand Down Expand Up @@ -613,6 +615,7 @@ private NsxOpObject getNsxOpObject(Network network) {

@Override
public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
boolean result = true;
for (LoadBalancingRule loadBalancingRule : rules) {
if (loadBalancingRule.getState() == FirewallRule.State.Active) {
continue;
Expand All @@ -638,12 +641,12 @@ public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) thro
.setAlgorithm(loadBalancingRule.getAlgorithm())
.build();
if (loadBalancingRule.getState() == FirewallRule.State.Add) {
return nsxService.createLbRule(networkRule);
result &= nsxService.createLbRule(networkRule);
} else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
return nsxService.deleteLbRule(networkRule);
result &= nsxService.deleteLbRule(networkRule);
}
}
return true;
return result;
}

@Override
Expand Down
28 changes: 23 additions & 5 deletions server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;

import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.element.NsxProviderVO;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
Expand Down Expand Up @@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir

@Inject
VMScheduleManager vmScheduleManager;
@Inject
NsxProviderDao nsxProviderDao;

private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
Expand All @@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private boolean _dailyOrHourly = false;
private int capacityReleaseInterval;
private ExecutorService _vmIpFetchThreadExecutor;
private List<KubernetesClusterHelper> kubernetesClusterHelpers;


private String _instance;
Expand All @@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private static final int NUM_OF_2K_BLOCKS = 512;
private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;

public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
return kubernetesClusterHelpers;
}

public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
this.kubernetesClusterHelpers = kubernetesClusterHelpers;
}

@Inject
private OrchestrationService _orchSrvc;

Expand Down Expand Up @@ -2528,11 +2542,15 @@ private boolean cleanupVmResources(long vmId) {
}

// cleanup port forwarding rules
if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
} else {
success = false;
s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
} else {
success = false;
s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
}
}

// cleanup load balancer rules
Expand Down
5 changes: 3 additions & 2 deletions server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,9 @@

<bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />


<bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
<bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
<property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
</bean>

<bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
<property name="consoleProxyAllocators"
Expand Down