Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 40cc10a

Browse files
harikrishna-patnalaharikrishna-patnala
authored
Allow volume migrations in ScaleIO within and across ScaleIO storage clusters (#7408)
* Live storage migration of volume in scaleIO within same storage scaleio cluster * Added migrate command * Recent changes of migration across clusters * Fixed uuid * recent changes * Pivot changes * working blockcopy api in libvirt * Checking block copy status * Formatting code * Fixed failures * code refactoring and some changes * Removed unused methods * removed unused imports * Unit tests to check if volume belongs to same or different storage scaleio cluster * Unit tests for volume livemigration in ScaleIOPrimaryDataStoreDriver * Fixed offline volume migration case and allowed encrypted volume migration * Added more integration tests * Support for migration of encrypted volumes across different scaleio clusters * Fix UI notifications for migrate volume * Data volume offline migration: save encryption details to destination volume entry * Offline storage migration for scaleio encrypted volumes * Allow multiple Volumes to be migrated with migrateVirtualMachineWithVolume API * Removed unused unittests * Removed duplicate keys in migrate volume vue file * Fix Unit tests * Add volume secrets if does not exists during volume migrations. secrets are getting cleared on package upgrades. * Fix secret UUID for encrypted volume migration * Added a null check for secret before removing * Added more unit tests * Fixed passphrase check * Add image options to the encypted volume conversion
1 parent 3748f32 commit 40cc10a

15 files changed

Lines changed: 1722 additions & 39 deletions

File tree

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@
212212
import com.cloud.storage.DiskOfferingVO;
213213
import com.cloud.storage.ScopeType;
214214
import com.cloud.storage.Storage.ImageFormat;
215+
import com.cloud.storage.Storage;
215216
import com.cloud.storage.StorageManager;
216217
import com.cloud.storage.StoragePool;
217218
import com.cloud.storage.VMTemplateVO;
@@ -2925,7 +2926,7 @@ protected Map<Volume, StoragePool> buildMapUsingUserInformation(VirtualMachinePr
29252926
* </ul>
29262927
*/
29272928
protected void executeManagedStorageChecksWhenTargetStoragePoolProvided(StoragePoolVO currentPool, VolumeVO volume, StoragePoolVO targetPool) {
2928-
if (!currentPool.isManaged()) {
2929+
if (!currentPool.isManaged() || currentPool.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) {
29292930
return;
29302931
}
29312932
if (currentPool.getId() == targetPool.getId()) {

‎engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
import java.util.HashMap;
3333
import java.util.List;
3434
import java.util.Map;
35+
import java.util.Random;
36+
import java.util.stream.Collectors;
3537

3638
import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
3739
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -46,6 +48,7 @@
4648
import org.mockito.Mock;
4749
import org.mockito.Mockito;
4850
import org.mockito.Spy;
51+
import org.mockito.stubbing.Answer;
4952
import org.mockito.runners.MockitoJUnitRunner;
5053

5154
import com.cloud.agent.AgentManager;
@@ -68,6 +71,7 @@
6871
import com.cloud.service.dao.ServiceOfferingDao;
6972
import com.cloud.storage.DiskOfferingVO;
7073
import com.cloud.storage.ScopeType;
74+
import com.cloud.storage.Storage;
7175
import com.cloud.storage.StorageManager;
7276
import com.cloud.storage.StoragePool;
7377
import com.cloud.storage.StoragePoolHostVO;
@@ -373,9 +377,26 @@ public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentS
373377
Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId();
374378
}
375379

380+
@Test
381+
public void allowVolumeMigrationsForPowerFlexStorage() {
382+
Mockito.doReturn(true).when(storagePoolVoMock).isManaged();
383+
Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(storagePoolVoMock).getPoolType();
384+
385+
virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class));
386+
387+
Mockito.verify(storagePoolVoMock).isManaged();
388+
Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId();
389+
}
390+
376391
@Test
377392
public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolEqualsTargetPool() {
378393
Mockito.doReturn(true).when(storagePoolVoMock).isManaged();
394+
// return any storage type except powerflex/scaleio
395+
List<Storage.StoragePoolType> values = Arrays.asList(Storage.StoragePoolType.values());
396+
when(storagePoolVoMock.getPoolType()).thenAnswer((Answer<Storage.StoragePoolType>) invocation -> {
397+
List<Storage.StoragePoolType> filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList());
398+
int randomIndex = new Random().nextInt(filteredValues.size());
399+
return filteredValues.get(randomIndex); });
379400

380401
virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, storagePoolVoMock);
381402

@@ -386,6 +407,12 @@ public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentS
386407
@Test(expected = CloudRuntimeException.class)
387408
public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolNotEqualsTargetPool() {
388409
Mockito.doReturn(true).when(storagePoolVoMock).isManaged();
410+
// return any storage type except powerflex/scaleio
411+
List<Storage.StoragePoolType> values = Arrays.asList(Storage.StoragePoolType.values());
412+
when(storagePoolVoMock.getPoolType()).thenAnswer((Answer<Storage.StoragePoolType>) invocation -> {
413+
List<Storage.StoragePoolType> filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList());
414+
int randomIndex = new Random().nextInt(filteredValues.size());
415+
return filteredValues.get(randomIndex); });
389416

390417
virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class));
391418
}

‎engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1648,6 +1648,10 @@ protected VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePool po
16481648
newVol.setPoolType(pool.getPoolType());
16491649
newVol.setLastPoolId(lastPoolId);
16501650
newVol.setPodId(pool.getPodId());
1651+
if (volume.getPassphraseId() != null) {
1652+
newVol.setPassphraseId(volume.getPassphraseId());
1653+
newVol.setEncryptFormat(volume.getEncryptFormat());
1654+
}
16511655
return volDao.persist(newVol);
16521656
}
16531657

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.log4j.Logger;
2323
import org.libvirt.Connect;
24+
import org.libvirt.Library;
2425
import org.libvirt.LibvirtException;
2526

2627
import com.cloud.hypervisor.Hypervisor;
@@ -44,6 +45,7 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio
4445
if (conn == null) {
4546
s_logger.info("No existing libvirtd connection found. Opening a new one");
4647
conn = new Connect(hypervisorURI, false);
48+
Library.initEventLoop();
4749
s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI);
4850
s_connections.put(hypervisorURI, conn);
4951
} else {

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java‎

Lines changed: 230 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,255 @@
2929
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
3030
import com.cloud.resource.CommandWrapper;
3131
import com.cloud.resource.ResourceWrapper;
32+
import com.cloud.storage.Storage;
3233

34+
import java.io.ByteArrayInputStream;
35+
import java.io.File;
36+
import java.io.IOException;
37+
import java.io.StringWriter;
3338
import java.util.Map;
3439
import java.util.UUID;
3540

41+
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
42+
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
3643
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
3744
import org.apache.cloudstack.storage.to.VolumeObjectTO;
45+
import org.apache.commons.lang3.ArrayUtils;
46+
import org.apache.commons.lang3.StringUtils;
3847
import org.apache.log4j.Logger;
48+
import org.libvirt.Connect;
49+
import org.libvirt.Domain;
50+
import org.libvirt.DomainBlockJobInfo;
51+
import org.libvirt.DomainInfo;
52+
import org.libvirt.TypedParameter;
53+
import org.libvirt.TypedUlongParameter;
54+
import org.libvirt.LibvirtException;
55+
import org.w3c.dom.Document;
56+
import org.w3c.dom.Element;
57+
import org.w3c.dom.NodeList;
58+
import org.xml.sax.SAXException;
59+
60+
import javax.xml.parsers.DocumentBuilder;
61+
import javax.xml.parsers.DocumentBuilderFactory;
62+
import javax.xml.parsers.ParserConfigurationException;
63+
import javax.xml.transform.OutputKeys;
64+
import javax.xml.transform.Transformer;
65+
import javax.xml.transform.TransformerException;
66+
import javax.xml.transform.TransformerFactory;
67+
import javax.xml.transform.dom.DOMSource;
68+
import javax.xml.transform.stream.StreamResult;
3969

4070
@ResourceWrapper(handles = MigrateVolumeCommand.class)
41-
public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVolumeCommand, Answer, LibvirtComputingResource> {
71+
public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVolumeCommand, Answer, LibvirtComputingResource> {
4272
private static final Logger LOGGER = Logger.getLogger(LibvirtMigrateVolumeCommandWrapper.class);
4373

4474
@Override
4575
public Answer execute(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
76+
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
77+
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
78+
79+
MigrateVolumeAnswer answer;
80+
if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) {
81+
answer = migratePowerFlexVolume(command, libvirtComputingResource);
82+
} else {
83+
answer = migrateRegularVolume(command, libvirtComputingResource);
84+
}
85+
86+
return answer;
87+
}
88+
89+
protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
90+
91+
// Source Details
92+
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
93+
String srcPath = srcVolumeObjectTO.getPath();
94+
final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath());
95+
final String vmName = srcVolumeObjectTO.getVmName();
96+
97+
// Destination Details
98+
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();
99+
String destPath = destVolumeObjectTO.getPath();
100+
final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath());
101+
Map<String, String> destDetails = command.getDestDetails();
102+
final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
103+
String destDiskLabel = null;
104+
105+
final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId;
106+
final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName;
107+
108+
Domain dm = null;
109+
try {
110+
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
111+
Connect conn = libvirtUtilitiesHelper.getConnection();
112+
dm = libvirtComputingResource.getDomain(conn, vmName);
113+
if (dm == null) {
114+
return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to can not find vm: " + vmName, null);
115+
}
116+
117+
DomainInfo.DomainState domainState = dm.getInfo().state ;
118+
if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) {
119+
return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null);
120+
}
121+
122+
final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
123+
PrimaryDataStoreTO spool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore();
124+
KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid());
125+
pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null);
126+
127+
String srcSecretUUID = null;
128+
String destSecretUUID = null;
129+
if (ArrayUtils.isNotEmpty(destVolumeObjectTO.getPassphrase())) {
130+
srcSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase());
131+
destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase());
132+
}
133+
134+
String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath, destSecretUUID);
135+
destDiskLabel = generateDestinationDiskLabel(diskdef);
136+
137+
TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0);
138+
TypedParameter[] parameters = new TypedParameter[1];
139+
parameters[0] = parameter;
140+
141+
dm.blockCopy(destDiskLabel, diskdef, parameters, Domain.BlockCopyFlags.REUSE_EXT);
142+
LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath));
143+
144+
return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID);
145+
146+
} catch (Exception e) {
147+
String msg = "Migrate volume failed due to " + e.toString();
148+
LOGGER.warn(msg, e);
149+
if (destDiskLabel != null) {
150+
try {
151+
dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC);
152+
} catch (LibvirtException ex) {
153+
LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage());
154+
}
155+
}
156+
return new MigrateVolumeAnswer(command, false, msg, null);
157+
} finally {
158+
if (dm != null) {
159+
try {
160+
dm.free();
161+
} catch (LibvirtException l) {
162+
LOGGER.trace("Ignoring libvirt error.", l);
163+
};
164+
}
165+
}
166+
}
167+
168+
protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException {
169+
int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found
170+
int waitTimeInSec = command.getWait();
171+
while (waitTimeInSec > 0) {
172+
DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0);
173+
if (blockJobInfo != null) {
174+
LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end));
175+
if (blockJobInfo.cur == blockJobInfo.end) {
176+
LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
177+
dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
178+
if (StringUtils.isNotEmpty(srcSecretUUID)) {
179+
libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
180+
}
181+
break;
182+
}
183+
} else {
184+
LOGGER.info("Failed to get the block copy status, trying to abort the job");
185+
dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC);
186+
}
187+
waitTimeInSec--;
188+
189+
try {
190+
Thread.sleep(timeBetweenTries);
191+
} catch (Exception ex) {
192+
// don't do anything
193+
}
194+
}
195+
196+
if (waitTimeInSec <= 0) {
197+
String msg = "Block copy is taking long time, failing the job";
198+
LOGGER.error(msg);
199+
try {
200+
dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC);
201+
} catch (LibvirtException ex) {
202+
LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage());
203+
}
204+
return new MigrateVolumeAnswer(command, false, msg, null);
205+
}
206+
207+
return new MigrateVolumeAnswer(command, true, null, destPath);
208+
}
209+
210+
private String generateDestinationDiskLabel(String diskXml) throws ParserConfigurationException, IOException, SAXException {
211+
212+
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
213+
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
214+
Document doc = dBuilder.parse(new ByteArrayInputStream(diskXml.getBytes("UTF-8")));
215+
doc.getDocumentElement().normalize();
216+
217+
Element disk = doc.getDocumentElement();
218+
String diskLabel = getAttrValue("target", "dev", disk);
219+
220+
return diskLabel;
221+
}
222+
223+
protected String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException {
224+
final String domXml = dm.getXMLDesc(0);
225+
226+
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
227+
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
228+
Document doc = dBuilder.parse(new ByteArrayInputStream(domXml.getBytes("UTF-8")));
229+
doc.getDocumentElement().normalize();
230+
231+
NodeList disks = doc.getElementsByTagName("disk");
232+
233+
for (int i = 0; i < disks.getLength(); i++) {
234+
Element disk = (Element)disks.item(i);
235+
String type = disk.getAttribute("type");
236+
if (!type.equalsIgnoreCase("network")) {
237+
String diskDev = getAttrValue("source", "dev", disk);
238+
if (StringUtils.isNotEmpty(diskDev) && diskDev.contains(srcVolumeId)) {
239+
setAttrValue("source", "dev", diskFilePath, disk);
240+
if (StringUtils.isNotEmpty(destSecretUUID)) {
241+
setAttrValue("secret", "uuid", destSecretUUID, disk);
242+
}
243+
StringWriter diskSection = new StringWriter();
244+
Transformer xformer = TransformerFactory.newInstance().newTransformer();
245+
xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
246+
xformer.transform(new DOMSource(disk), new StreamResult(diskSection));
247+
248+
return diskSection.toString();
249+
}
250+
}
251+
}
252+
253+
return null;
254+
}
255+
256+
private static String getAttrValue(String tag, String attr, Element eElement) {
257+
NodeList tagNode = eElement.getElementsByTagName(tag);
258+
if (tagNode.getLength() == 0) {
259+
return null;
260+
}
261+
Element node = (Element)tagNode.item(0);
262+
return node.getAttribute(attr);
263+
}
264+
265+
private static void setAttrValue(String tag, String attr, String newValue, Element eElement) {
266+
NodeList tagNode = eElement.getElementsByTagName(tag);
267+
if (tagNode.getLength() == 0) {
268+
return;
269+
}
270+
Element node = (Element)tagNode.item(0);
271+
node.setAttribute(attr, newValue);
272+
}
273+
274+
protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
46275
KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr();
47276

48277
VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData();
49278
PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore();
50279

51280
Map<String, String> srcDetails = command.getSrcDetails();
52-
53281
String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath();
54282

55283
VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData();

0 commit comments

Comments
 (0)