Skip to content

Commit

Permalink
Merge pull request apache#1602 from nvazquez/clonegranular
Browse files Browse the repository at this point in the history
CLOUDSTACK-9422: Granular 'vmware.create.full.clone' as Primary Storage setting### Introduction

For VMware, It is possible to decide creating VMs as full clones on ESX HV, adjusting `vmware.create.full.clone` global setting. We would like to introduce this property as a primary storage detail, and use its value instead of global setting's value.

We propose introducing `fullCloneFlag` on `PrimaryDataStoreTO` sent on `CopyCommand`. This way we can reconfigure `VmwareStorageProcessor` and `VmwareStorageSubsystemCommandHandler` similar as it was done for `nfsVersion` but refactoring it to be more general.

* pr/1602:
  CLOUDSTACK-9422: Granular VMware vms creation as full clones on HV

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
  • Loading branch information
rohityadavcloud committed Sep 21, 2016
2 parents 43f82c9 + bb275a5 commit 3a82636
Show file tree
Hide file tree
Showing 13 changed files with 340 additions and 68 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public class PrimaryDataStoreTO implements DataStoreTO {
private final String url;
private Map<String, String> details;
private static final String pathSeparator = "/";
private Boolean fullCloneFlag;

public PrimaryDataStoreTO(PrimaryDataStore dataStore) {
this.uuid = dataStore.getUuid();
Expand Down Expand Up @@ -144,4 +145,12 @@ public String toString() {
.append("]")
.toString();
}

public Boolean isFullCloneFlag() {
return fullCloneFlag;
}

public void setFullCloneFlag(Boolean fullCloneFlag) {
this.fullCloneFlag = fullCloneFlag;
}
}
10 changes: 10 additions & 0 deletions engine/components-api/src/com/cloud/capacity/CapacityManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public interface CapacityManager {
static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold";
static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor";
static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold";
static final String VmwareCreateCloneFullCK = "vmware.create.full.clone";

static final ConfigKey<Float> CpuOverprovisioningFactor = new ConfigKey<Float>(Float.class, CpuOverprovisioningFactorCK, "Advanced", "1.0",
"Used for CPU overprovisioning calculation; available CPU will be (actualCpuCapacity * cpu.overprovisioning.factor)", true, ConfigKey.Scope.Cluster, null);
Expand Down Expand Up @@ -63,6 +64,15 @@ public interface CapacityManager {
true,
ConfigKey.Scope.Cluster,
null);
static final ConfigKey<Boolean> VmwareCreateCloneFull =
new ConfigKey<Boolean>(
"Storage",
Boolean.class,
VmwareCreateCloneFullCK,
"false",
"If set to true, creates VMs as full clones on ESX hypervisor",
true,
ConfigKey.Scope.StoragePool);

public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.agent.manager.allocator.PodAllocator;
import com.cloud.capacity.CapacityManager;
import com.cloud.cluster.ClusterManager;
import com.cloud.configuration.Resource.ResourceType;
import com.cloud.dc.DataCenter;
Expand Down Expand Up @@ -120,6 +121,7 @@
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.vm.DiskProfile;
import com.cloud.vm.UserVmCloneSettingVO;
import com.cloud.vm.UserVmVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachine.State;
Expand All @@ -129,9 +131,15 @@
import com.cloud.vm.VmWorkMigrateVolume;
import com.cloud.vm.VmWorkSerializer;
import com.cloud.vm.VmWorkTakeVolumeSnapshot;
import com.cloud.vm.dao.UserVmCloneSettingDao;
import com.cloud.vm.dao.UserVmDao;

public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable {

public enum UserVmCloneType {
full, linked
}

private static final Logger s_logger = Logger.getLogger(VolumeOrchestrator.class);

@Inject
Expand Down Expand Up @@ -178,6 +186,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
ClusterManager clusterManager;
@Inject
StorageManager storageMgr;
@Inject
protected UserVmCloneSettingDao _vmCloneSettingDao;

private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
protected List<StoragePoolAllocator> _storagePoolAllocators;
Expand Down Expand Up @@ -1353,6 +1363,33 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
disk.setDetails(getDetails(volumeInfo, dataStore));

vm.addDisk(disk);

// If hypervisor is vSphere, check for clone type setting.
if (vm.getHypervisorType().equals(HypervisorType.VMware)) {
// retrieve clone flag.
UserVmCloneType cloneType = UserVmCloneType.linked;
Boolean value = CapacityManager.VmwareCreateCloneFull.valueIn(vol.getPoolId());
if (value != null && value) {
cloneType = UserVmCloneType.full;
}
try {
UserVmCloneSettingVO cloneSettingVO = _vmCloneSettingDao.findByVmId(vm.getId());
if (cloneSettingVO != null){
if (! cloneSettingVO.getCloneType().equals(cloneType.toString())){
cloneSettingVO.setCloneType(cloneType.toString());
_vmCloneSettingDao.update(cloneSettingVO.getVmId(), cloneSettingVO);
}
}
else {
UserVmCloneSettingVO vmCloneSettingVO = new UserVmCloneSettingVO(vm.getId(), cloneType.toString());
_vmCloneSettingDao.persist(vmCloneSettingVO);
}
}
catch (Throwable e){
s_logger.debug("[NSX_PLUGIN_LOG] ERROR: " + e.getMessage());
}
}

}
}

Expand Down
4 changes: 4 additions & 0 deletions engine/schema/src/com/cloud/vm/UserVmCloneSettingVO.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,8 @@ public long getVmId() {
public String getCloneType() {
return this.cloneType;
}

public void setCloneType(String cloneType) {
this.cloneType = cloneType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public UserVmCloneSettingDaoImpl() {
public void init() {
// Initialize the search builders.
vmIdSearch = createSearchBuilder();
vmIdSearch.and("vmId", vmIdSearch.entity().getCloneType(), Op.EQ);
vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), Op.EQ);
vmIdSearch.done();

cloneTypeSearch = createSearchBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.cloudstack.storage.RemoteHostEndPoint;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;

Expand All @@ -56,8 +57,10 @@
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.capacity.CapacityManager;
import com.cloud.configuration.Config;
import com.cloud.host.Host;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StoragePool;
Expand Down Expand Up @@ -153,7 +156,7 @@ protected Answer copyObject(DataObject srcData, DataObject destData, Host destHo
srcForCopy = cacheData = cacheMgr.createCacheObject(srcData, destScope);
}

CopyCommand cmd = new CopyCommand(srcForCopy.getTO(), destData.getTO(), _primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(srcForCopy.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value());
EndPoint ep = destHost != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(destHost) : selector.select(srcForCopy, destData);
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
Expand Down Expand Up @@ -201,6 +204,23 @@ protected Answer copyObject(DataObject srcData, DataObject destData, Host destHo
}
}

/**
* Adds {@code 'vmware.create.full.clone'} value for a given primary storage, whose HV is VMware, on datastore's {@code fullCloneFlag} field
* @param dataTO Dest data store TO
* @return dataTO including fullCloneFlag, if provided
*/
protected DataTO addFullCloneFlagOnVMwareDest(DataTO dataTO) {
if (dataTO != null && dataTO.getHypervisorType().equals(Hypervisor.HypervisorType.VMware)){
DataStoreTO dataStoreTO = dataTO.getDataStore();
if (dataStoreTO != null && dataStoreTO instanceof PrimaryDataStoreTO){
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) dataStoreTO;
Boolean value = CapacityManager.VmwareCreateCloneFull.valueIn(primaryDataStoreTO.getId());
primaryDataStoreTO.setFullCloneFlag(value);
}
}
return dataTO;
}

protected Answer copyObject(DataObject srcData, DataObject destData) {
return copyObject(srcData, destData, null);
}
Expand Down Expand Up @@ -257,7 +277,7 @@ protected Answer copyVolumeFromSnapshot(DataObject snapObj, DataObject volObj) {
ep = selector.select(srcData, volObj);
}

CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(srcData.getTO(), addFullCloneFlagOnVMwareDest(volObj.getTO()), _createVolumeFromSnapshotWait, VirtualMachineManager.ExecuteInSequence.value());
Answer answer = null;
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
Expand All @@ -280,7 +300,7 @@ protected Answer copyVolumeFromSnapshot(DataObject snapObj, DataObject volObj) {
}

protected Answer cloneVolume(DataObject template, DataObject volume) {
CopyCommand cmd = new CopyCommand(template.getTO(), volume.getTO(), 0, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(template.getTO(), addFullCloneFlagOnVMwareDest(volume.getTO()), 0, VirtualMachineManager.ExecuteInSequence.value());
try {
EndPoint ep = selector.select(volume.getDataStore());
Answer answer = null;
Expand Down Expand Up @@ -330,7 +350,7 @@ protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData)

objOnImageStore.processEvent(Event.CopyingRequested);

CopyCommand cmd = new CopyCommand(objOnImageStore.getTO(), destData.getTO(), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(objOnImageStore.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value());
EndPoint ep = selector.select(objOnImageStore, destData);
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
Expand Down Expand Up @@ -477,7 +497,7 @@ protected Answer createTemplateFromSnapshot(DataObject srcData, DataObject destD
ep = selector.select(srcData, destData);
}

CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _createprivatetemplatefromsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(srcData.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _createprivatetemplatefromsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
Answer answer = null;
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
Expand Down Expand Up @@ -513,7 +533,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
Scope selectedScope = pickCacheScopeForCopy(srcData, destData);
cacheData = cacheMgr.getCacheObject(srcData, selectedScope);

CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
CopyCommand cmd = new CopyCommand(srcData.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _backupsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
cmd.setCacheTO(cacheData.getTO());
cmd.setOptions(options);
EndPoint ep = selector.select(srcData, destData);
Expand All @@ -525,6 +545,7 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
answer = ep.sendMessage(cmd);
}
} else {
addFullCloneFlagOnVMwareDest(destData.getTO());
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, VirtualMachineManager.ExecuteInSequence.value());
cmd.setOptions(options);
EndPoint ep = selector.select(srcData, destData, StorageAction.BACKUPSNAPSHOT);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.storage.motion;

import static org.mockito.Mockito.when;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;

import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.any;

import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;

import com.cloud.agent.api.to.DataTO;
import com.cloud.capacity.CapacityManager;
import com.cloud.hypervisor.Hypervisor.HypervisorType;

@RunWith(PowerMockRunner.class)
@PrepareForTest(CapacityManager.class)
public class AncientDataMotionStrategyTest {

@Spy
@InjectMocks
private AncientDataMotionStrategy strategy = new AncientDataMotionStrategy();

@Mock
DataTO dataTO;
@Mock
PrimaryDataStoreTO dataStoreTO;
@Mock
ConfigKey<Boolean> vmwareKey;

private static final long POOL_ID = 1l;
private static final Boolean FULL_CLONE_FLAG = true;

@Before
public void setup() throws Exception {
MockitoAnnotations.initMocks(this);

replaceVmwareCreateCloneFullField();

when(vmwareKey.valueIn(POOL_ID)).thenReturn(FULL_CLONE_FLAG);

when(dataTO.getHypervisorType()).thenReturn(HypervisorType.VMware);
when(dataTO.getDataStore()).thenReturn(dataStoreTO);
when(dataStoreTO.getId()).thenReturn(POOL_ID);
}

private void replaceVmwareCreateCloneFullField() throws Exception {
Field field = CapacityManager.class.getDeclaredField("VmwareCreateCloneFull");
field.setAccessible(true);
// remove final modifier from field
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
field.set(null, vmwareKey);
}

@Test
public void testAddFullCloneFlagOnVMwareDest(){
strategy.addFullCloneFlagOnVMwareDest(dataTO);
verify(dataStoreTO).setFullCloneFlag(FULL_CLONE_FLAG);
}

@Test
public void testAddFullCloneFlagOnNotVmwareDest(){
when(dataTO.getHypervisorType()).thenReturn(HypervisorType.Any);
verify(dataStoreTO, never()).setFullCloneFlag(any(Boolean.class));
}

}
Loading

0 comments on commit 3a82636

Please sign in to comment.