diff --git a/pom.xml b/pom.xml index 2325f9fb4..782bf3a59 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ THE SOFTWARE. ec2 - 1.41 + 1.41-SNAPSHOT hpi Amazon EC2 plugin http://wiki.jenkins-ci.org/display/JENKINS/Amazon+EC2+Plugin diff --git a/src/main/java/hudson/plugins/ec2/AMITypeData.java b/src/main/java/hudson/plugins/ec2/AMITypeData.java index 6cb3b623f..a47b85588 100644 --- a/src/main/java/hudson/plugins/ec2/AMITypeData.java +++ b/src/main/java/hudson/plugins/ec2/AMITypeData.java @@ -6,4 +6,6 @@ public abstract class AMITypeData extends AbstractDescribableImpl { public abstract boolean isWindows(); public abstract boolean isUnix(); + + public abstract String getProductType(); } diff --git a/src/main/java/hudson/plugins/ec2/EC2Cloud.java b/src/main/java/hudson/plugins/ec2/EC2Cloud.java index 39c849df1..f8de81642 100644 --- a/src/main/java/hudson/plugins/ec2/EC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/EC2Cloud.java @@ -18,6 +18,7 @@ */ package hudson.plugins.ec2; +import static java.util.stream.Collectors.toList; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import com.amazonaws.ClientConfiguration; @@ -36,6 +37,7 @@ import com.cloudbees.plugins.credentials.SystemCredentialsProvider; import com.cloudbees.plugins.credentials.common.StandardListBoxModel; import com.cloudbees.plugins.credentials.domains.Domain; +import com.google.common.collect.Ordering; import edu.umd.cs.findbugs.annotations.CheckForNull; import edu.umd.cs.findbugs.annotations.Nullable; import hudson.ProxyConfiguration; @@ -54,24 +56,18 @@ import java.io.PrintWriter; import java.io.StringReader; import java.io.StringWriter; +import java.math.BigDecimal; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.Proxy; import java.net.URL; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.HashMap; -import java.util.UUID; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.Comparator; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; @@ -305,18 +301,80 @@ public SlaveTemplate getTemplate(String template) { * Gets {@link SlaveTemplate} that has the matching {@link Label}. */ public SlaveTemplate getTemplate(Label label) { - for (SlaveTemplate t : templates) { + Predicate applicableForLabel = t -> { if (t.getMode() == Node.Mode.NORMAL) { if (label == null || label.matches(t.getLabelSet())) { - return t; + return true; } } else if (t.getMode() == Node.Mode.EXCLUSIVE) { if (label != null && label.matches(t.getLabelSet())) { - return t; + return true; } } + return false; + }; + + List applicableTemplates = templates.stream() + .filter(applicableForLabel) + .collect(toList()); + + if (applicableTemplates.isEmpty()) { + return null; + } else if (applicableTemplates.size() == 1) { + return applicableTemplates.get(0); + } else { + LOGGER.log(Level.INFO, "Found multiple applicable templates for label '" + label + "'. Selecting the cheapest."); + return selectCheapest(applicableTemplates); } - return null; + } + + private SlaveTemplate selectCheapest(Collection templates) { + class TemplatePrice { + private final SlaveTemplate template; + private final BigDecimal price; + + private TemplatePrice(SlaveTemplate template, BigDecimal price) { + this.template = template; + this.price = price; + } + } + + Ordering priceLowestFirst = Ordering.natural().onResultOf(t -> t.price); + + return templates.stream() + .map(template -> new TemplatePrice(template, getSpotInstancePrice(template))) + .sorted(priceLowestFirst) + .peek(tp -> LOGGER.log(Level.INFO, "Price for template " + tp.template + " of instance type " + tp.template.type + " in zone " + tp.template.zone + " is " + tp.price)) + .findFirst() + .map(tp -> tp.template) + .orElse(null); + } + + private BigDecimal getTemplatePrice(SlaveTemplate template) { + if (template.spotConfig != null) { + return getSpotInstancePrice(template); + } else { + return getEc2InstancePrice(template); + } + } + + private BigDecimal getEc2InstancePrice(SlaveTemplate template) { + // TODO This is hard because the API for this is hard. For now just return 100 and call it a day. + return new BigDecimal(100); + } + + private BigDecimal getSpotInstancePrice(SlaveTemplate template) { + DescribeSpotPriceHistoryRequest request = new DescribeSpotPriceHistoryRequest(); + request.setAvailabilityZone(template.zone); + request.setInstanceTypes(Collections.singletonList(template.type.toString())); + request.setMaxResults(1); + request.setProductDescriptions(Collections.singletonList(template.amiType.getProductType())); + + AmazonEC2 ec2 = connect(); + DescribeSpotPriceHistoryResult response = ec2.describeSpotPriceHistory(request); + + // We asked for one so we'll get one + return new BigDecimal(response.getSpotPriceHistory().get(0).getSpotPrice()); } /** diff --git a/src/main/java/hudson/plugins/ec2/UnixData.java b/src/main/java/hudson/plugins/ec2/UnixData.java index f2a63e9be..2099f68af 100644 --- a/src/main/java/hudson/plugins/ec2/UnixData.java +++ b/src/main/java/hudson/plugins/ec2/UnixData.java @@ -93,6 +93,11 @@ public String getSshPort() { return sshPort == null || sshPort.isEmpty() ? "22" : sshPort; } + @Override + public String getProductType() { + return "Linux/UNIX"; + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/main/java/hudson/plugins/ec2/WindowsData.java b/src/main/java/hudson/plugins/ec2/WindowsData.java index b18682355..94bd9f417 100644 --- a/src/main/java/hudson/plugins/ec2/WindowsData.java +++ b/src/main/java/hudson/plugins/ec2/WindowsData.java @@ -59,6 +59,11 @@ public String getDisplayName() { } } + @Override + public String getProductType() { + return "Windows"; + } + @Override public int hashCode() { final int prime = 31; diff --git a/src/test/java/hudson/plugins/ec2/AmazonEC2CloudTest.java b/src/test/java/hudson/plugins/ec2/AmazonEC2CloudTest.java index 64097e213..988e456fe 100644 --- a/src/test/java/hudson/plugins/ec2/AmazonEC2CloudTest.java +++ b/src/test/java/hudson/plugins/ec2/AmazonEC2CloudTest.java @@ -23,13 +23,26 @@ */ package hudson.plugins.ec2; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryRequest; +import com.amazonaws.services.ec2.model.DescribeSpotPriceHistoryResult; +import com.amazonaws.services.ec2.model.InstanceType; +import com.amazonaws.services.ec2.model.SpotPrice; +import hudson.model.Label; +import hudson.model.Node; import hudson.slaves.Cloud; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; +import org.junit.*; import org.jvnet.hudson.test.JenkinsRule; +import org.mockito.Mockito; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; /** * @author Kohsuke Kawaguchi @@ -52,10 +65,67 @@ public void tearDown() throws Exception { @After public void testConfigRoundtrip() throws Exception { AmazonEC2Cloud orig = new AmazonEC2Cloud("us-east-1", true, "abc", "us-east-1", "ghi", "3", Collections. emptyList(),"roleArn", "roleSessionName"); - r.jenkins.clouds.add(orig); + r.jenkins.clouds.add(orig); r.submit(r.createWebClient().goTo("configure").getFormByName("config")); Cloud actual = r.jenkins.clouds.iterator().next(); r.assertEqualBeans(orig, actual, "cloudName,region,useInstanceProfileForCredentials,accessId,privateKey,instanceCap,roleArn,roleSessionName"); } + + @Test + public void testCheapestSpotInstanceUsed() { + + final String expensiveZone = "eu-west-1a"; + final String middleZone = "eu-west-1b"; + final String cheapZone = "eu-west-1c"; + + final String label = "myLabel"; + SlaveTemplate expensiveSpot = new SlaveTemplate("1", expensiveZone, new SpotConfiguration("0.33"), "default", "foo", InstanceType.M1Large, false, label, Node.Mode.NORMAL, "", "bar", "bbb", "aaa", "10", "fff", null, "-Xmx1g", false, "subnet 456", null, null, false, null, "iamInstanceProfile", false, false, false, null, true, "", false, true); + SlaveTemplate cheapestSpot = new SlaveTemplate("2", cheapZone, new SpotConfiguration("0.33"), "default", "foo", InstanceType.M1Large, false, label, Node.Mode.NORMAL, "", "bar", "bbb", "aaa", "10", "fff", null, "-Xmx1g", false, "subnet 456", null, null, false, null, "iamInstanceProfile", false, false, false, null, true, "", false, true); + SlaveTemplate standard = new SlaveTemplate("3", "eu-west-1b", null, "default", "foo", InstanceType.M1Large, false, label, Node.Mode.NORMAL, "", "bar", "bbb", "aaa", "10", "fff", null, "-Xmx1g", false, "subnet 456", null, null, false, null, "iamInstanceProfile", false, false, false, null, true, "", false, true); + SlaveTemplate middleSpot = new SlaveTemplate("4", middleZone, new SpotConfiguration("0.33"), "default", "foo", InstanceType.M1Large, false, label, Node.Mode.NORMAL, "", "bar", "bbb", "aaa", "10", "fff", null, "-Xmx1g", false, "subnet 456", null, null, false, null, "iamInstanceProfile", false, false, false, null, true, "", false, true); + + List templates = new ArrayList<>(); + // The expensive one is first in the list. This is important. + templates.add(expensiveSpot); + templates.add(cheapestSpot); + templates.add(standard); + templates.add(middleSpot); + + // Mock an Amazon EC2 Connection + AmazonEC2 ec2 = Mockito.mock(AmazonEC2.class); + Mockito.when(ec2.describeSpotPriceHistory(any(DescribeSpotPriceHistoryRequest.class))).thenAnswer( + invocation -> { + final String zone = invocation.getArgumentAt(0, DescribeSpotPriceHistoryRequest.class).getAvailabilityZone(); + + DescribeSpotPriceHistoryResult result = new DescribeSpotPriceHistoryResult(); + SpotPrice spotPrice = new SpotPrice(); + result.setSpotPriceHistory(Collections.singletonList(spotPrice)); + + spotPrice.setAvailabilityZone(zone); + spotPrice.setInstanceType(InstanceType.M1Large); + + if (zone.equals(expensiveZone)) { + spotPrice.setSpotPrice("34.3"); + } else if (zone.equals(middleZone)) { + spotPrice.setSpotPrice("1.12"); + } else if (zone.equals(cheapZone)) { + spotPrice.setSpotPrice("0.3324"); + } else { + // die + throw new IllegalArgumentException("Can't recognise zone"); + } + + return result; + }); + + // Spy on the EC2Cloud object to intercept the call to .connect() and return our mocked cloud + EC2Cloud rawCloud = new AmazonEC2Cloud("", false, "", "eu-west-1", "", "", templates); + EC2Cloud spiedCloud = Mockito.spy(rawCloud); + doReturn(ec2).when(spiedCloud).connect(); + + SlaveTemplate template = spiedCloud.getTemplate(Label.parse(label).iterator().next()); + + assertEquals(cheapestSpot.ami, template.ami); + } }