diff --git a/grails-app/conf/BootStrap.groovy b/grails-app/conf/BootStrap.groovy index 44e978bf..796bf5b7 100644 --- a/grails-app/conf/BootStrap.groovy +++ b/grails-app/conf/BootStrap.groovy @@ -162,6 +162,8 @@ class BootStrap { properties.setProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD, prop.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD)); if (prop.getProperty(IceOptions.URL_PREFIX) != null) properties.setProperty(IceOptions.URL_PREFIX, prop.getProperty(IceOptions.URL_PREFIX)); + if (prop.getProperty(IceOptions.IGNORE_CREDITS) != null) + properties.setProperty(IceOptions.IGNORE_CREDITS, prop.getProperty(IceOptions.IGNORE_CREDITS)); ReservationCapacityPoller reservationCapacityPoller = null; if ("true".equals(prop.getProperty("ice.reservationCapacityPoller"))) { diff --git a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java index f16a88dd..76187ca7 100644 --- a/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java +++ b/src/java/com/netflix/ice/basic/BasicLineItemProcessor.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -88,23 +89,60 @@ public long getEndMillis(String[] items) { } public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map usageDataByProduct, Map costDataByProduct, Map ondemandRate) { - if (StringUtils.isEmpty(items[accountIdIndex]) || - StringUtils.isEmpty(items[productIndex]) || - StringUtils.isEmpty(items[usageTypeIndex]) || - StringUtils.isEmpty(items[operationIndex]) || - StringUtils.isEmpty(items[usageQuantityIndex]) || - StringUtils.isEmpty(items[costIndex])) + + if (StringUtils.isEmpty(items[costIndex])) { + logger.info("Ignoring Record due to missing Cost - " + Arrays.toString(items)); return Result.ignore; + } + + double costValue = Double.parseDouble(items[costIndex]); + boolean credit = false; + + // make sure we don't ignore credits + if (costValue < 0) { + credit = true; + if (config.ignoreCredits || ! reformCredit(startMilli, items)) { + logger.info("Ignoring Credit - " + config.ignoreCredits); + return Result.ignore; + } + logger.info("Found Credit - " + Arrays.toString(items)); + } + + // fail-fast on records we can't process + if (StringUtils.isEmpty(items[accountIdIndex])) { + logger.info("Ignoring Record due to missing Account Id - " + Arrays.toString(items)); + return Result.ignore; + } + if (StringUtils.isEmpty(items[productIndex])) { + logger.info("Ignoring Record due to missing Product - " + Arrays.toString(items)); + return Result.ignore; + } + if (StringUtils.isEmpty(items[usageTypeIndex])) { + logger.info("Ignoring Record due to missing Usage Type - " + Arrays.toString(items)); + return Result.ignore; + } + if (StringUtils.isEmpty(items[operationIndex])) { + logger.info("Ignoring Record due to missing Operation - " + Arrays.toString(items)); + return Result.ignore; + } + if (StringUtils.isEmpty(items[usageQuantityIndex])) { + logger.info("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items)); + return Result.ignore; + } Account account = config.accountService.getAccountById(items[accountIdIndex]); - if (account == null) + if (account == null) { + logger.info("Ignoring Record due to missing Account - " + Arrays.toString(items)); return Result.ignore; + } double usageValue = Double.parseDouble(items[usageQuantityIndex]); - double costValue = Double.parseDouble(items[costIndex]); long millisStart; long millisEnd; + + Result result = Result.hourly; + try { millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]); millisEnd = amazonBillingDateFormat.parseMillis(items[endTimeIndex]); @@ -116,7 +154,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c Product product = config.productService.getProductByAwsName(items[productIndex]); boolean reservationUsage = "Y".equals(items[reservedIndex]); - ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue); + ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue, credit); product = reformedMetaData.product; Operation operation = reformedMetaData.operation; UsageType usageType = reformedMetaData.usageType; @@ -125,7 +163,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c int startIndex = (int)((millisStart - startMilli)/ AwsUtils.hourMillis); int endIndex = (int)((millisEnd + 1000 - startMilli)/ AwsUtils.hourMillis); - Result result = Result.hourly; if (product == Product.ec2_instance) { result = processEc2Instance(processDelayed, reservationUsage, operation, zone); } @@ -145,13 +182,15 @@ else if (product == Product.rds) { result = processRds(usageType); } - if (result == Result.ignore || result == Result.delay) + if (result == Result.ignore || result == Result.delay) { + logger.info("Record not processed - " + result + " - " + Arrays.toString(items)); return result; + } if (usageType.name.startsWith("TimedStorage-ByteHrs")) result = Result.daily; - boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : items[descriptionIndex].toLowerCase().contains("-month"); + boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") ); ReadWriteData usageData = usageDataByProduct.get(null); ReadWriteData costData = costDataByProduct.get(null); @@ -170,6 +209,10 @@ else if (result == Result.monthly) { int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24; usageValue = usageValue * endIndex / numHoursInMonth; costValue = costValue * endIndex / numHoursInMonth; + } else { + int maxEndIndex = usageData.getNum(); + if (endIndex > maxEndIndex) + endIndex = maxEndIndex; } if (monthlyCost) { @@ -213,7 +256,7 @@ else if (result == Result.monthly) { } catch (Exception e) { logger.error("failed to get RI price for " + tagGroup.region + " " + usageTypeForPrice); - resourceCostValue = -1; + resourceCostValue = Double.MIN_VALUE; } } @@ -234,7 +277,9 @@ else if (result == Result.monthly) { return result; for (int i : indexes) { - + if (credit) { + logger.debug("Handled Credit Index " + i + " - " + costValue); + } if (config.randomizer != null) { if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null) @@ -248,8 +293,8 @@ else if (result == Result.monthly) { Map usages = usageData.getData(i); Map costs = costData.getData(i); - addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3); - addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3); + addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true); + addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true); } else { resourceCostValue = usageValue * config.costPerMonitorMetricPerHour; @@ -259,9 +304,9 @@ else if (result == Result.monthly) { Map usagesOfResource = usageDataOfProduct.getData(i); Map costsOfResource = costDataOfProduct.getData(i); - if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3) { + if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true) { addValue(usagesOfResource, resourceTagGroup, usageValue, product != Product.monitor); - if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue < 0) { + if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue == Double.MIN_VALUE) { addValue(costsOfResource, resourceTagGroup, costValue, product != Product.monitor); } else { addValue(costsOfResource, resourceTagGroup, resourceCostValue, product != Product.monitor); @@ -345,7 +390,42 @@ private Result processRds(UsageType usageType) { return Result.hourly; } - protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost) { + protected boolean reformCredit(long startMilli, String[] items) { + + String[] split_description = items[descriptionIndex].split(":"); + + // If the credit doesn't have a start and end time then we... set to end to end of month + if (items[startTimeIndex].isEmpty()) { + int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24; + items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth-1).toString(amazonBillingDateFormat); + items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat); + logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ; + } + + // seperate credits into their own product for easy filtering/aggregation + items[productIndex]+=" credit"; + + // try to use the description to fetch some info about the credit if we have nothing + if (items[operationIndex].isEmpty()) { + if (split_description.length > 0) + items[operationIndex]=split_description[0]; + else + items[operationIndex]="credit"; + } + + if (items[usageTypeIndex].isEmpty()) { + items[usageTypeIndex]="credit"; + } + + if (items[usageQuantityIndex].isEmpty()) { + items[usageQuantityIndex] = "1"; + } + + return true; + + } + + protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) { Operation operation = null; UsageType usageType = null; @@ -432,6 +512,10 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu usageType = UsageType.getUsageType(usageTypeStr, operation, description); } + // This method resets product. Make sure we seperated out our credits + if (credit && ! product.name.endsWith("credit")) + product = new Product(product.name + " credit"); + return new ReformedMetaData(region, product, operation, usageType); } diff --git a/src/java/com/netflix/ice/common/IceOptions.java b/src/java/com/netflix/ice/common/IceOptions.java index c7d9bca9..45961920 100644 --- a/src/java/com/netflix/ice/common/IceOptions.java +++ b/src/java/com/netflix/ice/common/IceOptions.java @@ -100,6 +100,11 @@ public class IceOptions { */ public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize"; + /** + * Should we ignore credits or not? + */ + public static final String IGNORE_CREDITS = "ice.ignoreCredits"; + /** * Cost per monitor metric per hour, It's optional. */ diff --git a/src/java/com/netflix/ice/processor/ProcessorConfig.java b/src/java/com/netflix/ice/processor/ProcessorConfig.java index c009dc4e..1fda642b 100644 --- a/src/java/com/netflix/ice/processor/ProcessorConfig.java +++ b/src/java/com/netflix/ice/processor/ProcessorConfig.java @@ -41,6 +41,7 @@ public class ProcessorConfig extends Config { public final LineItemProcessor lineItemProcessor; public final Randomizer randomizer; public final double costPerMonitorMetricPerHour; + public final boolean ignoreCredits; public final String useCostForResourceGroup; @@ -87,6 +88,7 @@ public ProcessorConfig( customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(","); useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled"); + ignoreCredits = Boolean.parseBoolean(properties.getProperty(IceOptions.IGNORE_CREDITS, "true")); ProcessorConfig.instance = this; diff --git a/web-app/js/ice.js b/web-app/js/ice.js index f76e6011..9daaac01 100644 --- a/web-app/js/ice.js +++ b/web-app/js/ice.js @@ -159,7 +159,7 @@ ice.factory('highchart', function() { } var setupYAxis = function(isCost, showsps, factorsps) { - var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, min: 0, lineWidth: 2}; + var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, lineWidth: 2}; if (isCost) yAxis.labels = { formatter: function() { @@ -169,7 +169,7 @@ ice.factory('highchart', function() { hc_options.yAxis = [yAxis]; if (showsps) { - hc_options.yAxis.push({title:{text:metricname}, height: 100, min: 0, lineWidth: 2, offset: 0}); + hc_options.yAxis.push({title:{text:metricname}, height: 100, lineWidth: 2, offset: 0}); hc_options.yAxis[0].top = 150; hc_options.yAxis[0].height = 350; }