Add Support for Credits/Refunds

grails-app/conf/BootStrap.groovy

@@ -162,6 +162,8 @@ class BootStrap {
                     properties.setProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD, prop.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD));
                 if (prop.getProperty(IceOptions.URL_PREFIX) != null)
                     properties.setProperty(IceOptions.URL_PREFIX, prop.getProperty(IceOptions.URL_PREFIX));
+                if (prop.getProperty(IceOptions.IGNORE_CREDITS) != null)
+                    properties.setProperty(IceOptions.IGNORE_CREDITS, prop.getProperty(IceOptions.IGNORE_CREDITS));
 
                 ReservationCapacityPoller reservationCapacityPoller = null;
                 if ("true".equals(prop.getProperty("ice.reservationCapacityPoller"))) {

src/java/com/netflix/ice/basic/BasicLineItemProcessor.java

@@ -28,6 +28,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -88,23 +89,60 @@ public long getEndMillis(String[] items) {
     }
 
     public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map<Product, ReadWriteData> usageDataByProduct, Map<Product, ReadWriteData> costDataByProduct, Map<String, Double> ondemandRate) {
-        if (StringUtils.isEmpty(items[accountIdIndex]) ||
-            StringUtils.isEmpty(items[productIndex]) ||
-            StringUtils.isEmpty(items[usageTypeIndex]) ||
-            StringUtils.isEmpty(items[operationIndex]) ||
-            StringUtils.isEmpty(items[usageQuantityIndex]) ||
-            StringUtils.isEmpty(items[costIndex]))
+
+        if (StringUtils.isEmpty(items[costIndex])) {
+            logger.info("Ignoring Record due to missing Cost - " + Arrays.toString(items));
             return Result.ignore;
+        }
+
+        double costValue = Double.parseDouble(items[costIndex]);
+        boolean credit = false;
+
+        // make sure we don't ignore credits
+        if (costValue < 0) {
+            credit = true;
+            if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
+                logger.info("Ignoring Credit - " + config.ignoreCredits);
+                return Result.ignore;
+            }
+            logger.info("Found Credit - " + Arrays.toString(items));
+        }
+
+        // fail-fast on records we can't process
+        if (StringUtils.isEmpty(items[accountIdIndex])) {
+            logger.info("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
+            return Result.ignore;
+        } 
+        if (StringUtils.isEmpty(items[productIndex])) {
+            logger.info("Ignoring Record due to missing Product - " + Arrays.toString(items));
+            return Result.ignore;
+        }
+        if (StringUtils.isEmpty(items[usageTypeIndex])) {
+            logger.info("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
+            return Result.ignore;
+        }
+        if (StringUtils.isEmpty(items[operationIndex])) {
+            logger.info("Ignoring Record due to missing Operation - " + Arrays.toString(items));
+            return Result.ignore;
+        }
+        if (StringUtils.isEmpty(items[usageQuantityIndex])) {
+            logger.info("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
+            return Result.ignore;
+        }
 
         Account account = config.accountService.getAccountById(items[accountIdIndex]);
-        if (account == null)
+        if (account == null) {
+            logger.info("Ignoring Record due to missing Account - " + Arrays.toString(items));
             return Result.ignore;
+        }
 
         double usageValue = Double.parseDouble(items[usageQuantityIndex]);
-        double costValue = Double.parseDouble(items[costIndex]);
 
         long millisStart;
         long millisEnd;
+
+        Result result = Result.hourly;
+
         try {
             millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]);
             millisEnd = amazonBillingDateFormat.parseMillis(items[endTimeIndex]);
@@ -116,7 +154,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
 
         Product product = config.productService.getProductByAwsName(items[productIndex]);
         boolean reservationUsage = "Y".equals(items[reservedIndex]);
-        ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue);
+        ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue, credit);
         product = reformedMetaData.product;
         Operation operation = reformedMetaData.operation;
         UsageType usageType = reformedMetaData.usageType;
@@ -125,7 +163,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
         int startIndex = (int)((millisStart - startMilli)/ AwsUtils.hourMillis);
         int endIndex = (int)((millisEnd + 1000 - startMilli)/ AwsUtils.hourMillis);
 
-        Result result = Result.hourly;
         if (product == Product.ec2_instance) {
             result = processEc2Instance(processDelayed, reservationUsage, operation, zone);
         }
@@ -145,13 +182,15 @@ else if (product == Product.rds) {
             result = processRds(usageType);
         }
 
-        if (result == Result.ignore || result == Result.delay)
+        if (result == Result.ignore || result == Result.delay) {
+            logger.info("Record not processed  - " + result + " - " + Arrays.toString(items));
             return result;
+        }
 
         if (usageType.name.startsWith("TimedStorage-ByteHrs"))
             result = Result.daily;
 
-        boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : items[descriptionIndex].toLowerCase().contains("-month");
+        boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") );
 
         ReadWriteData usageData = usageDataByProduct.get(null);
         ReadWriteData costData = costDataByProduct.get(null);
@@ -170,6 +209,10 @@ else if (result == Result.monthly) {
             int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
             usageValue = usageValue * endIndex / numHoursInMonth;
             costValue = costValue * endIndex / numHoursInMonth;
+        } else {
+            int maxEndIndex = usageData.getNum();
+            if (endIndex > maxEndIndex)
+                endIndex = maxEndIndex;
         }
 
         if (monthlyCost) {
@@ -213,7 +256,7 @@ else if (result == Result.monthly) {
                 }
                 catch (Exception e) {
                     logger.error("failed to get RI price for " + tagGroup.region + " " + usageTypeForPrice);
-                    resourceCostValue = -1;
+                    resourceCostValue = Double.MIN_VALUE;
                 }
             }
 
@@ -234,7 +277,9 @@ else if (result == Result.monthly) {
             return result;
 
         for (int i : indexes) {
-
+            if (credit) {
+                logger.debug("Handled Credit Index " + i + " - " + costValue);
+            }
             if (config.randomizer != null) {
 
                 if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null)
@@ -248,8 +293,8 @@ else if (result == Result.monthly) {
                 Map<TagGroup, Double> usages = usageData.getData(i);
                 Map<TagGroup, Double> costs = costData.getData(i);
 
-                addValue(usages, tagGroup, usageValue,  config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
-                addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
+                addValue(usages, tagGroup, usageValue,  config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
+                addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
             }
             else {
                 resourceCostValue = usageValue * config.costPerMonitorMetricPerHour;
@@ -259,9 +304,9 @@ else if (result == Result.monthly) {
                 Map<TagGroup, Double> usagesOfResource = usageDataOfProduct.getData(i);
                 Map<TagGroup, Double> costsOfResource = costDataOfProduct.getData(i);
 
-                if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3) {
+                if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true) {
                     addValue(usagesOfResource, resourceTagGroup, usageValue, product != Product.monitor);
-                    if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue < 0) {
+                    if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue == Double.MIN_VALUE) {
                         addValue(costsOfResource, resourceTagGroup, costValue, product != Product.monitor);
                     } else {
                         addValue(costsOfResource, resourceTagGroup, resourceCostValue, product != Product.monitor);
@@ -345,7 +390,42 @@ private Result processRds(UsageType usageType) {
             return Result.hourly;
     }
 
-    protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost) {
+    protected boolean reformCredit(long startMilli, String[] items) {
+
+        String[] split_description = items[descriptionIndex].split(":");
+
+        // If the credit doesn't have a start and end time then we... set to end to end of month
+        if (items[startTimeIndex].isEmpty()) {
+            int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
+            items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth-1).toString(amazonBillingDateFormat);
+            items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
+            logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ;
+        }
+
+        // seperate credits into their own product for easy filtering/aggregation
+        items[productIndex]+=" credit";
+
+        // try to use the description to fetch some info about the credit if we have nothing
+        if (items[operationIndex].isEmpty()) {
+            if (split_description.length > 0)
+                items[operationIndex]=split_description[0];
+            else
+                items[operationIndex]="credit";
+        }
+
+        if (items[usageTypeIndex].isEmpty()) {
+            items[usageTypeIndex]="credit";
+        }
+
+        if (items[usageQuantityIndex].isEmpty()) {
+            items[usageQuantityIndex] = "1";
+        }
+
+        return true;
+
+    }
+
+    protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) {
 
         Operation operation = null;
         UsageType usageType = null;
@@ -432,6 +512,10 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu
             usageType = UsageType.getUsageType(usageTypeStr, operation, description);
         }
 
+        // This method resets product.  Make sure we seperated out our credits
+        if (credit && ! product.name.endsWith("credit"))
+            product = new Product(product.name + " credit");
+
         return new ReformedMetaData(region, product, operation, usageType);
     }
 

src/java/com/netflix/ice/common/IceOptions.java

@@ -100,6 +100,11 @@ public class IceOptions {
      */
     public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize";
 
+    /**
+     * Should we ignore credits or not?
+     */
+    public static final String IGNORE_CREDITS = "ice.ignoreCredits";
+
     /**
      * Cost per monitor metric per hour, It's optional.
      */

src/java/com/netflix/ice/processor/ProcessorConfig.java

@@ -41,6 +41,7 @@ public class ProcessorConfig extends Config {
     public final LineItemProcessor lineItemProcessor;
     public final Randomizer randomizer;
     public final double costPerMonitorMetricPerHour;
+    public final boolean ignoreCredits; 
 
     public final String useCostForResourceGroup;
 
@@ -87,6 +88,7 @@ public ProcessorConfig(
         customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(",");
 
         useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled");
+        ignoreCredits = Boolean.parseBoolean(properties.getProperty(IceOptions.IGNORE_CREDITS, "true"));
 
         ProcessorConfig.instance = this;
 

web-app/js/ice.js

@@ -159,7 +159,7 @@ ice.factory('highchart', function() {
   }
 
   var setupYAxis = function(isCost, showsps, factorsps) {
-    var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, min: 0, lineWidth: 2};
+    var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, lineWidth: 2};
     if (isCost)
       yAxis.labels = {
         formatter: function() {
@@ -169,7 +169,7 @@ ice.factory('highchart', function() {
     hc_options.yAxis = [yAxis];
 
     if (showsps) {
-      hc_options.yAxis.push({title:{text:metricname}, height: 100, min: 0, lineWidth: 2, offset: 0});
+      hc_options.yAxis.push({title:{text:metricname}, height: 100, lineWidth: 2, offset: 0});
       hc_options.yAxis[0].top = 150;
       hc_options.yAxis[0].height = 350;
     }