Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Support for Credits/Refunds #158

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions grails-app/conf/BootStrap.groovy
Original file line number Diff line number Diff line change
@@ -162,6 +162,8 @@ class BootStrap {
properties.setProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD, prop.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD));
if (prop.getProperty(IceOptions.URL_PREFIX) != null)
properties.setProperty(IceOptions.URL_PREFIX, prop.getProperty(IceOptions.URL_PREFIX));
if (prop.getProperty(IceOptions.IGNORE_CREDITS) != null)
properties.setProperty(IceOptions.IGNORE_CREDITS, prop.getProperty(IceOptions.IGNORE_CREDITS));

ReservationCapacityPoller reservationCapacityPoller = null;
if ("true".equals(prop.getProperty("ice.reservationCapacityPoller"))) {
122 changes: 103 additions & 19 deletions src/java/com/netflix/ice/basic/BasicLineItemProcessor.java
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.List;
import java.util.Map;

@@ -88,23 +89,60 @@ public long getEndMillis(String[] items) {
}

public Result process(long startMilli, boolean processDelayed, ProcessorConfig config, String[] items, Map<Product, ReadWriteData> usageDataByProduct, Map<Product, ReadWriteData> costDataByProduct, Map<String, Double> ondemandRate) {
if (StringUtils.isEmpty(items[accountIdIndex]) ||
StringUtils.isEmpty(items[productIndex]) ||
StringUtils.isEmpty(items[usageTypeIndex]) ||
StringUtils.isEmpty(items[operationIndex]) ||
StringUtils.isEmpty(items[usageQuantityIndex]) ||
StringUtils.isEmpty(items[costIndex]))

if (StringUtils.isEmpty(items[costIndex])) {
logger.info("Ignoring Record due to missing Cost - " + Arrays.toString(items));
return Result.ignore;
}

double costValue = Double.parseDouble(items[costIndex]);
boolean credit = false;

// make sure we don't ignore credits
if (costValue < 0) {
credit = true;
if (config.ignoreCredits || ! reformCredit(startMilli, items)) {
logger.info("Ignoring Credit - " + config.ignoreCredits);
return Result.ignore;
}
logger.info("Found Credit - " + Arrays.toString(items));
}

// fail-fast on records we can't process
if (StringUtils.isEmpty(items[accountIdIndex])) {
logger.info("Ignoring Record due to missing Account Id - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[productIndex])) {
logger.info("Ignoring Record due to missing Product - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageTypeIndex])) {
logger.info("Ignoring Record due to missing Usage Type - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[operationIndex])) {
logger.info("Ignoring Record due to missing Operation - " + Arrays.toString(items));
return Result.ignore;
}
if (StringUtils.isEmpty(items[usageQuantityIndex])) {
logger.info("Ignoring Record due to missing Usage Quantity - " + Arrays.toString(items));
return Result.ignore;
}

Account account = config.accountService.getAccountById(items[accountIdIndex]);
if (account == null)
if (account == null) {
logger.info("Ignoring Record due to missing Account - " + Arrays.toString(items));
return Result.ignore;
}

double usageValue = Double.parseDouble(items[usageQuantityIndex]);
double costValue = Double.parseDouble(items[costIndex]);

long millisStart;
long millisEnd;

Result result = Result.hourly;

try {
millisStart = amazonBillingDateFormat.parseMillis(items[startTimeIndex]);
millisEnd = amazonBillingDateFormat.parseMillis(items[endTimeIndex]);
@@ -116,7 +154,7 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c

Product product = config.productService.getProductByAwsName(items[productIndex]);
boolean reservationUsage = "Y".equals(items[reservedIndex]);
ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue);
ReformedMetaData reformedMetaData = reform(millisStart, config, product, reservationUsage, items[operationIndex], items[usageTypeIndex], items[descriptionIndex], costValue, credit);
product = reformedMetaData.product;
Operation operation = reformedMetaData.operation;
UsageType usageType = reformedMetaData.usageType;
@@ -125,7 +163,6 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
int startIndex = (int)((millisStart - startMilli)/ AwsUtils.hourMillis);
int endIndex = (int)((millisEnd + 1000 - startMilli)/ AwsUtils.hourMillis);

Result result = Result.hourly;
if (product == Product.ec2_instance) {
result = processEc2Instance(processDelayed, reservationUsage, operation, zone);
}
@@ -145,13 +182,15 @@ else if (product == Product.rds) {
result = processRds(usageType);
}

if (result == Result.ignore || result == Result.delay)
if (result == Result.ignore || result == Result.delay) {
logger.info("Record not processed - " + result + " - " + Arrays.toString(items));
return result;
}

if (usageType.name.startsWith("TimedStorage-ByteHrs"))
result = Result.daily;

boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : items[descriptionIndex].toLowerCase().contains("-month");
boolean monthlyCost = StringUtils.isEmpty(items[descriptionIndex]) ? false : ( items[descriptionIndex].toLowerCase().contains("-month") );

ReadWriteData usageData = usageDataByProduct.get(null);
ReadWriteData costData = costDataByProduct.get(null);
@@ -170,6 +209,10 @@ else if (result == Result.monthly) {
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
usageValue = usageValue * endIndex / numHoursInMonth;
costValue = costValue * endIndex / numHoursInMonth;
} else {
int maxEndIndex = usageData.getNum();
if (endIndex > maxEndIndex)
endIndex = maxEndIndex;
}

if (monthlyCost) {
@@ -213,7 +256,7 @@ else if (result == Result.monthly) {
}
catch (Exception e) {
logger.error("failed to get RI price for " + tagGroup.region + " " + usageTypeForPrice);
resourceCostValue = -1;
resourceCostValue = Double.MIN_VALUE;
}
}

@@ -234,7 +277,9 @@ else if (result == Result.monthly) {
return result;

for (int i : indexes) {

if (credit) {
logger.debug("Handled Credit Index " + i + " - " + costValue);
}
if (config.randomizer != null) {

if (tagGroup.product != Product.rds && tagGroup.product != Product.s3 && usageData.getData(i).get(tagGroup) != null)
@@ -248,8 +293,8 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usages = usageData.getData(i);
Map<TagGroup, Double> costs = costData.getData(i);

addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3);
addValue(usages, tagGroup, usageValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
addValue(costs, tagGroup, costValue, config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true);
}
else {
resourceCostValue = usageValue * config.costPerMonitorMetricPerHour;
@@ -259,9 +304,9 @@ else if (result == Result.monthly) {
Map<TagGroup, Double> usagesOfResource = usageDataOfProduct.getData(i);
Map<TagGroup, Double> costsOfResource = costDataOfProduct.getData(i);

if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3) {
if (config.randomizer == null || tagGroup.product == Product.rds || tagGroup.product == Product.s3 || credit == true) {
addValue(usagesOfResource, resourceTagGroup, usageValue, product != Product.monitor);
if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue < 0) {
if (!config.useCostForResourceGroup.equals("modeled") || resourceCostValue == Double.MIN_VALUE) {
addValue(costsOfResource, resourceTagGroup, costValue, product != Product.monitor);
} else {
addValue(costsOfResource, resourceTagGroup, resourceCostValue, product != Product.monitor);
@@ -345,7 +390,42 @@ private Result processRds(UsageType usageType) {
return Result.hourly;
}

protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost) {
protected boolean reformCredit(long startMilli, String[] items) {

String[] split_description = items[descriptionIndex].split(":");

// If the credit doesn't have a start and end time then we... set to end to end of month
if (items[startTimeIndex].isEmpty()) {
int numHoursInMonth = new DateTime(startMilli, DateTimeZone.UTC).dayOfMonth().getMaximumValue() * 24;
items[startTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth-1).toString(amazonBillingDateFormat);
items[endTimeIndex]=new DateTime(startMilli, DateTimeZone.UTC).plusHours(numHoursInMonth).toString(amazonBillingDateFormat);
logger.debug("Credit did not have a Time - Set to " + items[startTimeIndex] + "-" + items[endTimeIndex]) ;
}

// seperate credits into their own product for easy filtering/aggregation
items[productIndex]+=" credit";

// try to use the description to fetch some info about the credit if we have nothing
if (items[operationIndex].isEmpty()) {
if (split_description.length > 0)
items[operationIndex]=split_description[0];
else
items[operationIndex]="credit";
}

if (items[usageTypeIndex].isEmpty()) {
items[usageTypeIndex]="credit";
}

if (items[usageQuantityIndex].isEmpty()) {
items[usageQuantityIndex] = "1";
}

return true;

}

protected ReformedMetaData reform(long millisStart, ProcessorConfig config, Product product, boolean reservationUsage, String operationStr, String usageTypeStr, String description, double cost, boolean credit) {

Operation operation = null;
UsageType usageType = null;
@@ -432,6 +512,10 @@ else if (usageTypeStr.startsWith("HeavyUsage") || usageTypeStr.startsWith("Mediu
usageType = UsageType.getUsageType(usageTypeStr, operation, description);
}

// This method resets product. Make sure we seperated out our credits
if (credit && ! product.name.endsWith("credit"))
product = new Product(product.name + " credit");

return new ReformedMetaData(region, product, operation, usageType);
}

5 changes: 5 additions & 0 deletions src/java/com/netflix/ice/common/IceOptions.java
Original file line number Diff line number Diff line change
@@ -100,6 +100,11 @@ public class IceOptions {
*/
public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize";

/**
* Should we ignore credits or not?
*/
public static final String IGNORE_CREDITS = "ice.ignoreCredits";

/**
* Cost per monitor metric per hour, It's optional.
*/
2 changes: 2 additions & 0 deletions src/java/com/netflix/ice/processor/ProcessorConfig.java
Original file line number Diff line number Diff line change
@@ -41,6 +41,7 @@ public class ProcessorConfig extends Config {
public final LineItemProcessor lineItemProcessor;
public final Randomizer randomizer;
public final double costPerMonitorMetricPerHour;
public final boolean ignoreCredits;

public final String useCostForResourceGroup;

@@ -87,6 +88,7 @@ public ProcessorConfig(
customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(",");

useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled");
ignoreCredits = Boolean.parseBoolean(properties.getProperty(IceOptions.IGNORE_CREDITS, "true"));

ProcessorConfig.instance = this;

4 changes: 2 additions & 2 deletions web-app/js/ice.js
Original file line number Diff line number Diff line change
@@ -159,7 +159,7 @@ ice.factory('highchart', function() {
}

var setupYAxis = function(isCost, showsps, factorsps) {
var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, min: 0, lineWidth: 2};
var yAxis = {title:{text: (isCost ? 'Cost' : 'Usage') + " per " + (factorsps ? metricunitname : consolidate)}, lineWidth: 2};
if (isCost)
yAxis.labels = {
formatter: function() {
@@ -169,7 +169,7 @@ ice.factory('highchart', function() {
hc_options.yAxis = [yAxis];

if (showsps) {
hc_options.yAxis.push({title:{text:metricname}, height: 100, min: 0, lineWidth: 2, offset: 0});
hc_options.yAxis.push({title:{text:metricname}, height: 100, lineWidth: 2, offset: 0});
hc_options.yAxis[0].top = 150;
hc_options.yAxis[0].height = 350;
}