aws_billing.py

# coding=utf-8
from __future__ import division, absolute_import, print_function, unicode_literals
import csv
import boto3
from zipfile import ZipFile
import io
import tempfile
from decimal import Decimal
import datetime
from collections import defaultdict

service_alias = {
    'Amazon CloudFront': 'cloudfront',
    'Amazon Elastic Compute Cloud': 'ec2',
    'Amazon Simple Storage Service': 's3',
    'Amazon Simple Notification Service': 'sns',
    'Amazon Simple Queue Service': 'sqs'
}


def parse_date(date):
    if date == '':
        return None
    return datetime.datetime.strptime(date, '%Y-%m-%d %H:%M:%S')


def truncate_date(dt, period):
    if period == 'day':
        return dt.date()
    elif period == 'hour':
        return datetime.datetime(dt.year, dt.month, dt.day, dt.hour, 0, 0)


class DetailedBillingFile(object):

    @classmethod
    def fetch(cls, bucket, account, date):
        csvname = "%s-aws-billing-detailed-line-items-%d-%02d.csv" % (account, date.year, date.month)
        s3 = boto3.resource('s3')
        obj = s3.Object(bucket, "%s.zip" % csvname)

        with tempfile.TemporaryFile() as fd:
            obj.download_fileobj(fd)
            with ZipFile(fd, 'r') as zipfile:
                dbf = cls()
                dbf.parse_detailed_billing(io.TextIOWrapper(zipfile.open(csvname)))

        return dbf

    def parse_file(self, filename):
        with open(filename) as csvfile:
            self.parse_detailed_billing(csvfile)

    def parse_detailed_billing(self, csvfile):
        self.lines = []
        data = csv.reader(csvfile)
        header = next(data)
        for row in data:
            line = {}
            i = 0
            for col in header:
                line[col] = row[i]
                i += 1
            self.lines.append(line)

    def by_service(self, time_period='day', include_no_date=False):
        data = defaultdict(lambda: defaultdict(Decimal))
        for line in self.lines:
            start_date = parse_date(line['UsageStartDate'])

            if line['ReservedInstance'] == 'Y' and line['PricingPlanId'] == '':
                # This is probably a reserved instance charge for the remainder of the month,
                # which we can ignore for historical charges.
                continue

            if start_date is None and not include_no_date:
                # Undated charges are usually totals, corrections, or notes.
                continue

            if line['ProductName'] not in service_alias:
                # Unknown service
                continue

            name = service_alias[line['ProductName']]

            if line['ReservedInstance'] == 'Y':
                name += '.reserved'
            elif name == 'ec2':
                name += '.ondemand'

            data[truncate_date(start_date, time_period)][name] += Decimal(line['Cost'])
        return data

    def totals(self):
        costs = defaultdict(Decimal)
        for line in self.lines:
            if line['RecordType'] == 'LineItem':
                costs[line['ProductName']] += Decimal(line['Cost'])
        return costs

fetch_billing.py

# coding=utf-8
from __future__ import division, absolute_import, print_function, unicode_literals
from aws_billing import DetailedBillingFile
import argparse
import arrow

parser = argparse.ArgumentParser(description='Fetch AWS billing data and output Graphite format')
parser.add_argument('days', type=int, help="number of days back to fetch")
args = parser.parse_args()

prefix = 'ops.billing.aws.service'
since = arrow.utcnow().replace(days=-args.days).naive


for month in arrow.Arrow.range('month', since.replace(day=1, hour=0, minute=0, second=0), arrow.utcnow()):
    dbf = DetailedBillingFile.fetch('bucketname', 'account_id', month)
    for date, services in sorted(dbf.by_service(time_period='hour').items(), key=lambda d: d[0]):
        if date < since:
            continue
        for service, total in services.items():
            if total > 0:
                print("%s.%s %s %s" % (prefix, service, total, int(date.timestamp())))