dog2

#!/usr/bin/env python

import sys
#import matplotlib
#matplotlib.use('Agg')
#import matplotlib.pyplot as plt
#from matplotlib.backends.backend_pdf import PdfPages
import numpy as np
import csv
import copy
import plotly.offline
from plotly.graph_objs import *
from plotly import tools
from multirank.dynkw import autoDynKWRupt
import random

xkcd_colors = {}
with open('rgb.txt', 'r') as colors:
  for line in colors:
    #line_ = line.decode('ascii')
    if 'License' in line:
        continue
    name, color = line.split('#')
    name = name.strip()
    color = color.strip()
    xkcd_colors['xkcd:'+name] = '#'+color

colors = ['xkcd:red', 'xkcd:brown', 'xkcd:blue', 'xkcd:electric purple', 'xkcd:orange', 
          'xkcd:light blue', 'xkcd:teal', 'xkcd:magenta', 'xkcd:grey',
          'xkcd:turquoise', 'xkcd:cyan', 'xkcd:maroon',
          'xkcd:olive', 'xkcd:salmon', 'xkcd:beige', 'xkcd:navy blue', 'xkcd:black', 
          'xkcd:hot pink', 'xkcd:burnt yellow', 'xkcd:blue grey', 'xkcd:battleship grey', 'xkcd:pink', 'xkcd:yellow', 'xkcd:light purple']
colors_rgb = [xkcd_colors[color] for color in colors]

formats = ['lp', 'nl', 'bar', 'gms']

if len(sys.argv) == 1:
    print("dog <id>")
    sys.exit(0)

if int(sys.argv[1]) < 10:
    input = 'benchmarks/%s.csv' % sys.argv[2]
elif int(sys.argv[1]) < 20:
    input = 'benchmarks/%s_branches_stats.csv' % sys.argv[2]
else:
    input = 'benchmarks/%s_branches_stats_history.csv' % sys.argv[2]

#
# Read input file
#
if int(sys.argv[1]) < 20:
    data = []
    with open(input, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            data.append( row[1:] )
else:
    data = []
    with open(input, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            data.append( row )


if int(sys.argv[1]) < 10:
    legend = {}
    legend['simple_1'] = 'sum()'
    legend['simple_2'] = 'summation()'
    legend['simple_3'] = 'loop (e += t)'
    legend['simple_4'] = 'loop (e = e + t)'
    legend['simple_5'] = 'loop (e = t + e)'
    legend['simple_6'] = 'Sum()'
    #
    # Process data
    #
    p = {}
    labels = {}
    xticks = {}
    for row in data:
        if row[0] not in labels:
            labels[row[0]] = []
        if not row[1] in labels[row[0]]:
            if row[1] == 'master':
                append = True
                for i in range(len(labels[row[0]])):
                    if labels[row[0]][i][0].isalpha():
                        labels[row[0]].insert(i, row[1])
                        append = False
                        break
                if append:
                    labels[row[0]].append(row[1])
            else:
                labels[row[0]].append(row[1])
        if row[4] == "":
            continue

        if row[0] not in p:
            p[row[0]] = {}
        if row[2] not in p[row[0]]:
            p[row[0]][row[2]] = {}
        p[row[0]][row[2]][row[1]] = float(row[4])

elif int(sys.argv[1]) < 20:
    #
    # Process data
    #
    p = {}
    labels = {}
    xticks = {}
    for row in data:
        try:
            python, branch, problem, fileformat, part, minval, meanval, maxval, stddev  = row
        except:
            print("ERROR")
            print(row)
            raise
        if python not in labels:
            labels[python] = []
        if not branch in labels[python]:
            if branch == 'master':
                append = True
                for i in range(len(labels[python])):
                    if labels[python][i][0].isalpha():
                        labels[python].insert(i, branch)
                        append = False
                        break
                if append:
                    labels[python].append(branch)
            else:
                labels[python].append(branch)
        if meanval == "":
            continue

        if python not in p:
            p[python] = {}
        if problem not in p[python]:
            p[python][problem] = {}
        if branch not in p[python][problem]:
            p[python][problem][branch] = {}
        p[python][problem][branch][fileformat, part] = float(meanval)

elif int(sys.argv[1]) < 30:
    #
    # Process data
    #
    dates = []
    #python = set()
    #p = {}
    #labels = {}
    #xticks = {}

    info = {}

    for row in data:
        date, python, branch, problem, fileformat, part, minval, meanval, maxval, stdval = row

        if meanval == "":
            continue
        if part != 'total':
            continue
        if python.endswith('-cython'):
            python = python[:-7]
            branch = branch + '-cython'

        if len(dates) == 0 or dates[-1] != date:
            dates.append( date )
            
        if not python in info:
            info[python] = {}
        if not problem in info[python]:
            info[python][problem] = {}
        if not fileformat in info[python][problem]:
            info[python][problem][fileformat] = {}
        if not branch in info[python][problem][fileformat]:
            info[python][problem][fileformat][branch] = {}
        info[python][problem][fileformat][branch][date] = float(meanval)
    

if sys.argv[1] == "1" or sys.argv[1] == "2":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    skip = []
    if sys.argv[1] == "2":
        skip.append('simple_4')
        skip.append('simple_5')

    line_colors = {'simple_1':'r', 'simple_2':'g', 'simple_3':'b', 'simple_4':'c', 'simple_5':'m', 'simple_6':'y'}

    #
    # Plot 
    #
    for python_ in p:
        if python_.endswith('cython'):
            continue

        pversions = [python_]
        if python_+'-cython' in p:
            pversions.append( python_+'-cython' )

        for python in pversions:
          for line in p[python]:
            if line.startswith('simple'):
                if line in skip:
                    continue
                #if python.endswith('cython') and not i == 6:
                #    continue
                x_ = [-1]*len(labels[python_])
                y_ = [-1]*len(labels[python_])
                #print(line)
                #print(p[python][line])
                for x in p[python][line]:
                    i = labels[python_].index(x)
                    x_[i] = i
                    y_[i] = p[python][line][x]
                #print(x_)
                #print(y_)
                x__ = []
                y__ = []
                for i in range(len(x_)):
                    if x_[i] != -1:
                        x__.append(x_[i])
                        y__.append(y_[i])
                if len(x__) == 1:
                    if python.endswith('cython'):
                        plt.plot(x__, y__, 'o', markersize=3, label="cy: "+legend[line], c=line_colors[line])
                    else:
                        plt.plot(x__, y__, '+', markersize=7, label=legend[line], c=line_colors[line])
                else:
                    plt.plot(x__, y__, label=legend[line], c=line_colors[line])

        plt.xlabel("X Label")
        plt.ylabel("Y Label")
        plt.title("Simple Expressions: %s" % (python_))
        #plt.legend()
        #plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
        plt.legend(loc=2, borderaxespad=0.)

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), xticks[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "3":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    for python in p:
        for line in p[python]:
            if not line.endswith('_1'):
                continue
            name = line[:-2]
            x_ = [-1]*len(labels[python_])
            y_ = [-1]*len(labels[python_])

            for i in range(len(labels[python_])):
                if labels[python_][i] in p[python]["%s_4" % name]:
                    x_[i] = i
                    y_[i] = p[python]["%s_4" % name][labels[python_][i]] / p[python]["%s_1" % name][labels[python_][i]]

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', markersize=3, label=line[:-2])
            else:
                plt.plot(x__, y__, label=line[:-2])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("(Time e=e+x)/(Time sum(x))")
        plt.title("Slowdown of Simple Loops: %s" % (python))
        plt.legend()

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), labels[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "4":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    line_colors = {'simple':colors[0], 'const':colors[1], 'param':colors[2], 'mutable':colors[3], 'nonl':colors[4], 'bilinear':colors[5], 'nested':colors[6], 'weighted':colors[7]}

    for python_ in p:
        if python_.endswith('cython'):
            continue

        pversions = [python_]
        if python_+'-cython' in p:
            pversions.append( python_+'-cython' )

        for python in pversions:
          for line in p[python]:
            if not line.endswith('_1'):
                continue
            name = line[:-2]
            x_ = [-1]*len(labels[python_])
            y_ = [-1]*len(labels[python_])

            for i in range(len(labels[python_])):
                if labels[python_][i] in p[python][line]:
                    x_[i] = i
                    y_[i] = p[python][line][labels[python_][i]]

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 1:
                if python.endswith('cython'):
                    plt.plot(x__, y__, 'o', markersize=3, label="cy: "+line, c=line_colors[name])
                else:
                    plt.plot(x__, y__, '+', markersize=7, label=line, c=line_colors[name])
            else:
                plt.plot(x__, y__, label=line, c=line_colors[name])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Performance of sum(x): %s" % (python))
        plt.legend(loc=2, borderaxespad=0.)

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), xticks[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


#
# convert
#

elif sys.argv[1] == "10":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue
        for line in p[python]:
            if '_quick' in line:
                continue
            #print(python, line, labels[python][0])
            #print(list(p[python][line].keys()))
            x_ = [-1]*len(labels[python])
            y_ = [-1]*len(labels[python])

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    x_[i] = i
                    y_[i] = val

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 0:
                continue
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', label=line, markersize=3, c=colors[lines.index(line)])
            else:
                plt.plot(x__, y__, label=line, c=colors[lines.index(line)])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Time to Create a %s File: %s" % (format_, python))
        plt.legend()

        plt.xticks(range(len(labels[python])), xticks[python])
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "11":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    for format_ in formats:
      for python in p:
        for line in p[python]:
            if not (format_, 'total') in p[python][line][labels[python][0]]:
                continue
            x_ = [-1]*len(labels[python])
            y_ = [-1]*len(labels[python])

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    x_[i] = i
                    try:
                        y_[i] = p[python][line][labels[python][i]][format_, 'total']
                    except KeyError:
                        print(python, line, labels[python][i])
                        print(list(p[python][line][labels[python][i]].keys()))
                        #sys.exit(0)
                        raise

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            for i in range(len(y__)):
                y__[i] = y__[i]/max(y__[0],1e-7)
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', markersize=3, c=colors[lines.index(line)])
            else:
                plt.plot(x__, y__, label=line, c=colors[lines.index(line)])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Time to Create a %s File: %s" % (format_, python))
        plt.legend()

        plt.xticks(range(len(labels[python])), xticks[python])
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - comparison between master and other branches
#
elif sys.argv[1] == "12":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare runtime to generate problem files")
    ax.text(0,0.65,"    Master vs Branches")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        for label in labels[python]:
            tmp[label] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())

        for line in p[python]:
            if '_quick' in line:
                continue

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    tmp[labels[python][i]][ticks.index(line)] = val

        #print(tmp)
        for key in tmp:
            i = labels[python].index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Runtime")
        ax.set_title("Time to Create a %s File: %s" % (format_, python))
        ax.legend()
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')

        plt.subplots_adjust(bottom=0.5)
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - relative comparison between master and other branches
#
elif sys.argv[1] == "13":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare normalized runtime to generate problem files")
    ax.text(0,0.65,"    Master vs Branches")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        for label in labels[python]:
            tmp[label] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    tmp[labels[python][i]][ticks.index(line)] = val
        #print(tmp)
        #
        # Normalize data
        #
        if 'master' not in tmp:
            continue
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val
        #print(tmp)
        #
        # Create barchart
        #
        for key in tmp:
            i = labels[python].index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Normalized Runtime")
        ax.set_title("Normalized Time to Create a %s File: %s" % (format_, python))
        ax.legend()
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')

        plt.subplots_adjust(bottom=0.5)
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - relative comparison between master and other branches
#
elif sys.argv[1] == "14":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare normalized runtime to generate problem files")
    ax.text(0,0.65,"    Master vs ExprDev branch vs Problems using quicksum (w/ and w/o cython)")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']
    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)
        tmp['default'] = [-0.1] * len(ticks)
        tmp['cython'] = [-0.1] * len(ticks)
        tmp['quick'] = [-0.1] * len(ticks)
        tmp['quick+cython'] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][ticks.index(line)] = val
            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][ticks.index(line)] = val
            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][ticks.index(line)] = val
            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][ticks.index(line)] = val
                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][ticks.index(line)] = val
        ##print(tmp)
        #
        # Normalize data
        #
        ##print()
        ##print(len(tmp['master']))
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val
        #print(tmp)
        #
        # Create barchart
        #
        for key in tmp:
            i = labels_.index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Normalized Runtime")
        ax.set_title("Normalized Time to Create a %s File: %s" % (format_, python))
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')
        ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))
        #chartBox = ax.get_position()
        #ax.set_position([chartBox.x0, chartBox.y0, chartBox.width*0.4, chartBox.height])

        plt.subplots_adjust(bottom=0.5, right=0.7)
        pp.savefig()
        plt.clf()

    pp.close()

#
# RUNTIME DATA
#
elif sys.argv[1] == "15":
    print("Creating dog%s.csv" % sys.argv[1])

    csvdata = []

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']

    csvdata.append( ['PERFORMANCE TO CONSTRUCT PROBLEM FILES: '+sys.argv[2]] + [None]*12 )
    csvdata.append( [None, None, None] + ['NORMALIZED DATA', None, None, None, None] + ['RUNTIME DATA', None, None, None, None] )
    csvdata.append( ['Python', 'Format', 'Problem'] + labels_ + labels_ )

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)
        tmp['default'] = [-0.1] * len(ticks)
        tmp['cython'] = [-0.1] * len(ticks)
        tmp['quick'] = [-0.1] * len(ticks)
        tmp['quick+cython'] = [-0.1] * len(ticks)
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][ticks.index(line)] = val
            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][ticks.index(line)] = val
            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][ticks.index(line)] = val
            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][ticks.index(line)] = val
                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][ticks.index(line)] = val

        raw = tmp
        tmp = copy.deepcopy(tmp)
        #
        # Normalize data
        #
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val

        for i in range(len(ticks)):
            raw_data = [raw[key][i] for key in labels_]
            norm_data = [tmp[key][i] for key in labels_]
            csvdata.append( [python, format_, ticks[i]] + [None if val < 0.0 else val for val in norm_data] + [None if val < 0.0 else val for val in raw_data] )

    import csv
    with open("dog%s.csv" % sys.argv[1], 'w') as f:
        writer = csv.writer(f)
        for row in csvdata:
            writer.writerow(row)


elif sys.argv[1] == "16":
    print("Creating dog%s.csv" % sys.argv[1])

    csvdata = []

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']

    csvdata.append( ['PERFORMANCE SUMMARY ALL DATA: '+sys.argv[2]] + [None]*12 )
    csvdata.append( [None, None, None] + ['RUNTIME DATA', None, None, None, None] )
    csvdata.append( ['Python', 'Format', 'Problem', labels_[0], None, None, labels_[1], None, None, labels_[2], None, None, labels_[3], None, None, labels_[4], None, None] )
    csvdata.append( [None, None, None] + ['Construct', 'Write', 'Total']*5 )

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)*3
        tmp['default'] = [-0.1] * len(ticks)*3
        tmp['cython'] = [-0.1] * len(ticks)*3
        tmp['quick'] = [-0.1] * len(ticks)*3
        tmp['quick+cython'] = [-0.1] * len(ticks)*3
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)] = val
                val = p[python][line]['master'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)+1] = val
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)+2] = val

            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)] = val
                val = p[python][line]['expr_dev'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)+1] = val
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)+2] = val

            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)] = val
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)+1] = val
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)+2] = val

            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'construct'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)] = val
                    val = p[python][line_]['expr_dev'].get((format_, 'write_problem'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)+1] = val
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)+2] = val

                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'construct'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)] = val
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'write_problem'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)+1] = val
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)+2] = val

        raw = tmp

        for i in range(len(ticks)):
            raw_data = []
            for key in labels_:
                raw_data.append( raw[key][3*i] )
                raw_data.append( raw[key][3*i+1] )
                raw_data.append( raw[key][3*i+2] )
            csvdata.append( [python, format_, ticks[i]] + [None if val < 0.0 else val for val in raw_data] )

    import csv
    with open("dog%s.csv" % sys.argv[1], 'w') as f:
        writer = csv.writer(f)
        for row in csvdata:
            writer.writerow(row)

#
# Line graphs 
#
elif sys.argv[1] == "20":
    print("Creating dog%s*.pdf" % sys.argv[1])
    #

    for python, fileformat in sorted(info.keys()):

        pp = PdfPages('dog%s_%s_%s.pdf' % (sys.argv[1], python, fileformat))

        for problem in sorted(info[python, fileformat].keys()):
            line = 0
            print(python, fileformat, problem)
            fig, ax = plt.subplots()
            for branch in sorted(info[python, fileformat][problem].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_.append(i)
                    y_.append( info[python, fileformat][problem][branch].get(date, None) )
                    i += 1

                ax.plot(x_, y_, 'o-', markersize=2, label=branch, c=colors[line])
                line += 1

            ax.set_ylabel(problem)
            #ax.set_xlabel("Test Date")
            #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
            ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))

            plt.subplots_adjust(right=0.7)
            pp.savefig()
            plt.clf()

#
# History Graphs for MASTER
#
elif sys.argv[1] == "21":
    print("Creating dog%s*.pdf" % sys.argv[1])
    print(dates)
    if len(dates) > 25:
        dates = dates[-25:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]

    #info[python][problem][fileformat][branch][date] = float(meanval)
    for python in info:
      pp = PdfPages('dog%s_%s.pdf' % (sys.argv[1], python))
      fig, ax = plt.subplots()
      ax.text(0,0.75,"Figures that compare runtime to generate problem files")
      ax.text(0,0.70,"    Comparison for different file types")
      ax.text(0,0.65,"    Latest Master Tests vs Current Release")
      ax.axes.get_xaxis().set_visible(False)
      ax.axes.get_yaxis().set_visible(False)
      ax.axis('off')
      pp.savefig()
      plt.clf()
      plt_counter = 0

      fig = plt.figure()

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        ax = fig.add_subplot(5,1,plt_counter+1)
        line = 0
        for fileformat in sorted(info[python][problem].keys()):
            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            #print("HERE")
            #print( info[python][problem][fileformat].keys() )
            for branch in ['master']:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    y_.append( info[python][problem][fileformat][branch].get(date, None) )
                    i -= 1

                x_ = x_ + [1]
                val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                if val is not None:
                    val = float(val)
                y_.append( val )

                #print(fileformat)
                #print(x_)
                #print(y_)
                ax.plot(x_, y_, 'o-', markersize=2, label=fileformat, c=colors[line])
                #ax.plot(x_, y_, 'o-', markersize=2, label=branch, c=colors[line])
                line += 1

        ax.set_xlim(left=x_[0], right=x_[-1])
        ylow, yhi = ax.get_ylim()
        ax.text(x_[0]+0.05, (yhi-ylow)*0.8+ylow, problem, fontsize=9)
        if plt_counter == 0:
            #ax.set_title("Time to Write Model Files: %s" % (python), fontsize=9)
            ax.set_title("Time to Write Model Files", fontsize=9)
        #ax.set_ylabel(problem)
        if plt_counter == 4:
            ax.set_xlabel("Latest Master Tests vs Current Release", fontsize=9)
            plt.xticks(x_, x_[:-1] + ["v"+latest_version])
            ax.axes.get_xaxis().set_visible(True)
        else:
            ax.axes.get_xaxis().set_visible(False)
        for label in (ax.get_xticklabels() + ax.get_yticklabels()):
            label.set_fontsize(9)
        #ax.set_xlabel("Test Date")
        #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
        ax.legend(loc='upper center', bbox_to_anchor=(1.1,1), borderaxespad=0.)

        #plt.subplots_adjust(bottom=0.1)
        plt.subplots_adjust(right=0.8)
        plt_counter = (plt_counter + 1) % 5
        fig.canvas.draw()
        plt.show()
        if plt_counter == 0:
            print("HERE")
            pp.savefig()
            plt.clf()
            fig = plt.figure()

      pp.close()

#
# History Graphs for MASTER
#
elif sys.argv[1] == "22":
    formats = set()
    problems = set()
    results = {}

    print("Running dog22 %s" % sys.argv[1])
    if len(dates) > 50:
        dates = dates[-50:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]

    devbranches = sys.argv[3:]
    print(devbranches)
    for python in info:

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        problems.add(problem)
        for fileformat in sorted(info[python][problem].keys()):
            formats.add(fileformat)
            line = 0

            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            traces = []
            _val = None
            _baseline = None
            _missing = True
            latest = None
            for branch in ['master'] + devbranches:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    if branch in info[python][problem][fileformat].keys():
                        y_.append( info[python][problem][fileformat][branch].get(date, None) )
                    else:
                        y_.append( None )
                    i -= 1

                traces.append( Scatter(x=x_, y=y_, mode='lines+markers', name=branch, text=dates) )

                if branch == 'master':
                    val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                    if val is not None:
                        if y_[-1] is not None:
                            _val = y_[-1]
                            _missing = False
                        _baseline = float(val)
                        val = _baseline
                        latest = Scatter(x=x_, y=[val]*len(x_), mode='lines', name="v"+latest_version)

            if latest is not None:
                traces.insert(0, latest)
            data = ( traces )

            xaxis = dict(
                        title="Latest Tests vs Current Release", 
                        tickmode="array", 
                        tickvals=x_, 
                        ticktext=[str(v) for v in x_],
                        showticklabels=True, 
                        )
            layout = Layout( 
                        title="Python=%s  Problem=%s  Fileformat=%s" % (python, problem, fileformat),
                        showlegend=True,
                        xaxis=xaxis,
                        yaxis=dict(title="Runtime (Seconds)") )
            fig = Figure(data=data, layout=layout)
            fname = "dog22_%s_%s_%s.html" % (python, problem, fileformat)
            results[python, problem, fileformat] = (fname, _val, _baseline, _missing)
            plotly.offline.plot( fig, filename='results/'+fname, auto_open=False )

      pass
    #
    # Create main file
    #
    _formats = list(sorted(formats))
    _problems = list(sorted(problems))
    with open("results/index.html", 'w') as OUTPUT:
        OUTPUT.write('<style type="text/css">\n')
        OUTPUT.write('a.dark:link { color: lightgrey; } a.dark:visited { color: lightgrey; }\n')
        OUTPUT.write('a.light:link { color: darkgrey; } a.light:visited { color: darkgrey; }\n')
        OUTPUT.write("tr, th { text-align: right; border: 1px solid black; border-collapse: collapse;}\n")
        OUTPUT.write("td { padding: 5px; }\n")
        OUTPUT.write("td.missing_data { background-color: darkred; color: white; }\n")
        OUTPUT.write("td.no_baseline { background-color: red; }\n")
        OUTPUT.write("td.very_slow { background-color: gold; }\n")
        OUTPUT.write("td.slow { background-color: yellow; }\n")
        OUTPUT.write("td.ok { background-color: lightgreen; }\n")
        OUTPUT.write("td.fast { background-color: green; }\n")
        OUTPUT.write("td.very_fast { background-color: darkgreen; }\n")
        OUTPUT.write("td.none { background-color: black; }\n")
        OUTPUT.write("</style>\n")
        OUTPUT.write("<h2>Pyomo Performance Comparisons: Writing Problem Files</h2>\n")
        OUTPUT.write("<p>The following tables describe the performance ratio for the runtime of Pyomo branches relative to the most recent Pyomo release.</p>\n")
        OUTPUT.write("<p>The runtime performance includes the time to generate a Pyomo model and write a problem file (in various formats).</p>\n")
        # PYTHON
        for python in sorted(info.keys()):
            OUTPUT.write("<hr>\n")
            OUTPUT.write("<p>{0}</p>\n".format(python))
            OUTPUT.write('<table>\n')
            # HEADER
            OUTPUT.write("<tr>\n")
            OUTPUT.write("<th>Problem</th>")
            for f in _formats:
                OUTPUT.write("<th>{0}</th>".format(f))
            OUTPUT.write("</tr>\n")
            # PROBLEMS
            for problem in _problems:
                OUTPUT.write("<tr>\n")
                OUTPUT.write("<th>{0}</th>".format(problem))
                for f in _formats:
                    val = results.get( (python, problem, f), None)
                    if val is None:
                        OUTPUT.write('<td class="none"></td>')
                    else:
                        _url, _val, _baseline, _missing = val
                        if _missing:
                            OUTPUT.write('<td align="center" class="missing_data"><a class="dark" href="%s">NA</a></td>' % _url)
                        elif _baseline is None:
                            OUTPUT.write('<td align="center" class="no_baseline"><a class="dark" href="%s">X</a></td>' % _url)
                        else:
                            ratio = _val/_baseline
                            if ratio >= 1.10:
                                OUTPUT.write('<td align="center" class="very_slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio >= 1.05:
                                OUTPUT.write('<td align="center" class="slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio > 0.95:
                                OUTPUT.write('<td align="center" class="ok"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio > 0.90:
                                OUTPUT.write('<td align="center" class="fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,ratio))
                            else:
                                OUTPUT.write('<td align="center" class="very_fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,ratio))
                OUTPUT.write("</tr>\n")
            OUTPUT.write("</table>\n")
            OUTPUT.write("</body></html>\n")
        OUTPUT.write("<hr>\n")

#
# History Graphs for MASTER
#
elif sys.argv[1] == "23":
    formats = set()
    problems = set()
    results = {}
    lines = {}

    print("Running dog23 %s" % sys.argv[1])
    if len(dates) > 50:
        dates = dates[-50:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]
    devbranches = sys.argv[3:]
    print(devbranches)


    #
    # GRAPHS 1
    #
    for python in info:

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        problems.add(problem)
        for fileformat in sorted(info[python][problem].keys()):
            formats.add(fileformat)
            #fig, ax = plt.subplots()
            line = 0

            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            traces = []
            _val = None
            _baseline = None
            _missing = True
            latest = None
            for branch in ['master'] + devbranches:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    if branch in info[python][problem][fileformat].keys():
                        y_.append( info[python][problem][fileformat][branch].get(date, None) )
                    else:
                        y_.append( None )
                    i -= 1

                traces.append( Scatter(x=x_, y=y_, mode='lines+markers', name=branch, text=dates) )
                lines[python, problem, fileformat, branch] = (x_, y_)

                if branch == 'master':
                    val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                    if val is not None:
                        if y_[-1] is not None:
                            _val = y_[-1]
                            _missing = False
                        _baseline = float(val)
                        val = _baseline
                        latest = Scatter(x=x_, y=[val]*len(x_), mode='lines', name="v"+latest_version)

            if latest is not None:
                traces.insert(0, latest)
            data = ( traces )

            xaxis = dict(
                        title="Latest Tests vs Current Release", 
                        tickmode="array", 
                        tickvals=x_, 
                        ticktext=[str(v) for v in x_],
                        showticklabels=True, 
                        )
            layout = Layout( 
                        title="Python=%s  Problem=%s  Fileformat=%s" % (python, problem, fileformat),
                        showlegend=True,
                        xaxis=xaxis,
                        yaxis=dict(title="Runtime (Seconds)") )
            fig = Figure(data=data, layout=layout)
            fname = "dog23_%s_%s_%s.html" % (python, problem, fileformat)
            results[python, problem, fileformat] = (fname, _val, _baseline, _missing)
            plotly.offline.plot( fig, filename='results/'+fname, auto_open=False )

    #
    # GRAPHS 2
    #
    _formats = list(sorted(formats))
    _problems = list(sorted(problems))
    _python = list(sorted(info.keys()))
    results2 = {}
    for problem in _problems:
        for fileformat in _formats:
            traces = []
            _val = {}
            _baseline = {}
            for python in _python:
                x_, y_ = lines.get( (python, problem, fileformat, "master"), (None, None))
                _val[python] = None
                _baseline[python] = None
                if x_ is not None:
                    if y_[-1] is not None:
                        _val[python] = y_[-1]
                    val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                    if val is not None:
                        _baseline[python] = float(val)
                    traces.append( Scatter(x=x_, y=y_, mode='lines+markers', name=python, text=dates) )
            data = ( traces )

            fname = "dog23b_%s_%s.html" % (problem, fileformat)
            results2[problem, fileformat] = (fname, _val, _baseline, len(data) == 0)
            if len(data) > 0:
                xaxis = dict(
                            title="Latest Tests vs Current Release", 
                            tickmode="array", 
                            tickvals=x_, 
                            ticktext=[str(v) for v in x_],
                            showticklabels=True, 
                            )
                layout = Layout( 
                            title="Problem=%s  Fileformat=%s" % (problem, fileformat),
                            showlegend=True,
                            xaxis=xaxis,
                            yaxis=dict(title="Runtime (Seconds)") )
                fig = Figure(data=data, layout=layout)
                plotly.offline.plot( fig, filename='results/'+fname, auto_open=False )

    #
    # GRAPHS 3
    #
    if False:
        for fileformat in _formats:
            for python in _python:
                traces = []
                for problem in _problems:
                    x_, y_ = lines.get( (python, problem, fileformat, "master"), (None, None))
                    if x_ is not None:
                        i = len(y_)-1
                        while i >= 0 and y_[i] is None:
                            i = i - 1
                        y__ = copy.copy(y_)
                        if i >= 0:
                            val = y__[i]
                            for j in range(len(y__)):
                                if y__[j] is not None:
                                    y__[j] /= val
                            traces.append( Scatter(x=x_, y=y__, mode='lines+markers', name=problem, text=dates) )
                data = ( traces )

                fname = "dog23_relative_%s_%s.html" % (fileformat, python)
                xaxis = dict(
                            title="Performance Relative to Current Performance", 
                            tickmode="array", 
                            tickvals=x_, 
                            ticktext=[str(v) for v in x_],
                            showticklabels=True, 
                            )
                layout = Layout( 
                            title="Python=%s  Fileformat=%s" % (python, fileformat),
                            showlegend=True,
                            xaxis=xaxis,
                            yaxis=dict(title="Runtime (Seconds)") )
                fig = Figure(data=data, layout=layout)
                plotly.offline.plot( fig, filename='results/'+fname, auto_open=False )

    #
    # GRAPHS 4
    #

    #
    # Compute change points
    #
    ncp = {}
    cp = {}
    breaks = {}
    ymin = {}
    ymax = {}
    for fileformat in _formats:
        for python in _python:
            print((python, fileformat))
            ignore = set()
            vals_ = []
            ntraces = 0
            for problem in _problems:
                x_, y_ = lines.get( (python, problem, fileformat, "master"), (None, None))
                if x_ is not None:
                    ymax[python, fileformat, ntraces] = max(v for v in y_ if v is not None)
                    ymin[python, fileformat, ntraces] = min(v for v in y_ if v is not None)
                    npoints = len(y_)
                    for i in range(npoints):
                        if y_[i] is None:
                            ignore.add(i)
                    vals_.append(y_)
                    ntraces += 1
            if len(data) == 0:
                print("WARNING: No plots for Python=%s FileFormat=%s" % (python, fileformat))
                continue
            #
            # Compute change points
            #
            print(("NPOINTS", npoints))
            print(("NIGNORE", len(ignore)))
            vals = np.zeros( (ntraces, npoints-len(ignore)) )
            breakmap = [None]*(npoints-len(ignore))
            for i, y_ in enumerate(vals_):
                k = 0
                for j, val in enumerate(y_):
                    if j not in ignore:
                        vals[i, k] = val
                        breakmap[k] = j
                        k += 1
            #print(vals)
            numCh,chP = autoDynKWRupt(vals)
            ncp[python, fileformat] = numCh
            print((python, fileformat))
            print(("numCh", numCh))
            breaks[python, fileformat] = []
            for i in range(numCh):
                breaks[python, fileformat].append( breakmap[chP[i]] )
                cp[breakmap[chP[i]]] = cp.get(breakmap[chP[i]], 0) + 1
                print((breakmap[chP[i]], breakmap[chP[i]]-49))
                #for j, y_ in enumerate(vals_):
                    #traces.append( Scatter(x=[breaks[i]]*2, y=[min(vals_[i])val]*len(x_), mode='lines')
    #
    # Find the change points that are common across our python/fileformat statistical tests
    #
    tmp = list(sorted(cp.keys()))
    common_cp = set()
    for i in range(1,len(tmp)-1):
        if tmp[i-1] == tmp[i]-1 and tmp[i+1] == tmp[i]+1 and cp[tmp[i]] >=3 and cp[tmp[i]] > cp[tmp[i-1]] and cp[tmp[i]] > cp[tmp[i+1]]:
            print((tmp[i], tmp[i]-48.5, cp[tmp[i]]))
            common_cp.add(tmp[i])
    #
    # Now create the graph
    #
    for fileformat in _formats:
        for python in _python:
            traces = []
            for problem in _problems:
                x_, y_ = lines.get( (python, problem, fileformat, "master"), (None, None))
                if x_ is not None:
                    npoints = len(y_)
                    i = npoints-1
                    while i >= 0 and y_[i] is None:
                        i = i - 1
                    if i >= 0:
                        traces.append( Scatter(x=x_, y=y_, mode='lines+markers', name=problem, text=dates, marker=dict(color=colors_rgb[len(traces)]), line=dict(color=colors_rgb[len(traces)])) )
            data = ( traces )
            if len(data) == 0:
                print("WARNING: No plots for Python=%s FileFormat=%s" % (python, fileformat))
                continue
            #
            # Create graph
            #
            fig = tools.make_subplots(rows=len(traces), cols=1, shared_yaxes=False, shared_xaxes=True)
            for i in range(len(traces)):
                fig.append_trace( traces[i], i+1, 1 )
                for cp in common_cp:
                    color = 'black'
                    fig.append_trace( Scatter(x=[cp-48.5]*2, y=[ymin[python, fileformat, i], ymax[python, fileformat, i]], mode='lines', showlegend=False, line=dict(color=color, width=3)), i+1, 1 )
                #for j in range(ncp[python, fileformat]):
                    #if breaks[python, fileformat][j] in common_cp:
                    #    color = 'red'
                    #else:
                    #    color = 'black'
                    #fig.append_trace( Scatter(x=[breaks[python, fileformat][j]-48.5]*2, y=[ymin[python, fileformat, i], ymax[python, fileformat, i]], mode='lines', showlegend=False, line=dict(color=color, width=3)), i+1, 1 )
            fname = "dog23_stacked_%s_%s.html" % (fileformat, python)
            xaxis = dict(
                        title="Performance Relative to Current Performance", 
                        tickmode="array", 
                        tickvals=x_, 
                        ticktext=[str(v) for v in x_],
                        showticklabels=True, 
                        )
            fig['layout'].update(
                        title="Python=%s  Fileformat=%s" % (python, fileformat),
                        showlegend=True,
                        height=100*len(data))
                        #yaxis=dict(title="Runtime (Seconds)") )
                        #xaxis=xaxis)
            plotly.offline.plot( fig, filename='results/'+fname, auto_open=False )

    #
    # Create main file
    #
    with open("results/index.html", 'w') as OUTPUT:
        OUTPUT.write('<style type="text/css">\n')
        OUTPUT.write('a.dark:link { color: lightgrey; } a.dark:visited { color: lightgrey; }\n')
        OUTPUT.write('a.light:link { color: darkgrey; } a.light:visited { color: darkgrey; }\n')
        OUTPUT.write("tr, th { text-align: right; border: 1px solid black; border-collapse: collapse;}\n")
        OUTPUT.write("th.problem { text-align: left; border: 1px solid black; }\n")
        OUTPUT.write("th.python { text-align: center; border: 1px solid black; }\n")
        OUTPUT.write("th.gap { border: 1px white; }\n")
        OUTPUT.write("td { padding: 5px; }\n")
        OUTPUT.write("td.problem { text-align: left; border: 1px solid black; }\n")
        OUTPUT.write("td.missing_data { background-color: darkred; color: white; }\n")
        OUTPUT.write("td.no_baseline { background-color: red; }\n")
        OUTPUT.write("td.very_slow { background-color: gold; }\n")
        OUTPUT.write("td.slow { background-color: yellow; }\n")
        OUTPUT.write("td.ok { background-color: lightgreen; }\n")
        OUTPUT.write("td.fast { background-color: green; }\n")
        OUTPUT.write("td.very_fast { background-color: darkgreen; }\n")
        OUTPUT.write("td.none { background-color: black; }\n")
        OUTPUT.write("tr.gap { }\n")
        OUTPUT.write("</style>\n")
        OUTPUT.write("<h2>Pyomo Performance Comparisons: Writing Problem Files</h2>\n")
        # PYTHON
        OUTPUT.write("<hr>\n")

        #
        # TABLE 1
        #
        OUTPUT.write("<h3>Relative Performance Summary - Master</h3>\n")
        OUTPUT.write("<p>The following table describes the performance ratio for the runtime of the Pyomo master branch relative to the most recent Pyomo release.</p>\n")
        OUTPUT.write("<p>The table contains links to nightly performance for select branches of Pyomo.</p>\n")
        OUTPUT.write("<p>The runtime measure includes the time to generate a Pyomo model and write a problem file (in various formats).</p>\n")
        OUTPUT.write('<table>\n')
        # HEADER
        OUTPUT.write("<tr>\n")
        OUTPUT.write('<td></td>')
        nformats = len(_formats)
        for python in sorted(info.keys()):
            OUTPUT.write('<th class="gap"></th>')
            OUTPUT.write('<th colspan="{0}" class="python">{1}</th>'.format(nformats, python))
        OUTPUT.write("</tr>\n")
        # HEADER2
        OUTPUT.write("<tr>\n")
        OUTPUT.write('<th class="problem">Problem</th>')
        nformats = len(_formats)
        for python in info.keys():
            OUTPUT.write('<th class="gap"></th>')
            for f in _formats:
                OUTPUT.write('<th class="python">{0}</th>'.format(f))
        OUTPUT.write("</tr>\n")
        # PROBLEMS
        for problem in _problems:
            OUTPUT.write("<tr>\n")
            OUTPUT.write('<td class="problem">{0}</td>'.format(problem))
            for python in sorted(info.keys()):
                OUTPUT.write('<td class="gap"></td>')
                for f in _formats:
                    val = results.get( (python, problem, f), None)
                    if val is None:
                        OUTPUT.write('<td class="none"></td>')
                    else:
                        _url, _val, _baseline, _missing = val
                        if _missing:
                            OUTPUT.write('<td align="center" class="missing_data"><a class="dark" href="%s">NA</a></td>' % _url)
                        elif _baseline is None:
                            OUTPUT.write('<td align="center" class="no_baseline"><a class="dark" href="%s">X</a></td>' % _url)
                        else:
                            ratio = _val/_baseline
                            if ratio >= 1.10:
                                OUTPUT.write('<td align="center" class="very_slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio >= 1.05:
                                OUTPUT.write('<td align="center" class="slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio > 0.95:
                                OUTPUT.write('<td align="center" class="ok"><a class="light" href="%s">%2.2f</a></td>' % (_url,ratio))
                            elif ratio > 0.90:
                                OUTPUT.write('<td align="center" class="fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,ratio))
                            else:
                                OUTPUT.write('<td align="center" class="very_fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,ratio))
            OUTPUT.write("</tr>\n")
        OUTPUT.write("</table>\n")
        OUTPUT.write("<hr>\n")

        #
        # TABLE 2
        #
        OUTPUT.write("<h3>Absolute Performance Summary</h3>\n")
        OUTPUT.write("<p>The following table describes the runtime of the latest execution of the Pyomo master branch.</p>\n")
        OUTPUT.write("<p>The runtime measure includes the time to generate a Pyomo model and write a problem file (in various formats).</p>\n")
        OUTPUT.write('<table>\n')
        # HEADER
        OUTPUT.write("<tr>\n")
        OUTPUT.write('<td></td>')
        npython = len(info.keys())
        for f in _formats:
            OUTPUT.write('<th class="gap"></th>')
            OUTPUT.write('<th colspan="{0}" class="python">{1}</th>'.format(npython, f))
        OUTPUT.write("</tr>\n")
        # HEADER2
        OUTPUT.write("<tr>\n")
        OUTPUT.write('<th class="problem">Problem</th>')
        for f in _formats:
            OUTPUT.write('<th class="gap"></th>')
            for python in sorted(info.keys()):
                if python.startswith("python"):
                    _py = "py"+python[6:]
                else:
                    _py = python
                OUTPUT.write('<th class="python">{0}</th>'.format(_py))
        OUTPUT.write("</tr>\n")
        # PROBLEMS
        for problem in _problems:
            OUTPUT.write("<tr>\n")
            OUTPUT.write('<td class="problem">{0}</td>'.format(problem))
            for f in _formats:
                OUTPUT.write('<td class="gap"></td>')
                val = results2.get( (problem, f), None)
                for python in sorted(info.keys()):
                    if val is None:
                        OUTPUT.write('<td class="none"></td>')
                    else:
                        _url, _val, _baseline, _missing = val
                        _val = _val[python]
                        _baseline = _baseline[python]
                        if _missing:
                            OUTPUT.write('<td class="none"></td>')
                        elif _val is None:
                            OUTPUT.write('<td align="center" class="no_baseline"><a class="dark" href="%s">NA</a></td>' % _url)
                        elif _baseline is None:
                            OUTPUT.write('<td align="center" class="missing_data"><a class="dark" href="%s">%2.2f</a></td>' % (_url,_val))
                        else:
                            ratio = _val/_baseline
                            if ratio >= 1.10:
                                OUTPUT.write('<td align="right" class="very_slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,_val))
                            elif ratio >= 1.05:
                                OUTPUT.write('<td align="right" class="slow"><a class="light" href="%s">%2.2f</a></td>' % (_url,_val))
                            elif ratio > 0.95:
                                OUTPUT.write('<td align="right" class="ok"><a class="light" href="%s">%2.2f</a></td>' % (_url,_val))
                            elif ratio > 0.90:
                                OUTPUT.write('<td align="right" class="fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,_val))
                            else:
                                OUTPUT.write('<td align="right" class="very_fast"><a class="dark" href="%s">%2.2f</a></td>' % (_url,_val))
            OUTPUT.write("</tr>\n")
        OUTPUT.write("</table>\n")
        OUTPUT.write("<hr>\n")

        #
        # Relative Graph
        #
        if False:
            OUTPUT.write("<h3>Relative Performance Trends</h3>\n")
            OUTPUT.write("<p>The following graph shows the performance trends relative to the runtime of the latest execution of the Pyomo master branch.</p>\n")
            OUTPUT.write("<p>The runtime measure includes the time to generate a Pyomo model and write a problem file (in various formats).</p>\n")
            OUTPUT.write("<p></p>\n")
            OUTPUT.write('<table>\n')
            # HEADER
            OUTPUT.write("<tr>\n")
            OUTPUT.write('<td></td>')
            for fileformat in _formats:
                OUTPUT.write('<td class="python">%s</td>' % fileformat)
            OUTPUT.write("</tr>\n")
            # FILE FORMATS
            for python in _python:
                OUTPUT.write('<td align="left" class="problem">%s</td>' % python)
                for fileformat in _formats:
                    OUTPUT.write('<td class="very_fast"><a align="center" href="dog23_relative_%s_%s.html">___</a></td>' % (fileformat, python))
                OUTPUT.write("</tr>\n")
            OUTPUT.write("</table>\n")
            OUTPUT.write("<hr>\n")

        #
        # Stacked Graphs
        #
        OUTPUT.write("<h3>Performance Trends</h3>\n")
        OUTPUT.write("<p>The following graphs show the performance trends for Pyomo on each test problem.  The number in the table below is the number of statistical changepoints detected across Pyomo's performance for a python-fileformat pair.</p>\n")
        OUTPUT.write("<p>The runtime measure includes the time to generate a Pyomo model and write a problem file (in various formats).</p>\n")
        OUTPUT.write("<p></p>\n")
        OUTPUT.write('<table>\n')
        # HEADER
        OUTPUT.write("<tr>\n")
        OUTPUT.write('<td></td>')
        for fileformat in _formats:
            OUTPUT.write('<th class="python">%s</th>' % fileformat)
        OUTPUT.write("</tr>\n")
        # FILE FORMATS
        for python in _python:
            OUTPUT.write('<td align="left" class="problem">%s</td>' % python)
            for fileformat in _formats:
                OUTPUT.write('<td class="very_fast"><a class="dark" href="dog23_stacked_%s_%s.html">___</a></td>' % (fileformat, python))
            OUTPUT.write("</tr>\n")
        OUTPUT.write("</table>\n")
        OUTPUT.write("<hr>\n")

        OUTPUT.write("</body></html>\n")