dog

#!/usr/bin/env python

import sys
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import numpy as np
import csv
import copy

colors = ['xkcd:red', 'xkcd:brown', 'xkcd:blue', 'xkcd:purple', 'xkcd:orange', 
          'xkcd:light blue', 'xkcd:teal', 'xkcd:magenta', 'xkcd:grey',
          'xkcd:turquoise', 'xkcd:cyan', 'xkcd:maroon',
          'xkcd:olive', 'xkcd:salmon', 'xkcd:beige', 'xkcd:navy blue', 'xkcd:black', 
          'xkcd:hot pink', 'xkcd:burnt', 'xkcd:bluegray', 'xkcd:battleship grey', 'xkcd:pink', 'xkcd:yellow', 'xkcd:light purple']

formats = ['lp', 'nl', 'bar', 'gms']

if len(sys.argv) == 1:
    print("dog <id>")
    sys.exit(0)

if int(sys.argv[1]) < 10:
    input = 'benchmarks/%s.csv' % sys.argv[2]
elif int(sys.argv[1]) < 20:
    input = 'benchmarks/%s_branches_stats.csv' % sys.argv[2]
else:
    input = 'benchmarks/%s_branches_stats_history.csv' % sys.argv[2]

#
# Read input file
#
if int(sys.argv[1]) < 20:
    data = []
    with open(input, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            data.append( row[1:] )
else:
    data = []
    with open(input, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            data.append( row )


if int(sys.argv[1]) < 10:
    legend = {}
    legend['simple_1'] = 'sum()'
    legend['simple_2'] = 'summation()'
    legend['simple_3'] = 'loop (e += t)'
    legend['simple_4'] = 'loop (e = e + t)'
    legend['simple_5'] = 'loop (e = t + e)'
    legend['simple_6'] = 'Sum()'
    #
    # Process data
    #
    p = {}
    labels = {}
    xticks = {}
    for row in data:
        if row[0] not in labels:
            labels[row[0]] = []
        if not row[1] in labels[row[0]]:
            if row[1] == 'master':
                append = True
                for i in range(len(labels[row[0]])):
                    if labels[row[0]][i][0].isalpha():
                        labels[row[0]].insert(i, row[1])
                        append = False
                        break
                if append:
                    labels[row[0]].append(row[1])
            else:
                labels[row[0]].append(row[1])
        if row[4] == "":
            continue

        if row[0] not in p:
            p[row[0]] = {}
        if row[2] not in p[row[0]]:
            p[row[0]][row[2]] = {}
        p[row[0]][row[2]][row[1]] = float(row[4])

elif int(sys.argv[1]) < 20:
    #
    # Process data
    #
    p = {}
    labels = {}
    xticks = {}
    for row in data:
        try:
            python, branch, problem, fileformat, part, minval, meanval, maxval, stddev  = row
        except:
            print("ERROR")
            print(row)
            raise
        if python not in labels:
            labels[python] = []
        if not branch in labels[python]:
            if branch == 'master':
                append = True
                for i in range(len(labels[python])):
                    if labels[python][i][0].isalpha():
                        labels[python].insert(i, branch)
                        append = False
                        break
                if append:
                    labels[python].append(branch)
            else:
                labels[python].append(branch)
        if meanval == "":
            continue

        if python not in p:
            p[python] = {}
        if problem not in p[python]:
            p[python][problem] = {}
        if branch not in p[python][problem]:
            p[python][problem][branch] = {}
        p[python][problem][branch][fileformat, part] = float(meanval)

elif int(sys.argv[1]) < 30:
    #
    # Process data
    #
    dates = []
    #python = set()
    #p = {}
    #labels = {}
    #xticks = {}

    info = {}

    for row in data:
        date, python, branch, problem, fileformat, part, minval, meanval, maxval, stdval = row

        if meanval == "":
            continue
        if part != 'total':
            continue
        if python.endswith('-cython'):
            python = python[:-7]
            branch = branch + '-cython'

        if len(dates) == 0 or dates[-1] != date:
            dates.append( date )
            
        if not python in info:
            info[python] = {}
        if not problem in info[python]:
            info[python][problem] = {}
        if not fileformat in info[python][problem]:
            info[python][problem][fileformat] = {}
        if not branch in info[python][problem][fileformat]:
            info[python][problem][fileformat][branch] = {}
        info[python][problem][fileformat][branch][date] = float(meanval)
    

if sys.argv[1] == "1" or sys.argv[1] == "2":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    skip = []
    if sys.argv[1] == "2":
        skip.append('simple_4')
        skip.append('simple_5')

    line_colors = {'simple_1':'r', 'simple_2':'g', 'simple_3':'b', 'simple_4':'c', 'simple_5':'m', 'simple_6':'y'}

    #
    # Plot 
    #
    for python_ in p:
        if python_.endswith('cython'):
            continue

        pversions = [python_]
        if python_+'-cython' in p:
            pversions.append( python_+'-cython' )

        for python in pversions:
          for line in p[python]:
            if line.startswith('simple'):
                if line in skip:
                    continue
                #if python.endswith('cython') and not i == 6:
                #    continue
                x_ = [-1]*len(labels[python_])
                y_ = [-1]*len(labels[python_])
                #print(line)
                #print(p[python][line])
                for x in p[python][line]:
                    i = labels[python_].index(x)
                    x_[i] = i
                    y_[i] = p[python][line][x]
                #print(x_)
                #print(y_)
                x__ = []
                y__ = []
                for i in range(len(x_)):
                    if x_[i] != -1:
                        x__.append(x_[i])
                        y__.append(y_[i])
                if len(x__) == 1:
                    if python.endswith('cython'):
                        plt.plot(x__, y__, 'o', markersize=3, label="cy: "+legend[line], c=line_colors[line])
                    else:
                        plt.plot(x__, y__, '+', markersize=7, label=legend[line], c=line_colors[line])
                else:
                    plt.plot(x__, y__, label=legend[line], c=line_colors[line])

        plt.xlabel("X Label")
        plt.ylabel("Y Label")
        plt.title("Simple Expressions: %s" % (python_))
        #plt.legend()
        #plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
        plt.legend(loc=2, borderaxespad=0.)

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), xticks[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "3":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    for python in p:
        for line in p[python]:
            if not line.endswith('_1'):
                continue
            name = line[:-2]
            x_ = [-1]*len(labels[python_])
            y_ = [-1]*len(labels[python_])

            for i in range(len(labels[python_])):
                if labels[python_][i] in p[python]["%s_4" % name]:
                    x_[i] = i
                    y_[i] = p[python]["%s_4" % name][labels[python_][i]] / p[python]["%s_1" % name][labels[python_][i]]

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', markersize=3, label=line[:-2])
            else:
                plt.plot(x__, y__, label=line[:-2])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("(Time e=e+x)/(Time sum(x))")
        plt.title("Slowdown of Simple Loops: %s" % (python))
        plt.legend()

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), labels[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "4":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    line_colors = {'simple':colors[0], 'const':colors[1], 'param':colors[2], 'mutable':colors[3], 'nonl':colors[4], 'bilinear':colors[5], 'nested':colors[6], 'weighted':colors[7]}

    for python_ in p:
        if python_.endswith('cython'):
            continue

        pversions = [python_]
        if python_+'-cython' in p:
            pversions.append( python_+'-cython' )

        for python in pversions:
          for line in p[python]:
            if not line.endswith('_1'):
                continue
            name = line[:-2]
            x_ = [-1]*len(labels[python_])
            y_ = [-1]*len(labels[python_])

            for i in range(len(labels[python_])):
                if labels[python_][i] in p[python][line]:
                    x_[i] = i
                    y_[i] = p[python][line][labels[python_][i]]

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 1:
                if python.endswith('cython'):
                    plt.plot(x__, y__, 'o', markersize=3, label="cy: "+line, c=line_colors[name])
                else:
                    plt.plot(x__, y__, '+', markersize=7, label=line, c=line_colors[name])
            else:
                plt.plot(x__, y__, label=line, c=line_colors[name])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Performance of sum(x): %s" % (python))
        plt.legend(loc=2, borderaxespad=0.)

        #print(labels[python])
        plt.xticks(range(len(labels[python_])), xticks[python_])
        #plt.show()
        pp.savefig()
        plt.clf()

    pp.close()


#
# convert
#

elif sys.argv[1] == "10":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue
        for line in p[python]:
            if '_quick' in line:
                continue
            #print(python, line, labels[python][0])
            #print(list(p[python][line].keys()))
            x_ = [-1]*len(labels[python])
            y_ = [-1]*len(labels[python])

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    x_[i] = i
                    y_[i] = val

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            if len(x__) == 0:
                continue
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', label=line, markersize=3, c=colors[lines.index(line)])
            else:
                plt.plot(x__, y__, label=line, c=colors[lines.index(line)])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Time to Create a %s File: %s" % (format_, python))
        plt.legend()

        plt.xticks(range(len(labels[python])), xticks[python])
        pp.savefig()
        plt.clf()

    pp.close()


elif sys.argv[1] == "11":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    for format_ in formats:
      for python in p:
        for line in p[python]:
            if not (format_, 'total') in p[python][line][labels[python][0]]:
                continue
            x_ = [-1]*len(labels[python])
            y_ = [-1]*len(labels[python])

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    x_[i] = i
                    try:
                        y_[i] = p[python][line][labels[python][i]][format_, 'total']
                    except KeyError:
                        print(python, line, labels[python][i])
                        print(list(p[python][line][labels[python][i]].keys()))
                        #sys.exit(0)
                        raise

            x__ = []
            y__ = []
            for i in range(len(x_)):
                if x_[i] != -1:
                    x__.append(x_[i])
                    y__.append(y_[i])
            for i in range(len(y__)):
                y__[i] = y__[i]/max(y__[0],1e-7)
            if len(x__) == 1:
                plt.plot(x__, y__, 'o', markersize=3, c=colors[lines.index(line)])
            else:
                plt.plot(x__, y__, label=line, c=colors[lines.index(line)])
    
        plt.xlabel("Pyomo Releases/Branches")
        plt.ylabel("Runtime")
        plt.title("Time to Create a %s File: %s" % (format_, python))
        plt.legend()

        plt.xticks(range(len(labels[python])), xticks[python])
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - comparison between master and other branches
#
elif sys.argv[1] == "12":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare runtime to generate problem files")
    ax.text(0,0.65,"    Master vs Branches")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        for label in labels[python]:
            tmp[label] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())

        for line in p[python]:
            if '_quick' in line:
                continue

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    tmp[labels[python][i]][ticks.index(line)] = val

        #print(tmp)
        for key in tmp:
            i = labels[python].index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Runtime")
        ax.set_title("Time to Create a %s File: %s" % (format_, python))
        ax.legend()
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')

        plt.subplots_adjust(bottom=0.5)
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - relative comparison between master and other branches
#
elif sys.argv[1] == "13":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare normalized runtime to generate problem files")
    ax.text(0,0.65,"    Master vs Branches")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        for label in labels[python]:
            tmp[label] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            for i in range(len(labels[python])):
                if labels[python][i] in p[python][line]:
                    val = p[python][line][labels[python][i]].get((format_, 'total'), None)
                    if val is None:
                        continue
                    tmp[labels[python][i]][ticks.index(line)] = val
        #print(tmp)
        #
        # Normalize data
        #
        if 'master' not in tmp:
            continue
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val
        #print(tmp)
        #
        # Create barchart
        #
        for key in tmp:
            i = labels[python].index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Normalized Runtime")
        ax.set_title("Normalized Time to Create a %s File: %s" % (format_, python))
        ax.legend()
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')

        plt.subplots_adjust(bottom=0.5)
        pp.savefig()
        plt.clf()

    pp.close()

#
# Bar graphs - relative comparison between master and other branches
#
elif sys.argv[1] == "14":
    pp = PdfPages('dog%s.pdf' % sys.argv[1])
    print("Creating dog%s.pdf" % sys.argv[1])

    fig, ax = plt.subplots()
    ax.text(0,0.75,"Figures that compare normalized runtime to generate problem files")
    ax.text(0,0.65,"    Master vs ExprDev branch vs Problems using quicksum (w/ and w/o cython)")
    ax.text(0,0.5,"Figures: (python version) * (file type)")
    ax.text(0,0.25,"* Negative values represent missing data")
    ax.axes.get_xaxis().set_visible(False)
    ax.axes.get_yaxis().set_visible(False)
    ax.axis('off')
    pp.savefig()
    plt.clf()


    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()
    #print(ticks)

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']
    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        ind = np.arange(len(ticks))
        fig, ax = plt.subplots()
        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)
        tmp['default'] = [-0.1] * len(ticks)
        tmp['cython'] = [-0.1] * len(ticks)
        tmp['quick'] = [-0.1] * len(ticks)
        tmp['quick+cython'] = [-0.1] * len(ticks)
        width = 0.8/len(tmp.keys())
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][ticks.index(line)] = val
            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][ticks.index(line)] = val
            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][ticks.index(line)] = val
            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][ticks.index(line)] = val
                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][ticks.index(line)] = val
        ##print(tmp)
        #
        # Normalize data
        #
        ##print()
        ##print(len(tmp['master']))
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val
        #print(tmp)
        #
        # Create barchart
        #
        for key in tmp:
            i = labels_.index(key)
            ax.bar(ind - 0.4 + width/2 + width*i, tmp[key], width, label=key, color=colors[i])

        ax.set_xlabel("Pyomo Test Problems")
        ax.set_ylabel("Normalized Runtime")
        ax.set_title("Normalized Time to Create a %s File: %s" % (format_, python))
        ax.set_xticks(ind)
        ax.set_xticklabels(ticks, rotation='vertical')
        ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))
        #chartBox = ax.get_position()
        #ax.set_position([chartBox.x0, chartBox.y0, chartBox.width*0.4, chartBox.height])

        plt.subplots_adjust(bottom=0.5, right=0.7)
        pp.savefig()
        plt.clf()

    pp.close()

#
# RUNTIME DATA
#
elif sys.argv[1] == "15":
    print("Creating dog%s.csv" % sys.argv[1])

    csvdata = []

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']

    csvdata.append( ['PERFORMANCE TO CONSTRUCT PROBLEM FILES: '+sys.argv[2]] + [None]*12 )
    csvdata.append( [None, None, None] + ['NORMALIZED DATA', None, None, None, None] + ['RUNTIME DATA', None, None, None, None] )
    csvdata.append( ['Python', 'Format', 'Problem'] + labels_ + labels_ )

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)
        tmp['default'] = [-0.1] * len(ticks)
        tmp['cython'] = [-0.1] * len(ticks)
        tmp['quick'] = [-0.1] * len(ticks)
        tmp['quick+cython'] = [-0.1] * len(ticks)
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][ticks.index(line)] = val
            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][ticks.index(line)] = val
            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][ticks.index(line)] = val
            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][ticks.index(line)] = val
                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][ticks.index(line)] = val

        raw = tmp
        tmp = copy.deepcopy(tmp)
        #
        # Normalize data
        #
        for i in range(len(tmp['master'])):
            val = None
            if tmp['master'][i] < 0:
                for key in tmp:
                    if tmp[key][i] > -0.1 and (val is None or tmp[key][i] < val):
                        val = tmp[key][i]
            else:
                val = tmp['master'][i]
            if val is None:
                continue
            val += 1e-7
            for key in tmp:
                if tmp[key][i] > -0.1:
                    tmp[key][i] = (tmp[key][i]+1e-7)/val

        for i in range(len(ticks)):
            raw_data = [raw[key][i] for key in labels_]
            norm_data = [tmp[key][i] for key in labels_]
            csvdata.append( [python, format_, ticks[i]] + [None if val < 0.0 else val for val in norm_data] + [None if val < 0.0 else val for val in raw_data] )

    import csv
    with open("dog%s.csv" % sys.argv[1], 'w') as f:
        writer = csv.writer(f)
        for row in csvdata:
            writer.writerow(row)


elif sys.argv[1] == "16":
    print("Creating dog%s.csv" % sys.argv[1])

    csvdata = []

    lines = []
    for python in p:
        for line in p[python]:
            if not line in lines:
                lines.append(line)

    ticks = set()
    for python in p:
      for line in p[python]:
        if '_quick' in line:
            continue
        ticks.add(line)
    ticks = list(ticks)
    ticks.sort()

    labels_ = ['master', 'default', 'cython', 'quick', 'quick+cython']

    csvdata.append( ['PERFORMANCE SUMMARY ALL DATA: '+sys.argv[2]] + [None]*12 )
    csvdata.append( [None, None, None] + ['RUNTIME DATA', None, None, None, None] )
    csvdata.append( ['Python', 'Format', 'Problem', labels_[0], None, None, labels_[1], None, None, labels_[2], None, None, labels_[3], None, None, labels_[4], None, None] )
    csvdata.append( [None, None, None] + ['Construct', 'Write', 'Total']*5 )

    for format_ in formats:
      for python in p:
        if python.endswith('cython'):
            continue

        tmp = {}
        tmp['master'] = [-0.1] * len(ticks)*3
        tmp['default'] = [-0.1] * len(ticks)*3
        tmp['cython'] = [-0.1] * len(ticks)*3
        tmp['quick'] = [-0.1] * len(ticks)*3
        tmp['quick+cython'] = [-0.1] * len(ticks)*3
        #
        # Collect Data
        #
        for line in p[python]:
            if '_quick' in line:
                continue

            if 'master' in p[python][line]:
                val = p[python][line]['master'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)] = val
                val = p[python][line]['master'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)+1] = val
                val = p[python][line]['master'].get((format_, 'total'), None)
                if not val is None:
                    tmp['master'][3*ticks.index(line)+2] = val

            if 'expr_dev' in p[python][line]:
                val = p[python][line]['expr_dev'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)] = val
                val = p[python][line]['expr_dev'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)+1] = val
                val = p[python][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['default'][3*ticks.index(line)+2] = val

            if 'expr_dev' in p.get(python+"-cython",{}).get(line,{}):
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'construct'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)] = val
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'write_problem'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)+1] = val
                val = p[python+"-cython"][line]['expr_dev'].get((format_, 'total'), None)
                if not val is None:
                    tmp['cython'][3*ticks.index(line)+2] = val

            if '_' in line:
                line_ = line.split('_')
                line_.insert(-1,'quick')
                line_ = '_'.join(line_)
                if 'expr_dev' in p[python].get(line_,{}):
                    val = p[python][line_]['expr_dev'].get((format_, 'construct'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)] = val
                    val = p[python][line_]['expr_dev'].get((format_, 'write_problem'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)+1] = val
                    val = p[python][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick'][3*ticks.index(line)+2] = val

                if 'expr_dev' in p.get(python+"-cython",{}).get(line_,{}):
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'construct'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)] = val
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'write_problem'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)+1] = val
                    val = p[python+"-cython"][line_]['expr_dev'].get((format_, 'total'), None)
                    if not val is None:
                        tmp['quick+cython'][3*ticks.index(line)+2] = val

        raw = tmp

        for i in range(len(ticks)):
            raw_data = []
            for key in labels_:
                raw_data.append( raw[key][3*i] )
                raw_data.append( raw[key][3*i+1] )
                raw_data.append( raw[key][3*i+2] )
            csvdata.append( [python, format_, ticks[i]] + [None if val < 0.0 else val for val in raw_data] )

    import csv
    with open("dog%s.csv" % sys.argv[1], 'w') as f:
        writer = csv.writer(f)
        for row in csvdata:
            writer.writerow(row)

#
# Line graphs 
#
elif sys.argv[1] == "20":
    print("Creating dog%s*.pdf" % sys.argv[1])
    #

    for python, fileformat in sorted(info.keys()):

        pp = PdfPages('dog%s_%s_%s.pdf' % (sys.argv[1], python, fileformat))

        for problem in sorted(info[python, fileformat].keys()):
            line = 0
            print(python, fileformat, problem)
            fig, ax = plt.subplots()
            for branch in sorted(info[python, fileformat][problem].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_.append(i)
                    y_.append( info[python, fileformat][problem][branch].get(date, None) )
                    i += 1

                ax.plot(x_, y_, 'o-', markersize=2, label=branch, c=colors[line])
                line += 1

            ax.set_ylabel(problem)
            #ax.set_xlabel("Test Date")
            #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
            ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))

            plt.subplots_adjust(right=0.7)
            pp.savefig()
            plt.clf()

#
# History Graphs for MASTER
#
elif sys.argv[1] == "21":
    print("Creating dog%s*.pdf" % sys.argv[1])
    print(dates)
    if len(dates) > 100:
        dates = dates[-100:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]

    #info[python][problem][fileformat][branch][date] = float(meanval)
    for python in info:
      pp = PdfPages('dog%s_%s.pdf' % (sys.argv[1], python))
      fig, ax = plt.subplots()
      ax.text(0,0.75,"Figures that compare runtime to generate problem files")
      ax.text(0,0.70,"    Comparison for different file types")
      ax.text(0,0.65,"    Latest Master Tests vs Current Release")
      ax.axes.get_xaxis().set_visible(False)
      ax.axes.get_yaxis().set_visible(False)
      ax.axis('off')
      pp.savefig()
      plt.clf()
      plt_counter = 0

      fig = plt.figure()

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        ax = fig.add_subplot(5,1,plt_counter+1)
        line = 0
        for fileformat in sorted(info[python][problem].keys()):
            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            #print("HERE")
            #print( info[python][problem][fileformat].keys() )
            for branch in ['master']:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    y_.append( info[python][problem][fileformat][branch].get(date, None) )
                    i -= 1

                x_ = x_ + [1]
                val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                if val is not None:
                    val = float(val)
                y_.append( val )

                #print(fileformat)
                #print(x_)
                #print(y_)
                ax.plot(x_, y_, 'o-', markersize=2, label=fileformat, c=colors[line])
                #ax.plot(x_, y_, 'o-', markersize=2, label=branch, c=colors[line])
                line += 1

        ax.set_xlim(left=x_[0], right=x_[-1])
        ylow, yhi = ax.get_ylim()
        ax.text(x_[0]+0.05, (yhi-ylow)*0.8+ylow, problem, fontsize=9)
        if plt_counter == 0:
            #ax.set_title("Time to Write Model Files: %s" % (python), fontsize=9)
            ax.set_title("Time to Write Model Files", fontsize=9)
        #ax.set_ylabel(problem)
        if plt_counter == 4:
            ax.set_xlabel("Latest Master Tests vs Current Release", fontsize=9)
            plt.xticks(x_, x_[:-1] + ["v"+latest_version])
            ax.axes.get_xaxis().set_visible(True)
        else:
            ax.axes.get_xaxis().set_visible(False)
        for label in (ax.get_xticklabels() + ax.get_yticklabels()):
            label.set_fontsize(9)
        #ax.set_xlabel("Test Date")
        #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
        ax.legend(loc='upper center', bbox_to_anchor=(1.1,1), borderaxespad=0.)

        #plt.subplots_adjust(bottom=0.1)
        plt.subplots_adjust(right=0.8)
        plt_counter = (plt_counter + 1) % 5
        fig.canvas.draw()
        plt.show()
        if plt_counter == 0:
            print("HERE")
            pp.savefig()
            plt.clf()
            fig = plt.figure()

      pp.close()

#
# History Graphs for MASTER
#
elif sys.argv[1] == "22":
    print("Creating dog%s*.pdf" % sys.argv[1])
    print(dates)
    if len(dates) > 100:
        dates = dates[-100:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]

    devbranches = sys.argv[3:]
    print(devbranches)
    #info[python][problem][fileformat][branch][date] = float(meanval)
    for python in info:
      pp = PdfPages('dog%s_%s.pdf' % (sys.argv[1], python))
      fig, ax = plt.subplots()
      ax.text(0,0.75,"Figures that compare runtime to generate problem files")
      ax.text(0,0.70,"    Comparison for Master vs Specified Branches")
      ax.text(0,0.65,"    Latest Master Tests vs Current Release")
      ax.axes.get_xaxis().set_visible(False)
      ax.axes.get_yaxis().set_visible(False)
      ax.axis('off')
      pp.savefig()
      plt.clf()

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        for fileformat in sorted(info[python][problem].keys()):
            fig, ax = plt.subplots()
            line = 0

            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            #print("HERE")
            #print( info[python][problem][fileformat].keys() )
            for branch in ['master'] + devbranches:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    if branch in info[python][problem][fileformat].keys():
                        y_.append( info[python][problem][fileformat][branch].get(date, None) )
                    else:
                        y_.append( None )
                    i -= 1

                if branch == 'master':
                    x_ = x_ + [1]
                    val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
                    if val is not None:
                        y_.append( float(val) )
                    else:
                        y_.append( val )
                    _size = 4
                else:
                    x_ = x_ + [1]
                    y_.append( None )
                    _size = 2

                #print(fileformat)
                #print(branch)
                #print(x_)
                #print(y_)
                #ax.plot(x_, y_, 'o-', markersize=2, label=fileformat, c=colors[line])
                ax.plot(x_, y_, 'o-', markersize=_size, label=branch, c=colors[line])
                line += 1

            ax.set_xlim(left=x_[0], right=x_[-1])
            ax.set_title("Tests: %s %s %s" % (python, problem, fileformat))
            ax.set_ylabel("Runtime (Seconds)")
            ax.set_xlabel("Latest Tests vs Current Release")
            plt.xticks(x_, x_[:-1] + ["v"+latest_version])
            #ax.set_xlabel("Test Date")
            #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
            ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))

            plt.subplots_adjust(right=0.7)
            pp.savefig()
            plt.clf()

      pp.close()

#
# History Graphs for MASTER
#
elif sys.argv[1] == "23":
    print("Creating dog%s*.pdf" % sys.argv[1])
    print(dates)
    if len(dates) > 100:
        dates = dates[-100:]
    #
    def versiontuple(v):
        return tuple(map(int, (v.split("."))))
    _release_data = {}
    releases = {}
    fname = 'benchmarks/%s_releases_stats.csv' % sys.argv[2]
    with open(fname, 'r') as f:
        reader = csv.reader(f)
        for row in reader:
            _release_data[ tuple(row[1:6]) ] = row[6]
            #print(row[1:6], row[6])
            releases[ versiontuple(row[2]) ] = row[2]
    latest_version = releases[ sorted(releases.keys())[-1] ]

    devbranches = sys.argv[3:]
    #info[python][problem][fileformat][branch][date] = float(meanval)
    for python in info:
      pp = PdfPages('dog%s_%s.pdf' % (sys.argv[1], python))
      fig, ax = plt.subplots()
      ax.text(0,0.75,"Figures that compare normalized runtime to generate problem files")
      ax.text(0,0.70,"    Comparison for Master vs Specified Branches")
      ax.text(0,0.65,"    Latest Master Tests vs Current Release")
      ax.axes.get_xaxis().set_visible(False)
      ax.axes.get_yaxis().set_visible(False)
      ax.axis('off')
      pp.savefig()
      plt.clf()

      print(python)
      for problem in sorted(info[python].keys()):
        if 'quick' in problem:
            continue
        for fileformat in sorted(info[python][problem].keys()):
            fig, ax = plt.subplots()
            line = 0

            print(problem, fileformat)
            if 'master' not in list(info[python][problem][fileformat].keys()):
                continue
            val = _release_data.get((python, latest_version, problem, fileformat, 'total'), None)
            if val is None:
                continue
            latest_baseline = float(val)

            #print("HERE")
            #print( info[python][problem][fileformat].keys() )
            for branch in ['master'] + devbranches:  #sorted(info[python][problem][fileformat].keys()):

                i = 0
                y_ = []
                x_ = []
                for date in dates:
                    x_ = [i] + x_
                    if branch in info[python][problem][fileformat]:
                        val = info[python][problem][fileformat][branch].get(date, None)
                    else:
                        val = None
                    if not val is None:
                        val = val/latest_baseline
                    y_.append( val )
                    i -= 1

                if branch == 'master':
                    x_ = x_ + [1]
                    y_.append( 1.0 )
                    _size = 4
                else:
                    x_ = x_ + [1]
                    y_.append( None )
                    _size = 2

                #print(fileformat)
                #print(x_)
                #print(y_)
                #ax.plot(x_, y_, 'o-', markersize=2, label=fileformat, c=colors[line])
                ax.plot(x_, y_, 'o-', markersize=_size, label=branch, c=colors[line])
                line += 1

            ax.set_xlim(left=x_[0], right=x_[-1])
            ax.set_title("Tests: %s %s %s" % (python, problem, fileformat))
            ax.set_ylabel("Normalized Runtime")
            ax.set_xlabel("Latest Tests vs Current Release")
            plt.xticks(x_, x_[:-1] + ["v"+latest_version])
            #ax.set_xlabel("Test Date")
            #ax.set_title("Runtime Time to Create File %s.%s (%s)" % (problem, fileformat, python))
            ax.legend(loc='upper center', bbox_to_anchor=(1.25,1))

            plt.subplots_adjust(right=0.7)
            pp.savefig()
            plt.clf()

      pp.close()