Merge pull request #25 from clackary/refactor/benchmark-cleanup

refactor: Benchmark cleanup

Author: jfilling

Diff for: .gitignore

@@ -9,4 +9,5 @@ DerivedData
 .build/
 .swift-version
 *.swp
-main
+main
+SwiftBenchmark

Diff for: Benchmarks/BuildingSimulation/PyTorch/PyTorchSimulator.py

@@ -1,10 +1,14 @@
 import torch
 
-# Definitions
-
+# Simulation parameters
+trials = 100
+timesteps = 20
+warmup = 3
 dTime = 0.1
-π = 3.14159265359
+printGradToCompare = False
 
+# Definitions
+π = 3.14159265359
 
 # TubeType and other custom object holding primitives will be represented with a 1D Tensor,
 # and SimParams will compose them into a 2D tensor
@@ -70,18 +74,18 @@ class SimParamsIndices:
 
 def computeResistance(floor, tube, quanta):
     geometry_coeff = 10.0
-    
+
     tubingSurfaceArea = (floor[SlabTypeIndices.iarea] / tube[TubeTypeIndices.itubeSpacing]) * π * tube[TubeTypeIndices.idiameter]
     resistance_abs = tube[TubeTypeIndices.iresistivity] * tube[TubeTypeIndices.ithickness] / tubingSurfaceArea
-    
+
     resistance_corrected = resistance_abs * geometry_coeff
 
     return resistance_corrected
 
 
 def computeLoadPower(floor, tube, quanta):
     resistance_abs = computeResistance(floor, tube, quanta)
-    
+
     conductance = 1/resistance_abs
     dTemp = floor[SlabTypeIndices.itemp] - quanta[QuantaIndices.itemp]
     power = dTemp * conductance
@@ -146,14 +150,14 @@ def simulate(simParams):
         quanta = tankAndQuanta[1]
 
         quanta = updateQuanta(quanta)
-        
+
         quantaAndPower = computeLoadPower(slab, pexTube, quanta)
         quanta = quantaAndPower[0]
         powerToBuilding = quantaAndPower[1]
         quanta = updateQuanta(quanta)
-        
+
         slab = updateBuildingModel(powerToBuilding, slab)
-        
+
     return slab[SlabTypeIndices.itemp]
 
 import time
@@ -174,16 +178,12 @@ def fullPipe(simParams):
 totalForwardTime = 0
 totalGradientTime = 0
 
-timesteps = 20
-trials = 30
-warmup = 3
-printGradToCompare = False
 
-for i in range(trials):
-    
+for i in range(trials + warmup):
+
     inputs = SimParamsConstant
     forwardTime, forwardOutput = measure(fullPipe, inputs)
-    
+
     simParams = SimParamsConstant
     def getGradient(simParams):
         gradient = torch.autograd.grad(forwardOutput, inputs)
@@ -194,15 +194,16 @@ def getGradient(simParams):
 
     if printGradToCompare:
         print(gradient)
-    
+
     if i >= warmup:
         totalForwardTime += forwardTime
         totalGradientTime += gradientTime
 
 
-averageForwardTime = totalForwardTime / (trials - warmup)
-averageGradientTime = totalGradientTime / (trials - warmup)
+averageForwardTime = totalForwardTime / trials
+averageGradientTime = totalGradientTime / trials
 
-print("timesteps:", timesteps)
 print("trials:", trials)
-print("average forward and backwards pass (gradient) time", averageGradientTime)
+print("timesteps:", timesteps)
+print(f"average forward only time: {averageForwardTime} seconds")
+print(f"average forward and backwards (gradient) time: {averageGradientTime} seconds")

Diff for: Benchmarks/BuildingSimulation/Swift/main.swift

@@ -1,9 +1,13 @@
 import _Differentiation
 import Foundation
 
-// Definitions
-
+// Simulation parameters
+let trials = 100
+let timesteps = 20
 let dTime: Float = 0.1
+let printGradToCompare = false
+
+// Definitions
 let π = Float.pi
 
 struct SimParams: Differentiable {
@@ -216,11 +220,8 @@ func fullPipe(simParams: SimParams) -> Float {
     return loss
 }
 
-var trials = 30
-var timesteps = 20
 var totalPureForwardTime: Double = 0
 var totalGradientTime: Double = 0
-let printGradToCompare = false
 
 for _ in 0 ..< trials {
     let (forwardOnly, _) = try measure {
@@ -244,6 +245,7 @@ for _ in 0 ..< trials {
 let averagePureForward = totalPureForwardTime / Double(trials)
 let averageGradient = totalGradientTime / Double(trials)
 
-print("timesteps:", timesteps)
-print("trials:", trials)
-print("average forward and back (gradient) time:", averageGradient)
+print("trials: \(trials)")
+print("timesteps: \(timesteps)")
+print("average forward only time: \(averagePureForward) seconds")
+print("average forward and back (gradient) time: \(averageGradient) seconds")

Diff for: Benchmarks/BuildingSimulation/TensorFlow/TensorFlowSimulator.py

@@ -1,13 +1,19 @@
 
 import tensorflow as tf
 
+# Simulation parameters
+trials = 100
+timesteps = 20
+warmup = 3
+dTime = 0.1
+printGradToCompare = False
+
 @tf.function
 def doMath(a):
     return a * 2
 
 # Definitions
 
-dTime = 0.1
 π = 3.14159265359
 
 
@@ -74,10 +80,10 @@ class SimParamsIndices:
 @tf.function
 def computeResistance(floor, tube, quanta):
     geometry_coeff = 10.0
-    
+
     tubingSurfaceArea = (floor[SlabTypeIndices.iarea] / tube[TubeTypeIndices.itubeSpacing]) * π * tube[TubeTypeIndices.idiameter]
     resistance_abs = tube[TubeTypeIndices.iresistivity] * tube[TubeTypeIndices.ithickness] / tubingSurfaceArea
-    
+
     resistance_corrected = resistance_abs * geometry_coeff
 
     return resistance_corrected
@@ -86,7 +92,7 @@ def computeResistance(floor, tube, quanta):
 @tf.function
 def computeLoadPower(floor, tube, quanta):
     resistance_abs = computeResistance(floor, tube, quanta)
-    
+
     conductance = 1/resistance_abs
     dTemp = floor[SlabTypeIndices.itemp] - quanta[QuantaIndices.itemp]
     power = dTemp * conductance
@@ -193,14 +199,14 @@ def simulate(simParams):
         quanta = tankAndQuanta[1]
 
         quanta = updateQuanta(quanta)
-        
+
         quantaAndPower = computeLoadPower(slab, pexTube, quanta)
         quanta = quantaAndPower[0]
         powerToBuilding = quantaAndPower[1]
         quanta = updateQuanta(quanta)
-        
+
         slab = updateBuildingModel(powerToBuilding, slab)
-        
+
     return slab[SlabTypeIndices.itemp]
 
 
@@ -224,15 +230,10 @@ def fullPipe(simParams):
 totalForwardTime = 0
 totalGradientTime = 0
 
-timesteps = 20
-trials = 30
-warmup = 3
-printGradToCompare = False
-
 for i in range(trials + warmup):
-    
+
     forwardTime, forwardOutput = measure(fullPipe, SimParamsConstant)
-    
+
     simParams = tf.Variable(SimParamsConstant)
     def getGradient(simParams):
         with tf.GradientTape() as tape:
@@ -252,9 +253,10 @@ def getGradient(simParams):
         totalGradientTime += gradientTime
 
 
-averageForwardTime = totalForwardTime / (trials - warmup)
-averageGradientTime = totalGradientTime / (trials - warmup)
+averageForwardTime = totalForwardTime / trials
+averageGradientTime = totalGradientTime / trials
 
-print("timesteps:", timesteps)
 print("trials:", trials)
-print("average forward and backwards pass (gradient) time", averageGradientTime)
+print("timesteps:", timesteps)
+print(f"average forward only time: {averageForwardTime} seconds")
+print(f"average forward and backwards (gradient) time: {averageGradientTime} seconds")