Added logging to file, avoiding the need to redirect output. Minor code refactoring. Added new activations. Added default parameters where possible. Removed unnecessary parameter. Updated documentation.

Author: mdhimes

MARGE.py

@@ -46,10 +46,9 @@ fsize     = datsize.npy
 rmse_file = rmse
 r2_file   = r2
 
-weight_file = cnn_weights.h5
+weight_file = nn_weights.keras
 
 gridsearch    = False
-architectures = 2048r-2048r-2048r-2048r
 nodes         = 2048 2048  2048  2048
 layers        = dense dense dense dense
 lay_params    = None None None None
@@ -60,7 +59,6 @@ epochs     =  500
 patience   =   20
 batch_size =  256
 
-# cnn_weights.h5
 lengthscale = 1e-4
 max_lr      = 1e-3
 

doc/MARGE_user_manual/MARGE_user_manual.tex

@@ -46,45 +46,10 @@ fsize     = datsize.npy
 rmse_file = rmse
 r2_file   = r2
 
-weight_file = cnn_weights.h5
+weight_file = nn_weights.keras
 
 gridsearch = True
 
-# Names for each architecture
-architectures = 512r-512r-512r-512r
-                1024r-1024r-1024r-1024r
-                2048r-2048r-2048r-2048r
-                4096r-4096r-4096r-4096r
-                512r-512r-512r
-                1024r-1024r-1024r
-                2048r-2048r-2048r
-                4096r-4096r-4096r
-                4096l05-4096l05-4096l05-4096r
-                4096l05-4096l05-4096l05-4096l05
-                4096l10-4096l10-4096l10-4096l10
-                4096e05-4096e05-4096e05-4096e05
-                4096e10-4096e10-4096e10-4096e10
-                4096e15-4096e15-4096e15-4096e15
-                4096e20-4096e20-4096e20-4096e20
-                64cl05-4096l05-4096l05
-                64cl05-4096l05-4096l05-4096l05
-                64cl05-4096l05-4096l05-4096l05-4096l05
-                128cl05-4096l05-4096l05
-                128cl05-4096l05-4096l05-4096l05
-                128cl05-4096l05-4096l05-4096l05-4096l05
-                256cl05-4096l05-4096l05
-                256cl05-4096l05-4096l05-4096l05
-                256cl05-4096l05-4096l05-4096l05-4096l05
-                64ce05-4096e05-4096e05
-                64ce05-4096e05-4096e05-4096e05
-                64ce05-4096e05-4096e05-4096e05-4096e05
-                128ce05-4096e05-4096e05
-                128ce05-4096e05-4096e05-4096e05
-                128ce05-4096e05-4096e05-4096e05-4096e05
-                256ce05-4096e05-4096e05
-                256ce05-4096e05-4096e05-4096e05
-                256ce05-4096e05-4096e05-4096e05-4096e05
-
 # Number of nodes per hidden layer (that has nodes)
 nodes =  512  512  512  512
         1024 1024 1024 1024
@@ -265,7 +230,6 @@ epochs     =   50
 patience   =   20
 batch_size =  256
 
-# cnn_weights.h5
 lengthscale = 2e-4
 max_lr      = 2e-3
 

example/BART_example/MARGE.cfg

@@ -1,8 +1,8 @@
 Introduction
 ============
 
-This directory holds an example of how to run MARGE using BART for data 
-generation. It matches the use case demonstrated in Himes et al. (2022), except
+This directory holds an example of how to run MARGE, using BART for data 
+generation. It matches the use case demonstrated in Himes et al. (2020), except
 with reduced accuracy to reduce the runtime.
 
 ********************************** IMPORTANT **********************************

example/BART_example/MARGE_gridsearch.cfg

@@ -0,0 +1,204 @@
+# For details on these parameters, see the MARGE_rangetest.cfg or MARGE_run.cfg 
+# files.  This file will only detail the gridsearch parameters.
+
+[DEFAULT]
+resume = False
+seed = 0
+verb = 2
+
+[MARGE]
+datagen = False
+datagenfile = datagen
+cfile = None
+processdat = False
+preservedat = True
+
+NNModel   = True
+trainflag = True
+validflag = True
+testflag  = False
+
+optimize = 0
+optngpus = 4
+optnlays = 1 3
+optlayer = dense dense dense
+optnnode = 2 4 8 16 32 64 128 256
+optmaxconvnode = 256
+optactiv = relu elu leakyrelu sig tanh
+optactrng = 0.01 0.6
+optminlr = None
+optmaxlr = None
+
+TFR_file = circle
+buffer = 15
+ncores = 6
+
+normalize = False
+scale     = True
+scalelims = -1, 1
+
+inputdir  = inputs_quickexample
+outputdir = outputs_gridsearch
+plotdir   = plots
+datadir   = ../data
+preddir   = pred
+
+ishape = 1
+oshape = 2
+
+ilog = False
+olog = False
+
+xvals = None
+xlabel = None
+ylabel = None
+
+statsaxes = all
+
+fxmean = xmean.npy
+fymean = ymean.npy
+fxstd = xstd.npy
+fystd = ystd.npy
+fxmin = xmin.npy
+fxmax = xmax.npy
+fymin = ymin.npy
+fymax = ymax.npy
+fsize = datsize.npy
+
+rmse_file = rmse
+r2_file = r2
+
+weight_file = nn_weights.keras
+
+# Determines whether to perform a grid search
+gridsearch = True
+
+# Below are where the architectures for the grid search are specified.
+# Each architecture to be considered in the grid search is on its own line.
+# These are just provided for an example, and this is not an exhaustive 
+# grid search.
+
+# Sets the number of nodes per layer that has nodes.  Space-separated values.
+nodes = 64 256 16
+        16 16 16
+        16 16 16
+        16 16 16
+        16 16 16
+        16 16 16
+        64 64 64
+        64 64 64
+        64 64 64
+        64 64 64
+        64 64 64
+        256 256 256
+        256 256 256
+        256 256 256
+        256 256 256
+        256 256 256
+
+# Sets the activation function per layer that has nodes. Space separated.
+activations = relu elu sig
+              relu relu relu
+              elu elu elu
+              elu elu elu
+              elu elu elu
+              sig sig sig
+              relu relu relu
+              elu elu elu
+              elu elu elu
+              elu elu elu
+              sig sig sig
+              relu relu relu
+              elu elu elu
+              elu elu elu
+              elu elu elu
+              sig sig sig
+
+# Sets the parameter for each activation function
+# For activation functions that do not have a parameter, or to use the default, 
+# use None.  Space separated.
+act_params =  None 0.05585 None
+              None None None
+              0.01 0.01 0.01
+              0.05 0.05 0.05
+              0.10 0.10 0.10
+              None None None
+              None None None
+              0.01 0.01 0.01
+              0.05 0.05 0.05
+              0.10 0.10 0.10
+              None None None
+              None None None
+              0.01 0.01 0.01
+              0.05 0.05 0.05
+              0.10 0.10 0.10
+              None None None              
+
+# Sets the type of each hidden layer.  Space separated.
+layers = dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+         dense dense dense
+
+# Sets the parameter for each hidden layer.  For layers that do not have free 
+# parameters, or to use the default, use None.  Space separated.
+lay_params = None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+             None None None
+
+# Set the number of iterations through the data set to consider.
+# In a grid search, you usually cannot afford to train until early stopping, 
+# so select something where it is long enough to get a decent result.
+epochs     = 60
+
+# Set the early stopping criteria.  If there is no improvement after `patience` 
+# epochs, then terminate training.  Here, since it is set higher than the 
+# number of epochs, early stopping will never engage.
+patience   = 60
+
+# Set the batch size, or number of data cases to use in each iteration.
+batch_size = 256
+
+# Set the minimum and maximum learning rate
+lengthscale = 1e-3 # This parameter can also be called `min_lr`
+max_lr      = 1e-1
+
+# Sets the cyclical learning rate mode and how long each cycle is.
+# For more details, see the User Manual.
+clr_mode  = triangular2
+clr_steps = 6
+
+# Determines which test cases to plot.  Use integers to specify the index in the 
+# test set to plot.  Space or new-line separated.  Use None to not plot any.
+# Note that plotting only works for 1D output data.
+plot_cases = None
+
+# Determines the window size for a Svitsky-Golay filter in the output plots
+# Can be useful for high resolution spectra to better see trends in differences 
+# between predicted and true spectra.
+smoothing  = 0

example/BART_example/README

@@ -1,7 +1,10 @@
+# For details on these parameters, see the MARGE_rangetest.cfg or MARGE_run.cfg 
+# files.  This file will only detail the Bayesian optimization parameters.
+
 [DEFAULT]
 resume = False
 seed = 0
-verb = 1
+verb = 2
 
 [MARGE]
 datagenfile = datagen
@@ -15,14 +18,49 @@ trainflag = True
 validflag = True
 testflag  = False
 
+# Number of models to train per GPU during the optimization
 optimize = 200
+
+# Number of GPUs to use for the optimization
 optngpus = 1
+
+# Minimum and maximum number of layers to consider during the optimization
 optnlays = 1 3
+
+# Layer types to use.  Specify as many as the maximum for `optnlays`.
+# These are used in the order they are specified.  So, if a model with 2 layers 
+# is chosen, then it will use the first 2 layers specified here.
+# If you want to do something different, such as changing the first layer but 
+# keeping the last N layers the same, then you will need to run separate 
+# optimizations.
 optlayer = dense dense dense
+
+# The allowed values for layers with nodes.  For each layer with nodes, a  
+# value from this list will be chosen.  Values of 2^n are specified here since  
+# GPUs are optimized for calculations of that size.
 optnnode = 2 4 8 16 32 64 128 256
-optmaxconvnode = 256
+
+# The maximum number of nodes (feature maps) for convolutional layers.  In this 
+# example, any conv layer would have at most 128 feature maps.  This is enabled 
+# because model size can balloon quickly if you have a large number of 
+# feature maps, leading to out-of-memory errors.  In general, start with this 
+# value equal to your maximum of `optnnode`, then reduce it if you hit OOM errors.
+optmaxconvnode = 128
+
+# The activations functions to consider during the optimization.  Each layer has 
+# its own activation function selected from this list.
 optactiv = relu elu leakyrelu sig tanh
+
+# For activation functions with parameters, this sets the minimum and maximum values 
+# that can be chosen for it.
 optactrng = 0.01 0.6
+
+# If you wish to also optimize the learning rate at the same time, you can 
+# use these parameters to set the minimum and maximum LR.  Note that it can 
+# be very, very costly to do that, since it will require training a lot more 
+# models.  It is generally recommended to not use this, and instead run multiple 
+# optimizations for different learning rate policies (specified near the end 
+# of this file, using lengthscale/min_lr and max_lr).
 optminlr = None
 optmaxlr = None
 
@@ -62,12 +100,10 @@ rmse_file = rmse
 r2_file = r2
 statsaxes = all
 
-weight_file = nn_weights.h5.keras
+weight_file = nn_weights.keras
 
 gridsearch = False
 
-architectures = d64e
-
 nodes = 64
 
 activations = elu
@@ -82,7 +118,7 @@ epochs     = 60
 patience   = 60
 batch_size = 256
 
-lengthscale = 1e-3
+lengthscale = 1e-3  # Can also call this parameter min_lr
 max_lr      = 1e-1
 
 clr_mode  = triangular2