Merge pull request #12 from aai-institute/fix-list-parsing

Draft: Major Update

.gitattributes

@@ -0,0 +1,5 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text
+*.drawio filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

.github/workflows/testing

@@ -0,0 +1,36 @@
+name: check
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: "0 8 * * *"
+
+concurrency:
+  group: check-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: test with ${{ matrix.py }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        py:
+          - "3.11"
+        os:
+          - ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup python for test ${{ matrix.py }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.py }}
+      - name: Install tox
+        run: python -m pip install tox-gh>=1.2
+      - name: Setup test suite
+        run: tox -vv --notest
+      - name: Run test suite
+        run: tox --skip-pkg-install

.gitignore

@@ -1,3 +1,4 @@
+*.csv
 #
 .idea
 config_local.json

experiments/mnist/config.yaml

@@ -1,9 +1,9 @@
 ---
-__object__: laplace_flows.experiments.predefined.ExperimentCollection
+__object__: veriflow.experiments.predefined.ExperimentCollection
 name: mnist_basedist_comparison
 experiments:
     - &exp_laplace
-      __object__: laplace_flows.experiments.predefined.HyperoptExperiment
+      __object__: veriflow.experiments.predefined.HyperoptExperiment
       name: mnist_laplace
       scheduler: &scheduler 
         __object__: ray.tune.schedulers.ASHAScheduler
@@ -18,7 +18,7 @@ experiments:
         mode: min
       trial_config:
         dataset: &dataset
-          __object__: laplace_flows.experiments.datasets.MnistSplit
+          __object__: veriflow.experiments.datasets.MnistSplit
           digit: 0
         epochs: &epochs 20000
         patience: &patience 50
@@ -34,17 +34,17 @@ experiments:
 
         model_cfg: 
           type:
-            __class__: &model laplace_flows.flows.NiceFlow
+            __class__: &model veriflow.flows.NiceFlow
           params:
             coupling_layers: &coupling_layers 
-              __eval__: tune.choice([ 4, 6, 8])
+              __eval__: tune.choice([ 2, 3, 4, 5])
             coupling_nn_layers: &coupling_nn_layers 
-              __eval__: tune.choice([[w]*l for w in [50, 100], for l in range(4, 10)])
+              __eval__: tune.choice([[w]*l for w in [50, 100] for l in range(1, 3)])
             nonlinearity: &nonlinearity 
               __eval__: tune.choice([torch.nn.ReLU()])
             split_dim: &split_dim 50
             base_distribution: &distribution Laplace
-            permutation: &permutation half
+            permutation: &permutation LU
     - &exp_normal
       __overwrites__: *exp_laplace
       name: mnist_normal

experiments/mnist/config_best.yaml

@@ -0,0 +1,44 @@
+---
+__object__: laplace_flows.experiments.predefined.ExperimentCollection
+name: mnist_basedist_comparison
+experiments:
+    - &exp_laplace_best
+      __object__: laplace_flows.experiments.predefined.HyperoptExperiment
+      name: mnist_normal_best
+      scheduler: &scheduler 
+        __object__: ray.tune.schedulers.ASHAScheduler
+        max_t: 10000
+        grace_period: 10000
+        reduction_factor: 2
+      num_hyperopt_samples: &num_hyperopt_samples 8
+      gpus_per_trial: &gpus_per_trial 0
+      cpus_per_trial: &cpus_per_trial 1
+      tuner_params: &tuner_params
+        metric: val_loss
+        mode: min
+      trial_config:
+        dataset: &dataset
+          __object__: laplace_flows.experiments.datasets.MnistSplit
+          digit: 0
+        epochs: &epochs 10000
+        patience: &patience 50
+        batch_size: 32
+        optim_cfg: &optim 
+          optimizer:
+            __class__: torch.optim.Adam 
+          params:
+            lr: 0.005663140094965476
+            weight_decay: 0.0
+
+        model_cfg: 
+          type:
+            __class__: &model laplace_flows.flows.NiceFlow
+          params:
+            coupling_layers: 4
+            coupling_nn_layers: [50, 50]
+            nonlinearity: &nonlinearity 
+              __eval__: tune.choice([torch.nn.ReLU()])
+            split_dim: &split_dim 50
+            base_distribution: &distribution Normal
+            permutation: &permutation half
+

experiments/mnist/config_lu.yaml

@@ -0,0 +1,47 @@
+---
+__object__: veriflow.experiments.predefined.ExperimentCollection
+name: mnist_basedist_comparison
+experiments:
+    - &exp_laplace
+      __object__: veriflow.experiments.predefined.HyperoptExperiment
+      name: mnist_laplace
+      scheduler: &scheduler 
+        __object__: ray.tune.schedulers.ASHAScheduler
+        max_t: 10000
+        grace_period: 10000
+        reduction_factor: 2
+      num_hyperopt_samples: &num_hyperopt_samples 5
+      gpus_per_trial: &gpus_per_trial 0
+      cpus_per_trial: &cpus_per_trial 1
+      tuner_params: &tuner_params
+        metric: val_loss
+        mode: min
+      trial_config:
+        dataset: &dataset
+          __object__: veriflow.experiments.datasets.MnistSplit
+          digit: 0
+        epochs: &epochs 20000
+        patience: &patience 50
+        batch_size: &batch_size 
+          __eval__: tune.choice([8, 16, 32, 64])
+        optim_cfg: &optim 
+          optimizer:
+            __class__: torch.optim.Adam 
+          params:
+            lr: 
+              __eval__: tune.loguniform(1e-2, 5e-4)
+            weight_decay: 0.0
+
+        model_cfg: 
+          type:
+            __class__: &model veriflow.flows.LUFlow
+          params:
+            n_layers: &n_layers 
+              __eval__: tune.choice([ 2, 3, 4, 5, 6, 7, 8, 9, 10])
+            base_distribution: &distribution Laplace
+    - &exp_normal
+      __overwrites__: *exp_laplace
+      name: mnist_normal
+      model_cfg: 
+        params:
+          base_distribution: Normal