fixes

Author: sayakpaul

.github/workflows/benchmark.yml

@@ -37,7 +37,9 @@ jobs:
         run: |
           python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
           python -m uv pip install -e [quality,test]
-          python -m uv pip install pandas peft bitsandbytes torchprofile
+          python -m uv pip install pandas peft torchprofile
+          # Temporary.
+          pip install --force-reinstall https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl
       - name: Environment
         run: |
           python utils/print_env.py

benchmarks/benchmarking_utils.py

@@ -105,6 +105,7 @@ def post_benchmark(self, model):
     @torch.no_grad()
     def run_benchmark(self, scenario: BenchmarkScenario):
         # 0) Basic stats
+        logger.info(f"Running scenario: {scenario.name}.")
         model = model_init_fn(scenario.model_cls, **scenario.model_init_kwargs)
         num_params = round(calculate_params(model) / 1e6, 2)
         flops = round(calculate_flops(model, input_dict=scenario.get_model_input_dict()) / 1e6, 2)

benchmarks/run_all.py

@@ -44,7 +44,11 @@ def run_scripts():
         if file != "benchmarking_utils.py":
             logger.info(f"****** Running file: {file} ******")
             command = f"python {file}"
-            run_command(command.split())
+            try:
+                run_command(command)
+            except SubprocessCallException as e:
+                logger.error(f"Error running {file}: {e}")
+                continue
 
 
 def merge_csvs():