From c0082ef78c24a8f007d77eb771b8fa96afefbf45 Mon Sep 17 00:00:00 2001
From: Mart Ratas <mart.ratas@gmail.com>
Date: Fri, 1 Nov 2024 10:52:53 +0000
Subject: [PATCH] CU-8695hghww backwards compatibility workflow (#478)

* CU-8695hghww: Add bash script to run backwards compatibility

* CU-8695hghww: Rename backwards compatibility running bash script

* CU-8695hghww: Add new step to workflow to run model backwards compatibility

* CU-8695hghww: Fix model compatibility regression suite path

* CU-8695hghww: Simplify creation and removal of fake model folder
---
 .github/workflows/main.yml                    |  2 +
 .../check_backwards_compatibility.sh          | 43 +++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 tests/resources/model_compatibility/check_backwards_compatibility.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index fb027f62..b4a84f16 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -42,6 +42,8 @@ jobs:
           timeout 25m python -m unittest ${second_half_nl[@]}
       - name: Regression
         run: source tests/resources/regression/run_regression.sh
+      - name: Model backwards compatibility
+        run: source tests/resources/model_compatibility/check_backwards_compatibility.sh
       - name: Get the latest release version
         id: get_latest_release
         uses: actions/github-script@v6
diff --git a/tests/resources/model_compatibility/check_backwards_compatibility.sh b/tests/resources/model_compatibility/check_backwards_compatibility.sh
new file mode 100644
index 00000000..5e3fd2ae
--- /dev/null
+++ b/tests/resources/model_compatibility/check_backwards_compatibility.sh
@@ -0,0 +1,43 @@
+# CONSTANTs/ shouldn't change
+REGRESSION_MODULE="medcat.utils.regression.regression_checker"
+REGRESSION_OPTIONS="--strictness STRICTEST --require-fully-correct"
+
+# CHANGABLES
+# target models
+DL_LINK="https://cogstack-medcat-example-models.s3.eu-west-2.amazonaws.com/medcat-example-models/all_fake_medcat_models.zip"
+ZIP_FILE_NAME="all_fake_medcat_models.zip"
+# target regression set
+REGRESSION_TEST_SET="tests/resources/regression/testing/test_model_regresssion.yml"
+# folder to house models under test
+MODEL_FOLDER="fake_models"
+
+# START WORK
+
+echo "Downloading models"
+wget $DL_LINK
+# Create folder if it doesn't exit
+mkdir -p "$MODEL_FOLDER"
+echo "Uncompressing files"
+unzip $ZIP_FILE_NAME -d $MODEL_FOLDER
+echo "Cleaning up the overall zip"
+rm $ZIP_FILE_NAME
+for model_path in `ls $MODEL_FOLDER/*.zip`; do
+    if [ -f "$model_path" ]; then
+        echo "Processing $model_path"
+        python -m $REGRESSION_MODULE \
+            "$model_path" \
+            $REGRESSION_TEST_SET \
+            $REGRESSION_OPTIONS
+        # this is a sanity check - needst to run after so that the folder has been created
+        grep "MedCAT Version" "${model_path%.*}/model_card.json"
+        # clean up here so we don't leave both the .zip'ed model
+        # and the folder so we don't fill the disk
+        echo "Cleaning up at: ${model_path%.*}"
+        rm -rf ${model_path%.*}*
+    else
+        echo "No files found matching the pattern: $file"
+    fi
+done
+
+# Remove the fake model folder
+rm -r "$MODEL_FOLDER"