Skip to content

Commit

Permalink
Automate tracking jobs (#3318)
Browse files Browse the repository at this point in the history
* run and move the input files in regression/input_files

* call new run_tracking_inputs.py file from the ci (to test if it works as expected), removing print statements from the .py file too

* try to get run-tracking-inputs to run on my fork

* rename tracked mfiles in tracking dir to be _MFILE.DAT

* remove print statement

* delete commented out lines

* work before updating main

* parse args in run_tracking_inputs and call script in ci

* changes from review comments
  • Loading branch information
clmould authored Nov 14, 2024
1 parent 22543f1 commit 771df09
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 25 deletions.
27 changes: 2 additions & 25 deletions .github/workflows/process.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,19 +149,7 @@ jobs:
- name: Install PROCESS
run: pip install -e .
- name: Run regression input files
run: |
process -i tests/regression/input_files/large_tokamak.IN.DAT
mv tests/regression/input_files/large_tokamak.MFILE.DAT tracking/large_tokamak_MFILE.DAT
process -i tests/regression/input_files/st_regression.IN.DAT
mv tests/regression/input_files/st_regression.MFILE.DAT tracking/st_regression_MFILE.DAT
process -i tests/regression/input_files/large_tokamak_nof.IN.DAT
mv tests/regression/input_files/large_tokamak_nof.MFILE.DAT tracking/large_tokamak_nof_MFILE.DAT
process -i tests/regression/input_files/large_tokamak_once_through.IN.DAT
mv tests/regression/input_files/large_tokamak_once_through.MFILE.DAT tracking/large_tokamak_once_through_MFILE.DAT
process -i tests/regression/input_files/stellarator.IN.DAT
mv tests/regression/input_files/stellarator.MFILE.DAT tracking/stellarator_MFILE.DAT
process -i tests/regression/input_files/IFE.IN.DAT
mv tests/regression/input_files/IFE.MFILE.DAT tracking/IFE_MFILE.DAT
run: python tracking/run_tracking_inputs.py run tests/regression/input_files
- name: Archive tracked MFILEs
uses: actions/upload-artifact@v3
with:
Expand Down Expand Up @@ -216,18 +204,7 @@ jobs:
run: |
MSG=$(printf "%q " $COMMIT_MESSAGE)
git config --global --add safe.directory '*'
python tracking/tracking_data.py track process-tracking-data --mfile tracking/large_tokamak_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/large_tokamak_MFILE.DAT process-tracking-data/large_tokamak_MFILE_$(git rev-parse HEAD).DAT
python tracking/tracking_data.py track process-tracking-data --mfile tracking/st_regression_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/st_regression_MFILE.DAT process-tracking-data/st_regression_MFILE_$(git rev-parse HEAD).DAT
python tracking/tracking_data.py track process-tracking-data --mfile tracking/large_tokamak_nof_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/large_tokamak_nof_MFILE.DAT process-tracking-data/large_tokamak_nof_MFILE_$(git rev-parse HEAD).DAT
python tracking/tracking_data.py track process-tracking-data --mfile tracking/large_tokamak_once_through_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/large_tokamak_once_through_MFILE.DAT process-tracking-data/large_tokamak_once_through_MFILE_$(git rev-parse HEAD).DAT
python tracking/tracking_data.py track process-tracking-data --mfile tracking/stellarator_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/stellarator_MFILE.DAT process-tracking-data/stellarator_MFILE_$(git rev-parse HEAD).DAT
python tracking/tracking_data.py track process-tracking-data --mfile tracking/IFE_MFILE.DAT --hash ${{ github.sha }} --commit "${MSG}"
cp tracking/IFE_MFILE.DAT process-tracking-data/IFE_MFILE_$(git rev-parse HEAD).DAT
python tracking/run_tracking_inputs.py track process-tracking-data "${MSG}" ${{ github.sha }}
- name: Create the tracking dashboard
run: python tracking/tracking_data.py plot process-tracking-data --out tracking.html
- name: Archive tracking dashboard
Expand Down
73 changes: 73 additions & 0 deletions tracking/run_tracking_inputs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Run the tracked files and move into tracking directory."""

import argparse
from pathlib import Path
import shutil
import subprocess

from tracking_data import ProcessTracker


def run_and_move_tracked_files(arguments):
"""
Run PROCESS on the tracked files and move the MFiles to the tracking directory.
"""
path_to_tracking_files = Path(arguments.input_locs)

input_files = path_to_tracking_files.glob("*.IN.DAT")
tracking_dir = Path(__file__).parent
for file_path in input_files:
try:
subprocess.run(f"process -i {file_path}", shell=True, check=True)
except subprocess.CalledProcessError:
continue
created_mfile = file_path.with_name(
file_path.name.replace(".IN.DAT", ".MFILE.DAT")
)
moved_mfile = created_mfile.with_name(
created_mfile.name.replace(".MFILE.DAT", "_MFILE.DAT")
)
created_mfile.rename(tracking_dir / moved_mfile.name)


def tracking(arguments):
"""
Call the ProcessTracker, move the MFiles to the database and rename.
"""
path_to_tracking_mfiles = Path(__file__).parent
mfiles = path_to_tracking_mfiles.glob("*_MFILE.DAT")
for mfile_path in mfiles:
ProcessTracker(
mfile=mfile_path,
database=arguments.db,
message=arguments.commit,
hashid=arguments.hash,
)

copied_mfile = shutil.copy(mfile_path, Path(arguments.db) / mfile_path.name)
moved_mfile_name = copied_mfile.with_name(
copied_mfile.name.replace("_MFILE.DAT", f"_MFILE_{arguments.hash}.DAT")
)
copied_mfile.rename(moved_mfile_name)


if __name__ == "__main__":
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(help="Choose the usage mode", dest="command")

# Run command
subparser_run = subparsers.add_parser("run")
subparser_run.add_argument("input_locs", type=str)

# Tracking command
subparser_trk = subparsers.add_parser("track")
subparser_trk.add_argument("db", type=str)
subparser_trk.add_argument("commit", type=str)
subparser_trk.add_argument("hash", type=str)

arguments = parser.parse_args()

if arguments.command == "run":
run_and_move_tracked_files(arguments)
else:
tracking(arguments)

0 comments on commit 771df09

Please sign in to comment.