devstone_comparative.py

from __future__ import annotations
import argparse
import csv
import re
import subprocess
import sys
import time


DEFAULT_PARAMS = ((300, 10, 0, 0), (10, 300, 0, 0), (300, 300, 0, 0))
DEFAULT_MODEL_TYPES = ("LI", "HI", "HO", "HOmod")
DEFAULT_MAX_TIME = 1e10
DEFAULT_NUM_REPS = 30

PRELAUNCH_TASKS = {
    "adevs": [
        "cmake -S devstone/adevs/ -B devstone/adevs/build/ -D CMAKE_BUILD_TYPE=Release",
        "cmake --build devstone/adevs/build/ --target devstone",
    ],
    "cadmium": {
        "v1": [
            "cmake -S devstone/cadmium/ -B devstone/cadmium/build/ -D CMAKE_BUILD_TYPE=Release",
            "cmake --build devstone/cadmium/build/ --target devstone",
        ],
        "v2": {
            "sequential": [
                "cmake -S simulators/cadmium_v2/ -B simulators/cadmium_v2/build/ -D CMAKE_BUILD_TYPE=Release",
                "cmake --build simulators/cadmium_v2/build/ --target main_devstone",
            ],
            "parallel": [
                "cmake -S simulators/cadmium_v2/ -B simulators/cadmium_v2/build/ -D CMAKE_BUILD_TYPE=Release",
                "cmake --build simulators/cadmium_v2/build/ --target parallel_main_devstone",
            ],
        },
    },
    "xdevs": {
        "c": ["cd simulators/xdevs.c/ && make"],
        "cpp": ["cd simulators/xdevs.cpp/ && make"],
        "java": {
            "sequential": ["cd simulators/xdevs.java/ && mvn clean && mvn compile && mvn package"],
            "parallel": ["cd simulators/xdevs.java/ && mvn clean && mvn compile && mvn package"],
        },
        "rs": {
            "old": ["cargo build --manifest-path simulators/xdevs.rs/Cargo.toml --release"],
            "new": {
                "sequential": ["cargo build --manifest-path simulators/xdevs.rs-new/Cargo.toml --release  --example devstone"],
                "parallel": ["cargo build --manifest-path simulators/xdevs.rs-new/Cargo.toml --release --features par_all_no_couplings --example devstone"],
                "fullparallel": ["cargo build --manifest-path simulators/xdevs.rs-new/Cargo.toml --release --features par_all --example devstone"],
            }
        },
    },
}

COMMANDS = {
    "adevs": "devstone/adevs/bin/devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
    "cadmium": {
        "v1": "devstone/cadmium/build/cadmium-dynamic-devstone --kind={model_type} --width={width} --depth={depth} --int-cycles={int_cycles} --ext-cycles={ext_cycles}",
        "v2": {
            "sequential": "simulators/cadmium_v2/bin/main_devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
            "parallel": "simulators/cadmium_v2/bin/parallel_main_devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
        },
    },
    "pypdevs": {
        "standard": {
            "python": "python3 devstone/pythonpdevs/main.py -m {model_type} -w {width} -d {depth} -i {int_cycles} -e {ext_cycles}",
            "pypy": "pypy3 devstone/pythonpdevs/main.py -m {model_type} -w {width} -d {depth} -i {int_cycles} -e {ext_cycles}",
        },
        "minimal": {
            "python": "python3 devstone/pythonpdevs-minimal/main.py -m {model_type} -d {depth} -w {width} -i {int_cycles} -e {ext_cycles}",
            "pypy": "pypy3 devstone/pythonpdevs-minimal/main.py -m {model_type} -d {depth} -w {width} -i {int_cycles} -e {ext_cycles}",
        },
    },
    "xdevs": {
        "c": "simulators/xdevs.c/examples/devstone/devstone -w {width} -d {depth} -b {model_type} -m 1",
        "cpp": "simulators/xdevs.cpp/src/xdevs/examples/DevStone/DevStone -w {width} -d {depth} -b {model_type} -m 1",
        # "go": ,  # TODO add this
        "java": {
            "sequential": "java -cp simulators/xdevs.java/target/xdevs-2.0.3-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=Coordinator",
            "parallel": "java -cp simulators/xdevs.java/target/xdevs-2.0.3-jar-with-dependencies.jar xdevs.lib.performance.DevStoneSimulation --model={model_type} --width={width} --depth={depth} --delay-distribution=Constant-{int_cycles} --coordinator=CoordinatorParallel",
        },
        "py": "python3 simulators/xdevs.py/xdevs/examples/devstone/devstone.py {model_type} {width} {depth} {int_cycles} {ext_cycles}",
        "rs": {
            "old": "simulators/xdevs.rs/target/release/xdevs {model_type} {width} {depth} {int_cycles} {ext_cycles}",
            "new": {
                "sequential": "simulators/xdevs.rs-new/target/release/examples/devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
                "parallel": "simulators/xdevs.rs-new/target/release/examples/devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
                "fullparallel": "simulators/xdevs.rs-new/target/release/examples/devstone {model_type} {width} {depth} {int_cycles} {ext_cycles}",
            }
        },
    },
}

DEFAULT_RE = r"model creation time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*).*(?:engine|simulator) (?:set ?up|creation) time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*).*simulation time(?:[ \(s\)]*): ?([0-9.e-]+)([ nuµms]*)"

CUSTOM_RE = {}


def serialize_simengines(res: list[str], prefix: str, flavors: str | dict):
    if isinstance(flavors, str):
        res.append(prefix)
    else:
        for key, val in flavors.items():
            new_prefix = f'{prefix}-{key}' if prefix else key
            serialize_simengines(res, new_prefix, val)


def parse_args():
    parser = argparse.ArgumentParser(description='Script to compare DEVStone implementations with different engines')
    parser.add_argument('-m', '--model-types', help='DEVStone model type (LI, HI, HO, HOmod)')
    parser.add_argument('-w', '--width', type=int, help='Width of each coupled model.')
    parser.add_argument('-d', '--depth', type=int, help='Number of recursive levels of the model.')
    parser.add_argument('-i', '--int-cycles', type=int, help='Dhrystone cycles executed in internal transitions')
    parser.add_argument('-e', '--ext-cycles', type=int, help='Dhrystone cycles executed in external transitions')
    parser.add_argument('-n', '--num-rep', type=int, help='Number of repetitions per each engine and configuration')
    parser.add_argument('-a', '--include_engines', help='Add specific engines to perform the comparative')
    parser.add_argument('-r', '--exclude_engines', help='Exclude specific engines from the comparative')
    parser.add_argument('-o', '--out-file', help='Output file path')
    parser.add_argument('-p', '--params', help='Specify params in a condensed form: w1-d1-ic1-ec1, w2-d2-ic2-ec2...')

    args = parser.parse_args()

    if args.model_types:
        args.model_types = [x.strip() for x in args.model_types.split(",")]
    else:
        args.model_types = list(DEFAULT_MODEL_TYPES)

    if args.params:
        params = [x.strip().split("-") for x in args.params.split(",")]
        # args.params = [x.strip().split("-") for x in args.params.split(",")]
    elif args.depth and args.width:
        int_cycles = args.int_cycles or 0
        ext_cycles = args.ext_cycles or 0
        params = [(args.width, args.depth, int_cycles, ext_cycles)]
        # args.params = ((args.depth, args.width, int_cycles, ext_cycles),)
    else:
        params = list(DEFAULT_PARAMS)
    args.params = []
    for param in params:
        if len(param) == 5:
            args.params.append(param)
        elif len(param) == 4:
            for model in args.model_types:
                args.params.append((model, *param))
        else:
            raise ValueError(f'invalid number of params ({param})')

    engines = args.include_engines.split(',') if args.include_engines else list(COMMANDS.keys())
    args.include_engines = []
    for eng in engines:
        commands = COMMANDS
        for s in eng.split('-'):
            commands = commands.get(s, dict())
        subengines = []
        serialize_simengines(subengines, '', commands)
        if not subengines:
            raise RuntimeError(f'unknown simulation engine {eng}')
        subengines = [f'{eng}-{x}' if x else f'{eng}' for x in subengines]
        args.include_engines.extend(subengines)

    if args.exclude_engines:
        args.include_engines = [x for x in args.include_engines if x not in args.exclude_engines.split(",")]

    args.prelaunch = {}
    args.commands = {}
    args.regex = {}
    for eng in args.include_engines:
        prelaunch = PRELAUNCH_TASKS
        cmd = COMMANDS
        regex = CUSTOM_RE
        for s in eng.split('-'):
            if isinstance(prelaunch, dict):
                prelaunch = prelaunch.get(s, dict())
            cmd = cmd.get(s, dict())
            if not isinstance(regex, str):
                regex = regex.get(s, DEFAULT_RE)
        if isinstance(prelaunch, dict):
            prelaunch = None
        if not cmd:
            raise RuntimeError(f'unknown simulation engine {eng}')
        if isinstance(cmd, dict):
            # TODO instead of this, serialize all the flavors of the engine
            raise RuntimeError(f'There are more than one flavor for the engine {eng}')
        else:
            args.prelaunch[eng] = prelaunch
            args.commands[eng] = cmd
            args.regex[eng] = regex

    if not args.num_rep:
        args.num_rep = DEFAULT_NUM_REPS

    if not args.out_file:
        args.out_file = "devstone_%de_%dp_%d.csv" % (len(args.include_engines), len(args.params), int(time.time()))

    return args


def execute_prelaunch(cmd):
    try:
        result = subprocess.run(cmd.split(), stdout=subprocess.PIPE)
    except Exception as e:
        print(f"Error executing prelaunch command. ({str(e)})")
        return
    print(result.stdout)


def execute_cmd(cmd, regex, csv_writer):
    # Execute simulation
    try:
        result = subprocess.run(cmd.split(), stdout=subprocess.PIPE)
    except Exception as e:
        print(f"{engine}: Error executing simulation. ({str(e)})")
        return

    # Read data from output
    res_out = result.stdout.decode('UTF-8').strip()
    if not res_out:
        res_out = str(result.stderr).strip()
    res_out.replace("\n", " ")
    found = re.search(regex, res_out, flags=re.IGNORECASE | re.DOTALL)
    if not found:
        print(f"{engine}: Simulation execution times could not be extracted.")
        print(result.stdout)
        return
    groups = found.groups()
    if len(groups) == 3:
        model_time, engine_time, sim_time = tuple(map(float, found.groups()))
    elif len(groups) == 6:
        times = list(map(float, (groups[0], groups[2], groups[4])))
        units = (groups[1], groups[3], groups[5])
        for i in range(3):
            if units[i]:
                unit = units[i].strip()
                if unit.startswith('s'):
                    pass
                elif unit.startswith('ms'):
                    times[i] *= 1e-3
                elif unit.startswith('us') or unit.startswith('µs'):
                    times[i] *= 1e-6
                elif unit.startswith('ns'):
                    times[i] *= 1e-9
                else:
                    raise Exception(f"unknown time units: {unit}")
        times = tuple(times)
        model_time, engine_time, sim_time = times
    total_time = sum((model_time, engine_time, sim_time))

    # Write results into output file
    row = (engine, i_exec, model_type, width, depth, int_cycles, ext_cycles, model_time, engine_time, sim_time, total_time)
    csv_writer.writerow(row)


if __name__ == "__main__":
    sys.setrecursionlimit(10000)

    args = parse_args()

    if not args.include_engines:
        raise RuntimeError("No engines were selected.")

    with open(args.out_file, "w") as csv_file:
        csv_writer = csv.writer(csv_file, delimiter=';')
        csv_writer.writerow(("engine", "iter", "model", "width", "depth", "int_delay", "ext_delay", "model_time", "runner_time", "sim_time", "total_time"))

        for engine, engine_cmd in args.commands.items():
            engine_prelaunch = args.prelaunch[engine]
            if engine_prelaunch is not None:
                print(f'({engine}) executing prelaunch tasks...')
                for prelaunch_cmd in engine_prelaunch:
                    execute_prelaunch(prelaunch_cmd)
            engine_regex = args.regex[engine]
            for params in args.params:
                model_type, width, depth, int_cycles, ext_cycles = params
                engine_cmd_f = engine_cmd.format(model_type=model_type, width=width, depth=depth,
                                                 int_cycles=int_cycles, ext_cycles=ext_cycles)
                if not engine_cmd_f:
                    continue
                for i_exec in range(args.num_rep):
                    print(f'({i_exec}) {engine_cmd_f}')
                    execute_cmd(engine_cmd_f, engine_regex, csv_writer)
                csv_file.flush()