Add evaluation script, supporting pocr, pearl, gigascale, graspan

IlyaMuravjov · IlyaMuravjov · commit 6133a6761756 · 2024-03-03T00:14:33.000+03:00
diff --git a/cli/all_pairs_cflr_command_manager.py b/cli/all_pairs_cflr_command_manager.py
@@ -0,0 +1,195 @@
+import os
+import re
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Optional, Tuple
+
+import psutil
+
+from src.grammar.cnf_grammar_template import CnfGrammarTemplate
+from src.graph.label_decomposed_graph import LabelDecomposedGraph
+from src.problems.Base.template_cfg.utils import explode_indices
+
+
+def get_all_pairs_cflr_command_manager(
+        algo_settings: str,
+        graph_path: Path,
+        grammar_path: Path
+) -> "AllPairsCflrCommandManager":
+    return {
+        "pocr": PocrAllPairsCflrCommandManager,
+        "pearl": PearlAllPairsCflrCommandManager,
+        "gigascale": GigascaleAllPairsCflrCommandManager,
+        "graspan": GraspanAllPairsCflrCommandManager
+    }.get(algo_settings, PyAlgoAllPairsCflrCommandManager)(
+        algo_settings, graph_path, grammar_path
+    )
+
+
+class AllPairsCflrCommandManager(ABC):
+    def __init__(
+            self,
+            algo_settings: str,
+            graph_path: Path,
+            grammar_path: Path
+    ):
+        self.algo_settings = algo_settings
+        self.graph_path = graph_path
+        self.grammar_path = grammar_path
+
+    @abstractmethod
+    def create_command(self) -> str:
+        pass
+
+    # noinspection PyMethodMayBeStatic
+    def discard_stderr(self) -> bool:
+        return False
+
+    @property
+    def work_dir(self) -> Optional[Path]:
+        return None
+
+    # noinspection PyMethodMayBeStatic
+    def get_analysis_time(self, output: str) -> float:
+        return float(re.search(r"AnalysisTime\s+([\d.]+|NaN)", output).group(1))
+
+    # noinspection PyMethodMayBeStatic
+    def get_edge_count(self, output: str) -> int:
+        return re.search(r"#(SEdges|CountEdges)\s+([\d.]+|NaN)", output).group(2)
+
+
+class PyAlgoAllPairsCflrCommandManager(AllPairsCflrCommandManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def create_command(self) -> Optional[str]:
+        return f"python3 -m cli.run_all_pairs_cflr {self.algo_settings} {self.graph_path} {self.grammar_path}"
+
+
+class PocrAllPairsCflrCommandManager(AllPairsCflrCommandManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def create_command(self) -> Optional[str]:
+        return (
+            f'{self.grammar_path.stem} -pocr {self.graph_path}'
+            if self.grammar_path.stem in {"aa", "vf"}
+            else f'cfl -pocr {self.grammar_path} {self.graph_path}'
+        )
+
+
+class PearlAllPairsCflrCommandManager(AllPairsCflrCommandManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def create_command(self) -> Optional[str]:
+        return (
+            f'./{self.grammar_path.stem} {self.graph_path} -pearl -scc=false -gf=false'
+            if self.grammar_path.stem in {"aa", "vf"}
+            else None
+        )
+
+    @property
+    def work_dir(self) -> Optional[Path]:
+        return Path(os.environ['PEARL_DIR'])
+
+    def get_edge_count(self, output: str) -> int:
+        vedges_search = re.search(r"#VEdges\s+(\d+)", output)
+        if vedges_search:
+            return vedges_search.group(1)
+        return re.search(r"#AEdges\s+(\d+)", output).group(1)
+
+
+class GigascaleAllPairsCflrCommandManager(AllPairsCflrCommandManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def create_command(self) -> Optional[str]:
+        return (
+            f'./run.sh -wdlrb -i datasets/dacapo9/{self.graph_path.stem}'
+            if self.grammar_path.stem in {"java_points_to"}
+            else None
+        )
+
+    @property
+    def work_dir(self) -> Optional[Path]:
+        return Path(os.environ['GIGASCALE_DIR'])
+
+    # Gigascale sends [INFO] logs to stderr
+    def discard_stderr(self) -> bool:
+        return True
+
+    def get_analysis_time(self, output: str) -> float:
+        return self._get_analysis_time_and_edge_count(output)[0]
+
+    def get_edge_count(self, output: str) -> int:
+        return self._get_analysis_time_and_edge_count(output)[1]
+
+    @staticmethod
+    def _get_analysis_time_and_edge_count(output: str) -> Tuple[float, int]:
+        pattern = (r"benchmark\s+TC-time\s+TC-mem\s+v\s+e\s+vpt\s+avg\s+max\s+load/f\s+store/f\s*\n"
+                   r"\w+\s+"
+                   r"(\d+\.\d+)\s+"
+                   r"\d+(?:\.\d+)?\s+"
+                   r"\d+\s+"
+                   r"\d+\s+"
+                   r"(\d+)\s+"
+                   r"\d+(?:\.\d+)?\s+"
+                   r"\d+\s+"
+                   r"\d+\s+"
+                   r"\d+")
+
+        match = re.search(pattern, output)
+
+        tc_time, vpt = match.groups()
+        return float(tc_time), int(vpt)
+
+
+class GraspanAllPairsCflrCommandManager(AllPairsCflrCommandManager):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def create_command(self) -> Optional[str]:
+        grammar = CnfGrammarTemplate.read_from_pocr_cnf_file(self.grammar_path)
+        graph = LabelDecomposedGraph.read_from_pocr_graph_file(self.graph_path)
+
+        # Graspan doesn't support indexed symbols, we need to concat labels and indices
+        if graph.block_matrix_space.block_count > 1:
+            graph, grammar = explode_indices(graph, grammar)
+            graph_path = self.graph_path.parent / "graspan" / self.graph_path.name
+            os.makedirs(graph_path.parent, exist_ok=True)
+            graph.write_to_pocr_graph_file(graph_path)
+        else:
+            graph_path = self.graph_path
+
+        # Graspan doesn't support grammars with over 255 symbols, because
+        # each symbol is encoded with one byte and one symbol is reserved for epsilon
+        if len(grammar.symbols) > 255:
+            return None
+
+        grammar_path = self.grammar_path.parent / "graspan" / self.grammar_path.name
+        os.makedirs(grammar_path.parent, exist_ok=True)
+        grammar.write_to_pocr_cnf_file(grammar_path, include_starting=False)
+
+        return (
+            f'./run {graph_path} {grammar_path} 1 '
+            f'{int(psutil.virtual_memory().total / 10**9 * 0.9)} '
+            f'{os.cpu_count() * 2}'
+        )
+
+    @property
+    def work_dir(self) -> Optional[Path]:
+        return Path(os.environ['GRASPAN_DIR']) / "src"
+
+    def get_analysis_time(self, output: str) -> float:
+        return float(re.search(r"COMP TIME:\s*([\d.]+|NaN)", output).group(1))
+
+    def get_edge_count(self, output: str) -> int:
+        final_file = re.search(r"finalFile:\s*(.*)", output).group(1)
+        start_nonterm = CnfGrammarTemplate.read_from_pocr_cnf_file(self.grammar_path).start_nonterm
+        with open(final_file, "r") as file:
+            edges = set()
+            for line in file:
+                if line.split()[-1] == start_nonterm.label:
+                    edges.add((line.split()[0], line.split()[1]))
+        return len(edges)
diff --git a/cli/eval_all_pairs_cflr.py b/cli/eval_all_pairs_cflr.py
@@ -0,0 +1,176 @@
+import argparse
+import csv
+import os
+import shlex
+import signal
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional, List
+
+from cli.all_pairs_cflr_command_manager import get_all_pairs_cflr_command_manager
+
+# see `man timeout`
+TIMEOUT_EXIT_CODE = 124
+
+
+def check_file_for_completion(result_file_path, rounds):
+    try:
+        with open(result_file_path, 'r') as file:
+            reader = list(csv.reader(file))
+            if len(reader) - 1 >= rounds or any("OOT" in row or "OOM" in row for row in reader):
+                return True
+    except FileNotFoundError:
+        pass
+    return False
+
+
+def run_experiment(
+    algo_settings: str,
+    algo_name: str,
+    graph_path: Path,
+    grammar_path: Path,
+    rounds: int,
+    timeout: Optional[int],
+    result_file_path: Path
+):
+    graph_base_name = graph_path.stem
+    grammar_base_name = grammar_path.stem
+
+    if not os.path.exists(result_file_path):
+        with open(result_file_path, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["algo", "graph", "grammar", "edge_count", "ram_kb", "time_sec"])
+
+    if "--rewrite-grammar" in algo_settings:
+        algo_settings = algo_settings.replace("--rewrite-grammar", "")
+        rewritten_grammar_path = grammar_path.with_stem(grammar_path.stem + "_rewritten")
+        if os.path.exists(rewritten_grammar_path):
+            grammar_path = rewritten_grammar_path
+
+    for _ in range(rounds):
+        if check_file_for_completion(result_file_path, rounds):
+            return
+
+        command_manager = get_all_pairs_cflr_command_manager(algo_settings, graph_path, grammar_path)
+
+        temp_ram_file = Path("temp_ram_usage.txt").absolute()
+
+        base_command = command_manager.create_command()
+
+        if base_command is None:
+            edge_count, ram_kb, time_sec = "-", "-", "-"
+        else:
+            command = (f"/usr/bin/time -o {temp_ram_file} -f %M "
+                       + ("" if timeout is None else f"timeout {timeout}s ")
+                       + base_command)
+
+            process = subprocess.Popen(
+                shlex.split(command),
+                cwd=command_manager.work_dir,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL if command_manager.discard_stderr() else None
+            )
+            try:
+                output, _ = process.communicate()
+            except KeyboardInterrupt:
+                process.send_signal(signal.SIGINT)
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                raise
+            if process.returncode == 0:
+                output = output.decode()
+                time_sec = command_manager.get_analysis_time(output)
+                edge_count = command_manager.get_edge_count(output)
+                with open(temp_ram_file, 'r') as f:
+                    ram_kb = f.read().strip()
+            elif process.returncode == TIMEOUT_EXIT_CODE:
+                print("    Runner process timed out")
+                edge_count, ram_kb, time_sec = "OOT", "OOT", "OOT"
+            else:
+                print(
+                    f"   Runner process terminated with return code {process.returncode}\n"
+                    f"   (interpreting as out of memory error)"
+                )
+                edge_count, ram_kb, time_sec = "OOM", "OOM", "OOM"
+
+        with open(result_file_path, 'a', newline='') as csvfile:
+            print(f"   {edge_count} {ram_kb} {time_sec}")
+            writer = csv.writer(csvfile)
+            writer.writerow([
+                {algo_name},
+                os.path.basename(graph_base_name),
+                os.path.basename(grammar_base_name),
+                edge_count,
+                ram_kb,
+                time_sec
+            ])
+
+
+def eval_all_pairs_cflr(
+    algo_config: Path,
+    data_config: Path,
+    result_path: Path,
+    rounds: Optional[int],
+    timeout: Optional[int],
+):
+    with open(algo_config, mode='r') as algo_file:
+        algo_reader = csv.DictReader(algo_file)
+        for algo_row in algo_reader:
+            algo_name = algo_row['algo_name']
+            print(f"Running algorithm: {algo_name}")
+            algo_settings = algo_row['algo_settings']
+            algo_result_path = os.path.join(result_path, algo_name)
+            if not os.path.exists(algo_result_path):
+                os.makedirs(algo_result_path)
+
+            with open(data_config, mode='r') as data_file:
+                data_reader = csv.DictReader(data_file)
+                for data_row in data_reader:
+                    graph_path = Path(data_row['graph_path']).absolute()
+                    grammar_path = Path(data_row['grammar_path']).absolute()
+                    print(f"  Processing data: {graph_path.stem}, {grammar_path.stem}")
+                    result_file_name = f"{graph_path.stem}_{grammar_path.stem}.csv"
+                    result_file_path = Path(os.path.join(algo_result_path, result_file_name))
+
+                    run_experiment(
+                        algo_settings=algo_settings,
+                        algo_name=algo_name,
+                        graph_path=graph_path,
+                        grammar_path=grammar_path,
+                        rounds=rounds,
+                        timeout=timeout,
+                        result_file_path=result_file_path
+                    )
+
+
+def main(raw_args: List[str]):
+    parser = argparse.ArgumentParser(
+        description='Evaluates all vertex pairs Context-Free Language Reachability (CFL-R) algorithms.'
+    )
+
+    parser.add_argument('algo_config', type=str,
+                        help='Path to the algo-config csv file.')
+    parser.add_argument('data_config', type=str,
+                        help='Path to the data-config csv file.')
+    parser.add_argument('result_path', type=str,
+                        help='Path to save the results.')
+    parser.add_argument('--rounds', type=int, default=1,
+                        help='Number of rounds to run each configuration.')
+    parser.add_argument('--timeout', type=int, default=None,
+                        help='Timeout for each run in seconds.')
+
+    args = parser.parse_args(raw_args)
+    eval_all_pairs_cflr(
+        algo_config=Path(args.algo_config),
+        data_config=Path(args.data_config),
+        result_path=Path(args.result_path),
+        rounds=args.rounds,
+        timeout=args.timeout
+    )
+
+
+if __name__ == "__main__":       # pragma: no cover
+    main(raw_args=sys.argv[1:])  # pragma: no cover
diff --git a/cli/run_all_pairs_cflr.py b/cli/run_all_pairs_cflr.py
@@ -4,15 +4,15 @@
 from time import time
 from typing import Optional, List
 
-from src.algo_setting.preprocessor_setting import preprocess_graph_and_grammar
-from src.utils.time_limit import time_limit, TimeoutException
 from src.algo_setting.algo_setting import AlgoSetting
 from src.algo_setting.algo_settings_manager import AlgoSettingsManager
+from src.algo_setting.preprocessor_setting import preprocess_graph_and_grammar
 from src.grammar.cnf_grammar_template import CnfGrammarTemplate
 from src.graph.label_decomposed_graph import LabelDecomposedGraph
 from src.problems.Base.template_cfg.template_cfg_all_pairs_reachability_impls import \
     ALL_PAIRS_CFL_REACHABILITY_ALGO_NAMES, \
     get_all_pairs_cfl_reachability_algo
+from src.utils.time_limit import time_limit, TimeoutException
 
 
 def run_all_pairs_cflr(
diff --git a/requirements.txt b/requirements.txt
@@ -7,3 +7,4 @@ suitesparse-graphblas==7.4.4.0
 python-graphblas==2023.7.0
 pandas==2.0.3
 numpy==1.23.5
+psutil==5.9.8
diff --git a/src/grammar/cnf_grammar_template.py b/src/grammar/cnf_grammar_template.py
diff --git a/src/graph/label_decomposed_graph.py b/src/graph/label_decomposed_graph.py
