From 41282a7518e11ebd25d7d08382b828fea93bf106 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:28:41 +0100 Subject: [PATCH] use a different way of testing the plugins --- .github/workflows/plugin_test.yaml | 67 +++------- scripts/plugin_test.py | 179 ++++++++++++++++++++++++++ tests/nomad/test_pynxtools_plugins.py | 45 +++++++ 3 files changed, 240 insertions(+), 51 deletions(-) create mode 100644 scripts/plugin_test.py create mode 100644 tests/nomad/test_pynxtools_plugins.py diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index 6b1b2e459..1c0990378 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -1,6 +1,5 @@ -# This workflow will install Python 3.9 and run the tests of all supported plugins. +# This workflow will install Python 3.11 and run the tests of all supported plugins. # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: test plugins on: @@ -19,66 +18,32 @@ jobs: strategy: fail-fast: false matrix: - include: - - - plugin: pynxtools-ellips - branch: main - tests_to_run: tests/. - - plugin: pynxtools-raman - branch: main - tests_to_run: tests/. - - plugin: pynxtools-mpes - branch: main - tests_to_run: tests/. - - plugin: pynxtools-stm - branch: main - tests_to_run: tests/. - - plugin: pynxtools-xps - branch: main - tests_to_run: tests/. - - plugin: pynxtools-xrd - branch: main - tests_to_run: tests/. - # - plugin: pynxtools-apm - # branch: main - # tests_to_run: tests/. - # - plugin: pynxtools-xrd - # branch: update-tests - # tests_to_run: tests/. - # - plugin: pynxtools-em - # branch: main - # tests_to_run: tests/. + plugin: + - pynxtools-ellips + - pynxtools-raman + - pynxtools-mpes + - pynxtools-stm + - pynxtools-xps + - pynxtools-xrd steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive - - name: Set up Python 3.9 + submodules: recursive + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.11 - name: Install dependencies run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv pip install --system coverage coveralls - - name: Install package + uv pip install coverage coveralls + - name: Install pynxtools run: | uv pip install ".[dev]" - - name: Clone ${{ matrix.plugin }} repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - repository: FAIRmat-NFDI/${{ matrix.plugin }} - path: ${{ matrix.plugin }} - ref: ${{ matrix.branch }} - name: Install nomad run: | uv pip install nomad-lab@git+https://gitlab.mpcdf.mpg.de/nomad-lab/nomad-FAIR.git - - name: Install ${{ matrix.plugin }} - run: | - cd ${{ matrix.plugin }} - uv pip install . - - name: Run ${{ matrix.plugin }} tests + - name: Run plugin tests run: | - cd ${{ matrix.plugin }} - pytest ${{ matrix.tests_to_run }} \ No newline at end of file + python scripts/plugin_test.py --plugin ${{ matrix.plugin }} diff --git a/scripts/plugin_test.py b/scripts/plugin_test.py new file mode 100644 index 000000000..2bfc92c24 --- /dev/null +++ b/scripts/plugin_test.py @@ -0,0 +1,179 @@ +import os +import subprocess +import sys +import argparse +import logging +from typing import List, Dict, Optional + + +# Set up logging configuration +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) +stream_handler = logging.StreamHandler() +logger.addHandler(stream_handler) + +# Plugin configuration list with plugin details (name, branch, tests directory) +PLUGIN_CONFIGS: List[Dict[str, str]] = [ + {"plugin": "pynxtools-apm", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-ellips", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-em", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-raman", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-mpes", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-stm", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-xps", "branch": "main", "tests_to_run": "tests/."}, + {"plugin": "pynxtools-xrd", "branch": "main", "tests_to_run": "tests/."}, +] + + +def get_tests_to_run(tests_to_run_default: str) -> str: + """ + Determine which tests should be run based on the IN_NOMAD_DISTRO environment variable. + + If IN_NOMAD_DISTRO is set to "true", only tests in the "tests/nomad" directory will be run. + Otherwise, the default test directory provided in the configuration is used. + + Args: + tests_to_run_default (str): The default directory of tests to run if IN_NOMAD_DISTRO is not set to "true". + + Returns: + str: The directory of tests to run. + """ + if os.getenv("IN_NOMAD_DISTRO", "false").lower() == "true": + return "tests/nomad" + return tests_to_run_default + + +def run_command( + command: List[str], cwd: Optional[str] = None +) -> Optional[subprocess.CompletedProcess]: + """ + Run a command using subprocess and capture its output. + + Args: + command (List[str]): The command to run, given as a list of arguments. + cwd (Optional[str]): The directory in which to run the command. Defaults to None (current directory). + + Returns: + Optional[subprocess.CompletedProcess]: The result of the subprocess run command. + """ + try: + result = subprocess.run( + command, cwd=cwd, capture_output=True, text=True, check=False + ) + + if result.stdout: + logger.info(result.stdout) + if result.stderr: + logger.error(result.stderr) + + return result + except Exception as e: + logger.error(f"Error running command {command}: {e}") + return None + + +def clone_and_checkout(plugin: str, branch: str) -> None: + """ + Clone a repository and checkout the specified branch. + + Args: + plugin (str): The name of the plugin repository to clone. + branch (str): The branch to checkout after cloning the repository. + """ + logger.info(f"Setting up tests for {plugin} (branch: {branch})") + clone_command = [ + "git", + "clone", + f"https://github.com/FAIRmat-NFDI/{plugin}.git", + "--branch", + branch, + "--depth", + "1", + ] + result = run_command(clone_command) + + if result is None or result.returncode != 0: + logger.error(f"Failed to clone repository for '{plugin}' (branch '{branch}').") + return + + logger.info(f"Repository cloned successfully for '{plugin}' (branch '{branch}').") + + +def install_local_plugin(plugin: str) -> None: + """ + Install the plugin using pip. + + Args: + plugin (str): The name of the plugin to install. + """ + logger.info(f"Install local {plugin}") + install_command = ["uv", "pip", "install", f"./{plugin}"] + result = run_command(install_command) + + if result is None or result.returncode != 0: + logger.error(f"Failed to install repository '{plugin}'.") + return + + logger.info(f"Repository installed successfully for '{plugin}'.") + + +def run_tests(plugin: str, tests_to_run: str) -> None: + """ + Run the tests for a specified plugin. + + Args: + plugin (str): The plugin for which to run tests. + tests_to_run (str): The directory containing the tests to run. + """ + pytest_command = ["pytest", f"./{plugin}/{tests_to_run}"] + + result = run_command(pytest_command) + + if result is None or result.returncode != 0: + logger.error(f"Error running tests for {plugin}: {plugin}") + sys.exit(1) + + logger.info(f"Successfully ran tests for '{plugin}'.") + + +if __name__ == "__main__": + # Set up command line argument parsing + parser = argparse.ArgumentParser(description="Run plugin tests.") + parser.add_argument("--plugin", help="Specific plugin to test") + + args = parser.parse_args() + + # Determine which plugins to test based on the arguments passed + if args.plugin: + plugins_to_test = [ + cfg for cfg in PLUGIN_CONFIGS if cfg["plugin"] == args.plugin + ] + else: + plugins_to_test = ( + PLUGIN_CONFIGS # Run all plugins if no specific plugin is provided + ) + + # If no plugins are specified, exit with an error + if not plugins_to_test: + logger.error("No plugins specified or found for testing.") + sys.exit(1) + + # Iterate through the selected plugins and run the tests + for config in plugins_to_test: + plugin, branch, tests_to_run_default = ( + config["plugin"], + config["branch"], + config["tests_to_run"], + ) + tests_to_run = get_tests_to_run(tests_to_run_default=tests_to_run_default) + # Clone the repository, checkout the branch, and + clone_and_checkout(plugin, branch) + + # If IN_NOMAD_DISTRO is false, install the plugin locally + if os.getenv("IN_NOMAD_DISTRO", "false").lower() == "false": + install_local_plugin(plugin) + + # run the plugin's tests + run_tests(plugin, tests_to_run) diff --git a/tests/nomad/test_pynxtools_plugins.py b/tests/nomad/test_pynxtools_plugins.py new file mode 100644 index 000000000..e122e7f9b --- /dev/null +++ b/tests/nomad/test_pynxtools_plugins.py @@ -0,0 +1,45 @@ +import os +import subprocess +import pytest + + +@pytest.mark.parametrize( + "plugin_name", + [ + # pytest.param("pynxtools-apm"), + # pytest.param("pynxtools-ellips"), + # pytest.param("pynxtools-em"), + pytest.param("pynxtools-mpes"), + # pytes.param("pynxtools-raman"), + # pytes.param("pynxtools-stm"), + # pytest.param("pynxtools-xps"), + # pytest.param("pynxtools-xrd"), + ], +) +def test_plugins(plugin_name): + """ + Test the execution of plugin tests for each given plugin. This simulates + the environment where the IN_NOMAD_DISTRO environment variable is set to "true". In this + mode, only tests in the "tests/nomad" directory are executed for the selected plugin. + + The function performs the following steps: + 1. Sets the IN_NOMAD_DISTRO environment variable to "true" to simulate the desired environment. + 2. Executes the `plugin_test.py` script using the subprocess module, passing the plugin name + through the `--plugin` flag to run tests for the specified plugin. + 3. Asserts that the subprocess completes successfully (i.e., the return code is 0). + 4. Captures the output from the subprocess and checks that there are no errors in the + + """ + # Set the IN_NOMAD_DISTRO environment variable to True for sweep + os.environ["IN_NOMAD_DISTRO"] = "true" + + # Run the main plugin_test.py script with the --sweep argument + result = subprocess.run( + ["python", "scripts/plugin_test.py", "--plugin", plugin_name], + capture_output=True, + text=True, + check=False, + ) + + # Assert that the subprocess ran successfully + assert result.returncode == 0, f"Tests failed. Output: {result.stderr}"