Add workflow tests (#256)

* Fix mapping of in-outputs

* General workflow tests

* Add geometry optimization test

* Add molecular dynamics test

* Add phonon test

* Revert removed utils test

* Move log_output fixture

* Add single point test

* Add GW tests

* Add elastic test

* Make labels private

* Add dmft test

* Move coveralls actions, add nomad infra

* Try only python 3.12

* Update pyproject.toml

* Add beyond_dft tests

* Extend beyond_dft test

* Add equation_of_state test

* Add max_ent test

* Add photon_polarization test

* Add tb test

* Add thermodyanamics test

* Add xs test

Author: ladinesa

.github/workflows/actions.yml

@@ -6,12 +6,17 @@ on: [push]
 # `pull-requests` is for permission to pull request
 permissions:
   contents: write
+  packages: write
+  attestations: write
+  id-token: write
   checks: write
   pull-requests: write
 
 env:
   UV_SYSTEM_PYTHON: true
+  REGISTRY: ghcr.io
   UV_VERSION: 0.7
+
 jobs:
   install-and-test:
     runs-on: ubuntu-latest
@@ -21,6 +26,16 @@ jobs:
         python_version: ["3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
+
+    - uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Start containers
+      run: |
+        docker compose up -d --quiet-pull
     - name: Set up Python ${{matrix.python_version}}
       uses: actions/setup-python@v5
       with:

docker-compose.yml

@@ -0,0 +1,93 @@
+name: nomad-simulations-test
+services:
+  # broker for celery
+  rabbitmq:
+    restart: unless-stopped
+    image: rabbitmq:3.11.5
+    container_name: rabbitmq
+    environment:
+      - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
+      - RABBITMQ_DEFAULT_USER=rabbitmq
+      - RABBITMQ_DEFAULT_PASS=rabbitmq
+      - RABBITMQ_DEFAULT_VHOST=/
+    ports:
+      - "5672:5672"
+    volumes:
+      - rabbitmq:/var/lib/rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  # the search engine
+  elastic:
+    restart: unless-stopped
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.24
+    container_name: elastic
+    environment:
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - discovery.type=single-node
+    ports:
+      - "9200:9200"
+    volumes:
+      - elastic:/usr/share/elasticsearch/data
+    healthcheck:
+      test:
+        - "CMD"
+        - "curl"
+        - "--fail"
+        - "--silent"
+        - "http://elastic:9200/_cat/health"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 60s
+
+  # the user data db
+  mongo:
+    restart: unless-stopped
+    image: mongo:5.0.6
+    container_name: mongo
+    environment:
+      - MONGO_DATA_DIR=/data/db
+      - MONGO_LOG_DIR=/dev/null
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo:/data/db
+      - ./.volumes/mongo:/backup
+    command: mongod --logpath=/dev/null # --quiet
+    healthcheck:
+      test:
+        - "CMD"
+        - "mongo"
+        - "mongo:27017/test"
+        - "--quiet"
+        - "--eval"
+        - "'db.runCommand({ping:1}).ok'"
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 10s
+
+  keycloak:
+    image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-test-keycloak:v1.0
+    platform: linux/amd64
+    container_name: keycloak
+    environment:
+      - KC_HTTP_PORT=8008
+      - KC_HOSTNAME=http://localhost:8008
+    ports:
+      - 8008:8008
+      - 9000:9000
+
+volumes:
+  mongo:
+    name: "mongo"
+  elastic:
+    name: "elastic"
+  rabbitmq:
+    name: "rabbitmq"
+

nomad.yaml

@@ -0,0 +1,12 @@
+keycloak:
+  server_url: 'http://localhost:8008/'
+  public_server_url: 'http://localhost:8008/'
+  realm_name: 'fairdi_nomad_test'
+  username: 'admin'
+  password: 'password'
+client:
+  url: 'http://localhost:8000/fairdi/nomad/latest/api'
+  user: 'admin'
+  password: 'password'
+fs:
+  staging: '/tmp/volumes/fs/staging'

src/nomad_simulations/schema_packages/workflow/beyond_dft.py

@@ -16,15 +16,15 @@
 
 
 class BeyondDFTModel(SimulationWorkflowModel):
-    label = 'DFT+ workflow parameters'
+    _label = 'DFT+ workflow parameters'
 
 
 class BeyondDFTResults(SimulationWorkflowResults):
     """
     Contains reference to DFT outputs.
     """
 
-    label = 'DFT+ workflow results'
+    _label = 'DFT+ workflow results'
 
     dft = SubSection(sub_section=ElectronicStructureResults)
 

src/nomad_simulations/schema_packages/workflow/dmft.py

@@ -10,11 +10,11 @@
 
 
 class DFTTBDDMFTModel(BeyondDFTModel):
-    label = 'DMFT+MaxEnt workflow parameters'
+    _label = 'DFT+TB+DMFT workflow parameters'
 
 
 class DFTTBDMFTResults(BeyondDFTResults):
-    label = 'DMFT+MaxEnt workflow results'
+    _label = 'DFT+TB+DMFT workflow results'
 
 
 class DFTTBDMFTWorkflow(BeyondDFTWorkflow):
@@ -43,8 +43,11 @@ def normalize(self, archive: EntryArchive, logger: BoundLogger) -> None:
 
         super().normalize(archive, logger)
 
-        if self.task and not self.task[-1].name:
-            self.task[-1].name = 'DMFT'
+        if len(self.tasks) == 3 and not self.tasks[1].name:
+            self.tasks[1].name = 'TB'
+
+        if self.tasks and not self.tasks[-1].name:
+            self.tasks[-1].name = 'DMFT'
 
 
 m_package.__init_metainfo__()

src/nomad_simulations/schema_packages/workflow/elastic.py

@@ -13,6 +13,7 @@
 from structlog.stdlib import BoundLogger
 
 from nomad_simulations.schema_packages.general import Program
+from nomad_simulations.schema_packages.utils import log
 
 from .general import INCORRECT_N_TASKS, SimulationWorkflow, SimulationWorkflowModel
 from .thermodynamics import ThermodynamicsResults
@@ -21,6 +22,8 @@
 
 
 class ElasticModel(SimulationWorkflowModel):
+    _label = 'Elastic model'
+
     program = Quantity(
         type=Reference(Program),
         shape=[],
@@ -129,6 +132,8 @@ class StrainDiagrams(ArchiveSection):
 
 
 class ElasticResults(ThermodynamicsResults):
+    _label = 'Elastic results'
+
     n_deformations = Quantity(
         type=np.int32,
         shape=[],
@@ -382,17 +387,21 @@ class Elastic(SimulationWorkflow):
     Definitions for an elastic workflow.
     """
 
-    task_label = 'Deformation'
+    _task_label = 'Deformation'
 
-    def map_inputs(self, archive: EntryArchive, logger: BoundLogger) -> None:
+    @log
+    def map_inputs(self, archive: EntryArchive) -> None:
         if not self.model:
             self.model = ElasticModel()
-        super().map_inputs(archive, logger)
+        logger = self.map_inputs.__annotations__['logger']
+        super().map_inputs(archive, logger=logger)
 
-    def map_outputs(self, archive: EntryArchive, logger: BoundLogger) -> None:
+    @log
+    def map_outputs(self, archive: EntryArchive) -> None:
         if not self.results:
             self.results = ElasticResults()
-        super().map_outputs(archive, logger)
+        logger = self.map_outputs.__annotations__['logger']
+        super().map_outputs(archive, logger=logger)
 
     def normalize(self, archive: EntryArchive, logger: BoundLogger) -> None:
         super().normalize(archive, logger)

src/nomad_simulations/schema_packages/workflow/equation_of_state.py

@@ -36,6 +36,8 @@
 
 
 class EquationOfStateModel(SimulationWorkflowModel):
+    _label = 'EquationOfState workflow parameters'
+
     program = Quantity(
         type=Reference(Program),
         shape=[],
@@ -123,6 +125,8 @@ class EOSFit(ArchiveSection):
 
 
 class EquationOfStateResults(SimulationWorkflowResults):
+    _label = 'EquationOfState workflow results'
+
     n_points = Quantity(
         type=np.int32,
         shape=[],
@@ -153,41 +157,48 @@ class EquationOfStateResults(SimulationWorkflowResults):
 
     def normalize(self, archive: EntryArchive, logger: BoundLogger) -> None:
         if self.n_points is None:
-            self.n_points = len(archive.data.outputs)
+            try:
+                self.n_points = len(archive.data.outputs)
+            except Exception:
+                logger.error('No Outputs found.')
 
         if self.energies is None:
             try:
-                self.energies = [
+                energies = [
                     outputs.total_energies[-1].value.to('joule').magnitude
                     for outputs in archive.data.outputs
                 ]
+                if energies:
+                    self.energies = energies
             except Exception:
                 logger.error('Total energy not found in outputs.')
 
         if self.volumes is None:
             try:
                 volumes = []
                 for system in archive.data.model_system:
+                    lattice_vectors = None
                     for cell in system.cell:
                         if cell.lattice_vectors is not None:
-                            cell = system.atoms.lattice_vectors.to('m').magnitude
-                            volumes.append(get_volume(cell))
+                            lattice_vectors = cell.lattice_vectors.to('m').magnitude
                             break
-                self.volumes = volumes
+                    volumes.append(get_volume(lattice_vectors))
+                if volumes:
+                    self.volumes = volumes
             except Exception:
-                pass
+                logger.error('Error getting volume from model_system.')
+                return
+
+        to_fit = self.energies is not None and self.volumes is not None
 
-        if self.n_points != len(self.energies) != len(self.volumes):
-            logger.error('Iconsistent size of energies and volumes.')
+        if to_fit and not (self.n_points == len(self.energies) == len(self.volumes)):
+            logger.error('Inconsistent size of energies and volumes.')
+            to_fit = False
 
-        if (
-            not self.eos_fit
-            and self.results.volumes is not None
-            and self.results.energies is not None
-        ):
+        if not self.eos_fit and to_fit:
             # convert to ase units in order for function optimization to work
-            volumes = self.results.volumes.to('angstrom ** 3').magnitude
-            energies = self.results.energies.to('eV').magnitude
+            volumes = self.volumes.to('angstrom ** 3').magnitude
+            energies = self.energies.to('eV').magnitude
             for function_name, ase_name in FUNCTION_NAMES.items():
                 try:
                     eos = aseEOS(volumes, energies, ase_name)
@@ -202,7 +213,7 @@ def normalize(self, archive: EntryArchive, logger: BoundLogger) -> None:
                         equilibrium_energy=eos.e0 * ureg.eV,
                         rms_error=rms_error,
                     )
-                    self.results.eos_fit.append(eos_fit)
+                    self.eos_fit.append(eos_fit)
                 except Exception:
                     logger.warning('EOS fit unsuccesful.')
 
@@ -212,7 +223,7 @@ class EquationOfState(ParallelWorkflow):
     Definitions for equation of state workflow.
     """
 
-    task_label = 'Volume'
+    _task_label = 'Volume'
 
     @log
     def map_inputs(self, archive: EntryArchive) -> None: