Added EBSD map downsampling and bugfixing of non working AM example, …

…this completes the implementation of the tech partner HDF5 parsers for EBSD, next step linting and adding EDS reader (for IKZ, etc)

pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py

@@ -28,14 +28,14 @@
 from diffpy.structure import Lattice, Structure
 from orix import plot
 from orix.crystal_map import create_coordinate_arrays, CrystalMap, PhaseList
-from orix.quaternion import Rotation
+from orix.quaternion import Rotation, Orientation
 from orix.vector import Vector3d
 
 import matplotlib.pyplot as plt
 
 from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser
 from pynxtools.dataconverter.readers.em.utils.hfive_utils import \
-    read_strings_from_dataset, read_first_scalar, format_euler_parameterization
+    read_strings_from_dataset
 from pynxtools.dataconverter.readers.em.examples.ebsd_database import \
     ASSUME_PHASE_NAME_TO_SPACE_GROUP
 
@@ -164,6 +164,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str):
                 angles = [fp[f"{sub_grp_name}/Lattice Constant Alpha"][0],
                             fp[f"{sub_grp_name}/Lattice Constant Beta"][0],
                             fp[f"{sub_grp_name}/Lattice Constant Gamma"][0]]
+                # TODO::available examples support reporting in angstroem and degree
                 self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] \
                     = np.asarray(a_b_c, np.float32) * 0.1
                 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] \
@@ -210,22 +211,17 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
 
         for i in np.arange(0, n_pts):
             # check shape of internal virtual chunked number array
-            r = Rotation.from_matrix([np.reshape(dat[i][0], (3, 3))])
-            self.tmp[ckey]["euler"][i, :] = r.to_euler(degrees=False)
+            oris = Orientation.from_matrix([np.reshape(dat[i][0], (3, 3))])
+            self.tmp[ckey]["euler"][i, :] = oris.to_euler(degrees=False)
             self.tmp[ckey]["ci"][i] = dat[i][2]
             self.tmp[ckey]["phase_id"][i] = dat[i][3] + 1  # APEX seems to define
             # notIndexed as -1 and the first valid phase id 0
-
-
+        if np.isnan(self.tmp[ckey]["euler"]).any():
+            raise ValueError(f"Conversion of om2eu unexpectedly resulted in NaN !")
         # TODO::convert orientation matrix to Euler angles via om_eu but what are conventions !
         # orix based transformation ends up in positive half space and with degrees=False
         # as radiants but the from_matrix command above might miss one rotation
 
-        # inconsistency f32 in file although specification states float
-        # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"],
-        #                                 direction='lab2crystal',
-        #                                degrees=is_degrees)
-
         # compute explicit hexagon grid cells center of mass pixel positions
         # TODO::currently assuming s_x and s_y are already the correct center of mass
         # distances for hexagonal or square tiling of R^2

pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py

@@ -151,6 +151,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str):
                 a_b_c = values[0:3]
                 angles = values[3:6]
                 self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c * 0.1
+                # TODO::all examples indicate reporting in angstroem
                 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles
 
                 # Space Group, no, H5T_NATIVE_INT32, (1, 1), Space group index.
@@ -202,15 +203,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
             self.tmp[ckey]["euler"] = np.zeros((n_pts[0], 3), np.float32)
             column_id = 0
             for angle in ["phi1", "PHI", "phi2"]:
+                # TODO::available examples support that Bruker reports Euler triplets in degree
                 self.tmp[ckey]["euler"][:, column_id] \
-                    = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32)
+                    = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) / 180. * np.pi
                 column_id += 1
             self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"])
             n_pts = n_pts[0]
-        # inconsistency f32 in file although specification states float
-        # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"],
-        #                                 direction='lab2crystal',
-        #                                degrees=is_degrees)
 
         # index of phase, 0 if not indexed
         # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0

pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py

@@ -152,6 +152,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str):
                 values = np.asarray(fp[f"{sub_grp_name}/LatticeConstants"][:].flatten())
                 a_b_c = values[0:3]
                 angles = values[3:6]
+                # TODO::available examples support that community H5EBSD reports lattice constants in angstroem
                 self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c * 0.1
                 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles
 
@@ -205,15 +206,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
             self.tmp[ckey]["euler"] = np.zeros((n_pts[0], 3), np.float32)
             column_id = 0
             for angle in ["phi1", "PHI", "phi2"]:
+                # TODO::available examples support that community H5EBSD reports Euler triplets in degree
                 self.tmp[ckey]["euler"][:, column_id] \
-                    = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32)
+                    = np.asarray(fp[f"{grp_name}/{angle}"][:], np.float32) / 180. * np.pi
                 column_id += 1
             self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"])
             n_pts = n_pts[0]
-        # inconsistency f32 in file although specification states float
-        # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"],
-        #                                 direction='lab2crystal',
-        #                                degrees=is_degrees)
 
         # index of phase, 0 if not indexed
         # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0

pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py

@@ -36,7 +36,7 @@
 import matplotlib.pyplot as plt
 
 from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser
-from pynxtools.dataconverter.readers.em.utils.hfive_utils import \
+from pynxtools.dataconverter.readers.em.utils.hfive_utils import EULER_SPACE_SYMMETRY, \
     read_strings_from_dataset, read_first_scalar, format_euler_parameterization
 from pynxtools.dataconverter.readers.em.examples.ebsd_database import \
     ASSUME_PHASE_NAME_TO_SPACE_GROUP
@@ -164,6 +164,7 @@ def parse_and_normalize_group_ebsd_phases(self, fp, ckey: str):
                     angles = [fp[f"{sub_grp_name}/Lattice Constant alpha"][()],
                               fp[f"{sub_grp_name}/Lattice Constant beta"][()],
                               fp[f"{sub_grp_name}/Lattice Constant gamma"][()]]
+                    # TODO::available examples support reporting in angstroem and degree
                     self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] \
                         = np.asarray(a_b_c, np.float32) * 0.1
                     self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] \
@@ -209,6 +210,12 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
 
         n_pts = self.tmp[ckey]["n_x"] * self.tmp[ckey]["n_y"]
         self.tmp[ckey]["euler"] = np.zeros((n_pts, 3), np.float32)
+        # TODO::available examples support that rumour that in EDAX file sometimes values
+        # of Euler angle triplets are larger than mathematically possible
+        # unfortunately there is no confirmation from EDAX what is the reported unit and
+        # normalization for each software version, TODO::here rad is assumed but then values
+        # as large as 12.... should not be possible
+        # TODO::there has to be a mechanism which treats these dirty scan points!
         self.tmp[ckey]["euler"][:, 0] = np.asarray(fp[f"{grp_name}/Phi1"][:], np.float32)
         self.tmp[ckey]["euler"][:, 1] = np.asarray(fp[f"{grp_name}/Phi"][:], np.float32)
         self.tmp[ckey]["euler"][:, 2] = np.asarray(fp[f"{grp_name}/Phi2"][:], np.float32)
@@ -217,11 +224,19 @@ def parse_and_normalize_group_ebsd_data(self, fp, ckey: str):
 
         # given no official EDAX OimAnalysis spec we cannot define for sure if
         # phase_id == 0 means just all was indexed with the first/zeroth phase or nothing
-        # was indexed, TODO::assuming it means all indexed:
+        # was indexed, TODO::assuming it means all indexed with first phase:
         if np.all(fp[f"{grp_name}/Phase"][:] == 0):
             self.tmp[ckey]["phase_id"] = np.zeros(n_pts, np.int32) + 1
         else:
             self.tmp[ckey]["phase_id"] = np.asarray(fp[f"{grp_name}/Phase"][:], np.int32)
+        # TODO::mark scan points as dirty
+        # the line below shows an example how this could be achieved
+        # is_dirty = np.zeros((n_pts,), bool)
+        # for column_id in [0, 1, 2]:
+        #    is_dirty = is_dirty & np.abs(self.tmp[ckey]["euler"][:, column_id]) > EULER_SPACE_SYMMETRY
+        # print(f"Found {np.sum(is_dirty)} scan points which are marked now as dirty!")
+        # self.tmp[ckey]["phase_id"][is_dirty] = 0
+
         # promoting int8 to int32 no problem
         self.tmp[ckey]["ci"] = np.asarray(fp[f"{grp_name}/CI"][:], np.float32)
         # normalize pixel coordinates to physical positions even though the origin can still dangle somewhere

pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py

@@ -36,7 +36,8 @@
 import matplotlib.pyplot as plt
 
 from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser
-from pynxtools.dataconverter.readers.em.utils.hfive_utils import read_strings_from_dataset
+from pynxtools.dataconverter.readers.em.utils.hfive_utils import \
+    read_strings_from_dataset, format_euler_parameterization
 
 
 class HdfFiveOxfordReader(HdfFiveBaseParser):
@@ -171,17 +172,17 @@ def parse_and_normalize_slice_ebsd_phases(self, fp, ckey: str):
                     = read_strings_from_dataset(fp[f"{sub_grp_name}/Reference"][()])
 
                 # Lattice Angles, yes, H5T_NATIVE_FLOAT, (1, 3), Three columns for the alpha, beta and gamma angles in radians
-                is_degrees = False
                 if read_strings_from_dataset(fp[f"{sub_grp_name}/Lattice Angles"].attrs["Unit"]) == "rad":
-                    is_degrees = False
-                angles = np.asarray(fp[f"{sub_grp_name}/Lattice Angles"][:].flatten()) / np.pi * 180.
+                    angles = np.asarray(fp[f"{sub_grp_name}/Lattice Angles"][:].flatten())
+                else:
+                    raise ValueError(f"Unexpected case that Lattice Angles are not reported in rad !")
                 self.tmp[ckey]["phases"][int(phase_id)]["alpha_beta_gamma"] = angles
 
                 # Lattice Dimensions, yes, H5T_NATIVE_FLOAT, (1, 3), Three columns for a, b and c dimensions in Angstroms
-                is_nanometer = False
                 if read_strings_from_dataset(fp[f"{sub_grp_name}/Lattice Dimensions"].attrs["Unit"]) == "angstrom":
-                    is_nanometer = False
-                a_b_c = np.asarray(fp[f"{sub_grp_name}/Lattice Dimensions"][:].flatten()) * 0.1
+                    a_b_c = np.asarray(fp[f"{sub_grp_name}/Lattice Dimensions"][:].flatten()) * 0.1
+                else:
+                    raise ValueError(f"Unexpected case that Lattice Dimensions are not reported in angstroem !")
                 self.tmp[ckey]["phases"][int(phase_id)]["a_b_c"] = a_b_c
 
                 # Space Group, no, H5T_NATIVE_INT32, (1, 1), Space group index.
@@ -216,17 +217,11 @@ def parse_and_normalize_slice_ebsd_data(self, fp, ckey: str):
                 raise ValueError(f"Unable to parse {grp_name}/{req_field} !")
 
         # Euler, yes, H5T_NATIVE_FLOAT, (size, 3), Orientation of Crystal (CS2) to Sample-Surface (CS1).
-        is_degrees = False
-        is_negative = False
         if read_strings_from_dataset(fp[f"{grp_name}/Euler"].attrs["Unit"]) == "rad":
-            is_degrees = False
-        self.tmp[ckey]["euler"] = np.asarray(fp[f"{grp_name}/Euler"], np.float32)
-        # TODO::handle possible case of negative Euler angles (examples though do not indicate)
-        # that AZTec reports negative Euler angles...
-        # inconsistency f32 in file although specification states float
-        # Rotation.from_euler(euler=fp[f"{grp_name}/Euler"],
-        #                                 direction='lab2crystal',
-        #                                degrees=is_degrees)
+            self.tmp[ckey]["euler"] = np.asarray(fp[f"{grp_name}/Euler"], np.float32)
+        else:
+            raise ValueError(f"Unexpected case that Euler angle are not reported in rad !")
+        self.tmp[ckey]["euler"] = format_euler_parameterization(self.tmp[ckey]["euler"])
 
         # Phase, yes, H5T_NATIVE_INT32, (size, 1), Index of phase, 0 if not indexed
         # no normalization needed, also in NXem_ebsd the null model notIndexed is phase_identifier 0