Runthrough with all HDF5 examples from the community was successful b…

…ut the calibration of ipf and roi axis coordinates uses pixels currently but should use pixel times scan unit, next steps: i) implement this and tree-based downsampling, ii) linting, styling
FAIRmat-NFDI · Oct 24, 2023 · fea7842 · fea7842
1 parent ba45b67
commit fea7842
Show file tree

Hide file tree

Showing 7 changed files with 75 additions and 59 deletions.
diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_apex.py
@@ -123,7 +123,7 @@ def parse_and_normalize_group_ebsd_header(self, fp, ckey: str):
             raise ValueError(f"Grid Type {grid_type} is currently not supported !")
         self.tmp[ckey]["grid_type"] = grid_type
         self.tmp[ckey]["s_x"] = fp[f"{self.prfx}/Sample/Step X"][0]
-        self.tmp[ckey]["s_unit"] = "µm"  # TODO::always micron?
+        self.tmp[ckey]["s_unit"] = "um"  # "µm"  # TODO::always micron?
         self.tmp[ckey]["n_x"] = fp[f"{self.prfx}/Sample/Number Of Columns"][0]
         self.tmp[ckey]["s_y"] = fp[f"{self.prfx}/Sample/Step Y"][0]
         self.tmp[ckey]["n_y"] = fp[f"{self.prfx}/Sample/Number Of Rows"][0]

diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_bruker.py
@@ -115,7 +115,7 @@ def parse_and_normalize_group_ebsd_header(self, fp, ckey: str):
         self.tmp[ckey]["n_x"] = fp[f"{grp_name}/NCOLS"][()]
         self.tmp[ckey]["n_y"] = fp[f"{grp_name}/NROWS"][()]
         self.tmp[ckey]["s_x"] = fp[f"{grp_name}/SEPixelSizeX"][()]
-        self.tmp[ckey]["s_unit"] = "µm"  # TODO::always micron?
+        self.tmp[ckey]["s_unit"] = "um"  # "µm"  # TODO::always micron?
         self.tmp[ckey]["s_y"] = fp[f"{grp_name}/SEPixelSizeY"][()]
         # TODO::check that all data are consistent
         # TODO::what is y and x depends on coordinate system

diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_ebsd.py
@@ -116,7 +116,7 @@ def parse_and_normalize_group_ebsd_header(self, fp, ckey: str):
         self.tmp[ckey]["n_x"] = fp[f"{grp_name}/NCOLS"][()]
         self.tmp[ckey]["n_y"] = fp[f"{grp_name}/NROWS"][()]
         self.tmp[ckey]["s_x"] = fp[f"{grp_name}/SEPixelSizeX"][()]
-        self.tmp[ckey]["s_unit"] = "µm"  # TODO::always micron?
+        self.tmp[ckey]["s_unit"] = "um"  # "µm"  # TODO::always micron?
         self.tmp[ckey]["s_y"] = fp[f"{grp_name}/SEPixelSizeY"][()]
         # TODO::check that all data are consistent
         # TODO::what is y and x depends on coordinate system

diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_edax.py
@@ -122,7 +122,7 @@ def parse_and_normalize_group_ebsd_header(self, fp, ckey: str):
             raise ValueError(f"Grid Type {grid_type} is currently not supported !")
         self.tmp[ckey]["grid_type"] = grid_type
         self.tmp[ckey]["s_x"] = read_first_scalar(fp[f"{grp_name}/Step X"])
-        self.tmp[ckey]["s_unit"] = "µm"  # TODO::always micron?
+        self.tmp[ckey]["s_unit"] = "um"  # "µm"  # TODO::always micron?
         self.tmp[ckey]["n_x"] = read_first_scalar(fp[f"{grp_name}/nColumns"])
         self.tmp[ckey]["s_y"] = read_first_scalar(fp[f"{grp_name}/Step Y"])
         self.tmp[ckey]["n_y"] = read_first_scalar(fp[f"{grp_name}/nRows"])

diff --git a/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py b/pynxtools/dataconverter/readers/em/subparsers/hfive_oxford.py
@@ -129,7 +129,7 @@ def parse_and_normalize_slice_ebsd_header(self, fp, ckey: str):
         # X Step, yes, H5T_NATIVE_FLOAT, (1, 1), Map: Step size along x-axis in micrometers. Line scan: step size along the line scan in micrometers.
         if read_strings_from_dataset(fp[f"{grp_name}/X Step"].attrs["Unit"]) == "um":
             self.tmp[ckey]["s_x"] = fp[f"{grp_name}/X Step"][0]
-            self.tmp[ckey]["s_unit"] = "µm"
+            self.tmp[ckey]["s_unit"] = "um"  # "µm"
         else:
             raise ValueError(f"Unexpected X Step Unit attribute !")
         # Y Step, yes, H5T_NATIVE_FLOAT, (1, 1), Map: Step size along y-axis in micrometers. Line scan: Always set to 0.

diff --git a/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py b/pynxtools/dataconverter/readers/em/subparsers/nxs_hfive.py
@@ -159,7 +159,7 @@ def process_into_template(self, inp: dict, template: dict) -> dict:
                 else:
                     print(f"{key}, {val}")
 
-        # self.process_roi_overview(inp, template)
+        self.process_roi_overview(inp, template)
         self.process_roi_ebsd_maps(inp, template)
         return template
 
@@ -202,11 +202,15 @@ def process_roi_overview_ebsd_based(self,
         template[f"{trg}/data/@IMAGE_VERSION"] = f"1.2"
         template[f"{trg}/data/@SUBCLASS_VERSION"] = np.int64(15)
 
-        template[f"{trg}/AXISNAME[axis_x]"] = {"compress": np.asarray(inp["scan_point_x"], np.float32), "strength": 1}
-        template[f"{trg}/AXISNAME[axis_x]/@long_name"] = f"Coordinate along x-axis ({inp['s_unit']})"
+        template[f"{trg}/AXISNAME[axis_x]"] \
+            = {"compress": np.asarray(inp["scan_point_x"], np.float32), "strength": 1}
+        template[f"{trg}/AXISNAME[axis_x]/@long_name"] \
+            = f"Coordinate along x-axis ({inp['s_unit']})"
         template[f"{trg}/AXISNAME[axis_x]/@units"] = f"{inp['s_unit']}"
-        template[f"{trg}/AXISNAME[axis_y]"] = {"compress": np.asarray(inp["scan_point_y"], np.float32), "strength": 1}
-        template[f"{trg}/AXISNAME[axis_y]/@long_name"] = f"Coordinate along y-axis ({inp['s_unit']})"
+        template[f"{trg}/AXISNAME[axis_y]"] \
+            = {"compress": np.asarray(inp["scan_point_y"], np.float32), "strength": 1}
+        template[f"{trg}/AXISNAME[axis_y]/@long_name"] \
+            = f"Coordinate along y-axis ({inp['s_unit']})"
         template[f"{trg}/AXISNAME[axis_y]/@units"] =  f"{inp['s_unit']}"
         return template
 
@@ -229,6 +233,7 @@ def process_roi_xmap(self, inp: dict, roi_id: int, template: dict) -> dict:
                 (inp["n_y"], inp["n_x"]), (inp["s_y"], inp["s_x"]))
             xaxis = coordinates["x"]
             yaxis = coordinates["y"]
+            print(f"xmi {np.min(xaxis)}, xmx {np.max(xaxis)}, ymi {np.min(yaxis)}, ymx {np.max(yaxis)}")
             del coordinates
         else:
             raise ValueError(f"Downsampling for too large EBSD maps is currently not supported !")
@@ -263,6 +268,8 @@ def process_roi_xmap(self, inp: dict, roi_id: int, template: dict) -> dict:
                                                     structures=inp["phase"]),
                                prop={},
                                scan_unit=inp["s_unit"])
+        del xaxis
+        del yaxis
         # "bc": inp["band_contrast"]}, scan_unit=inp["s_unit"])
         print(self.xmap)
         return template
@@ -275,27 +282,26 @@ def process_roi_phases(self, inp: dict, roi_id: int, template: dict) -> dict:
         n_pts_indexed = np.sum(inp["phase_id"] != 0)
         print(f"n_pts {n_pts}, n_pts_indexed {n_pts_indexed}")
         template[f"{prfx}/number_of_scan_points"] = np.uint32(n_pts)
-        template[f"{prfx}/indexing_rate"] = np.float64(100. n_pts_indexed / n_pts)
+        template[f"{prfx}/indexing_rate"] = np.float64(100. * n_pts_indexed / n_pts)
         template[f"{prfx}/indexing_rate/@units"] = f"%"
-        template[f"{prfx}/phase{phase_id}/number_of_scan_points"] = np.uint32(0)
-        template[f"{prfx}/phase{phase_id}/phase_identifier"] = np.uint32(phase_id)
-        template[f"{prfx}/phase{phase_id}/phase_name"] = f"notIndexed"
+        grp_name = f"{prfx}/EM_EBSD_CRYSTAL_STRUCTURE_MODEL[phase{phase_id}]"
+        template[f"{grp_name}/number_of_scan_points"] = np.uint32(0)
+        template[f"{grp_name}/phase_identifier"] = np.uint32(phase_id)
+        template[f"{grp_name}/phase_name"] = f"notIndexed"
 
         for pyxem_phase_id in np.arange(0, np.max(self.xmap.phase_id) + 1):
             # this loop is implicitly ignored as when xmap is None
             print(f"inp[phases].keys(): {inp['phases'].keys()}")
             if (pyxem_phase_id + 1) not in inp["phases"].keys():
                 raise ValueError(f"{pyxem_phase_id + 1} is not a key in inp['phases'] !")
-            # if isinstance(inp["phases"][phase_id], dict) is True:
-            # phase_id of pyxem notIndexed is -1 while for NeXus it is 0 so add + 1 in naming schemes
-            trg = f"{prfx}/phase{pyxem_phase_id + 1}"
+            # phase_id of pyxem notIndexed is -1 while for NeXus
+            # it is 0 so add + 1 in naming schemes
+            trg = f"{prfx}/EM_EBSD_CRYSTAL_STRUCTURE_MODEL[phase{pyxem_phase_id + 1}]"
             template[f"{trg}/number_of_scan_points"] \
                 = np.uint32(np.sum(self.xmap.phase_id == pyxem_phase_id))
-            # print(f"{pyxem_phase_id + 1}, " \
-            #       f"{np.uint32(np.sum(self.xmap.phase_id == pyxem_phase_id))}," \
-            #       f" {inp['phases'][pyxem_phase_id + 1]['phase_name']}")
             template[f"{trg}/phase_identifier"] = np.uint32(pyxem_phase_id + 1)
-            template[f"{trg}/phase_name"] = f"{inp['phases'][pyxem_phase_id + 1]['phase_name']}"
+            template[f"{trg}/phase_name"] \
+                = f"{inp['phases'][pyxem_phase_id + 1]['phase_name']}"
 
             self.process_roi_phase_inverse_pole_figures(roi_id, pyxem_phase_id, template)
         return template
@@ -345,55 +351,64 @@ def process_roi_phase_inverse_pole_figures(self,
             # 0 is y while 1 is x !
 
             trg = f"/ENTRY[entry{self.entry_id}]/ROI[roi{roi_id}]/ebsd/indexing" \
-                  f"/phase{pyxem_phase_id + 1}/ipf{idx + 1}"
+                  f"/EM_EBSD_CRYSTAL_STRUCTURE_MODEL[phase{pyxem_phase_id + 1}]" \
+                  f"/MS_IPF[ipf{idx + 1}]"
             template[f"{trg}/projection_direction"] = np.asarray([0., 0., 1.], np.float32)
 
             # add the IPF color map
-            template[f"{trg}/DATA[map]/title"] \
+            mpp = f"{trg}/DATA[map]"
+            template[f"{mpp}/title"] \
                 = f"Inverse pole figure {projection_directions[idx][0]} {phase_name}"
-            template[f"{trg}/DATA[map]/@signal"] = "data"
-            template[f"{trg}/DATA[map]/@axes"] = ["axis_y", "axis_x"]
-            template[f"{trg}/DATA[map]/@AXISNAME_indices[axis_x_indices]"] = np.uint32(0)
-            template[f"{trg}/DATA[map]/@AXISNAME_indices[axis_y_indices]"] = np.uint32(1)
-            template[f"{trg}/DATA[map]/DATA[data]"] = {"compress": ipf_rgb_map, "strength": 1}
-            template[f"{trg}/DATA[map]/DATA[data]/@CLASS"] = "IMAGE"  # required, H5Web, RGB
-            template[f"{trg}/DATA[map]/DATA[data]/@IMAGE_VERSION"] = "1.2"
-            template[f"{trg}/DATA[map]/DATA[data]/@SUBCLASS_VERSION"] = np.int64(15)
-
-            template[f"{trg}/DATA[map]/AXISNAME[axis_x]"] \
+            template[f"{mpp}/@signal"] = "data"
+            template[f"{mpp}/@axes"] = ["axis_y", "axis_x"]
+            template[f"{mpp}/@AXISNAME_indices[axis_x_indices]"] = np.uint32(0)
+            template[f"{mpp}/@AXISNAME_indices[axis_y_indices]"] = np.uint32(1)
+            template[f"{mpp}/DATA[data]"] = {"compress": ipf_rgb_map, "strength": 1}
+            template[f"{mpp}/DATA[data]/@CLASS"] = "IMAGE"  # required, H5Web, RGB
+            template[f"{mpp}/DATA[data]/@IMAGE_VERSION"] = "1.2"
+            template[f"{mpp}/DATA[data]/@SUBCLASS_VERSION"] = np.int64(15)
+
+            template[f"{mpp}/AXISNAME[axis_x]"] \
                 = {"compress": np.asarray(self.xmap.x, np.float32), "strength": 1}
-            template[f"{trg}/DATA[map]/AXISNAME[axis_x]/@long_name"] \
+            template[f"{mpp}/AXISNAME[axis_x]/@long_name"] \
                 = f"Coordinate along x-axis ({self.xmap.scan_unit})"
-            template[f"{trg}/DATA[map]/AXISNAME[axis_x]/@units"] \
-                = f"{self.xmap.scan_unit}"
-            template[f"{trg}/DATA[map]/AXISNAME[axis_y]"] \
+            template[f"{mpp}/AXISNAME[axis_x]/@units"] = f"{self.xmap.scan_unit}"
+            template[f"{mpp}/AXISNAME[axis_y]"] \
                 = {"compress": np.asarray(self.xmap.y, np.float32), "strength": 1}
-            template[f"{trg}/DATA[map]/AXISNAME[axis_y]/@long_name"] \
+            template[f"{mpp}/AXISNAME[axis_y]/@long_name"] \
                 = f"Coordinate along y-axis ({self.xmap.scan_unit})"
-            template[f"{trg}/DATA[map]/AXISNAME[axis_y]/@units"] \
-                = f"{self.xmap.scan_unit}"
+            template[f"{mpp}/AXISNAME[axis_y]/@units"] = f"{self.xmap.scan_unit}"
 
             # add the IPF color map legend/key
-            template[f"{trg}/DATA[legend]/title"] \
+            lgd = f"{trg}/DATA[legend]"
+            template[f"{lgd}/title"] \
                 = f"Inverse pole figure {projection_directions[idx][0]} {phase_name}"
             # template[f"{trg}/title"] = f"Inverse pole figure color key with SST"
-            template[f"{trg}/DATA[legend]/@signal"] = "data"
-            template[f"{trg}/DATA[legend]/@axes"] = ["axis_y", "axis_x"]
-            template[f"{trg}/DATA[legend]/@AXISNAME_indices[axis_x_indices]"] = np.uint32(0)
-            template[f"{trg}/DATA[legend]/@AXISNAME_indices[axis_y_indices]"] = np.uint32(1)
-            template[f"{trg}/DATA[legend]/DATA[data]"] = {"compress": img, "strength": 1}
-            template[f"{trg}/DATA[legend]/DATA[data]/@CLASS"] = "IMAGE"  # required by H5Web to plot RGB maps
-            template[f"{trg}/DATA[legend]/DATA[data]/@IMAGE_VERSION"] = "1.2"
-            template[f"{trg}/DATA[legend]/DATA[data]/@SUBCLASS_VERSION"] = np.int64(15)
-
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_x]"] \
-                = {"compress": np.asarray(np.linspace(1, np.shape(img)[0], num=np.shape(img)[0], endpoint=True), np.uint32), "strength": 1}
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_x]/@long_name"] = "Pixel along x-axis"
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_x]/@units"] = "px"
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_y]"] \
-                = {"compress": np.asarray(np.linspace(1, np.shape(img)[1], num=np.shape(img)[1], endpoint=True), np.uint32), "strength": 1}
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_y]/@long_name"] = "Pixel along y-axis"
-            template[f"{trg}/DATA[legend]/AXISNAME[axis_y]/@units"] = "px"
+            template[f"{lgd}/@signal"] = "data"
+            template[f"{lgd}/@axes"] = ["axis_y", "axis_x"]
+            template[f"{lgd}/@AXISNAME_indices[axis_x_indices]"] = np.uint32(0)
+            template[f"{lgd}/@AXISNAME_indices[axis_y_indices]"] = np.uint32(1)
+            template[f"{lgd}/data"] = {"compress": img, "strength": 1}
+            template[f"{lgd}/data/@CLASS"] = f"IMAGE"  # required by H5Web to plot RGB maps
+            template[f"{lgd}/data/@IMAGE_VERSION"] = f"1.2"
+            template[f"{lgd}/data/@SUBCLASS_VERSION"] = np.int64(15)
+
+            template[f"{lgd}/AXISNAME[axis_x]"] \
+                = {"compress": np.asarray(np.linspace(1,
+                                                      np.shape(img)[0],
+                                                      num=np.shape(img)[0],
+                                                      endpoint=True), np.uint32),
+                   "strength": 1}
+            template[f"{lgd}/AXISNAME[axis_x]/@long_name"] = "Pixel along x-axis"
+            template[f"{lgd}/AXISNAME[axis_x]/@units"] = "px"
+            template[f"{lgd}/AXISNAME[axis_y]"] \
+                = {"compress": np.asarray(np.linspace(1,
+                                                      np.shape(img)[1],
+                                                      num=np.shape(img)[1],
+                                                      endpoint=True), np.uint32),
+                   "strength": 1}
+            template[f"{lgd}/AXISNAME[axis_y]/@long_name"] = "Pixel along y-axis"
+            template[f"{lgd}/AXISNAME[axis_y]/@units"] = "px"
 
         # call process_roi_ipf_color_key
         return template
diff --git a/test.all.sh b/test.all.sh
@@ -9,10 +9,11 @@
 # 026_0007.h5 026_0027.h5 026_0029.h5 026_0030.h5 026_0033.h5 026_0039.h5 026_0041.h5 delmic hdf5 have no ebsd data
 # 173_0056.h5oina has only eds data
 
-Examples="026_0046.h5oina 026_0049.h5oina 026_0050.h5oina 026_0052.h5oina 066_0013.h5 066_0015.h5 066_0016.h5 066_0023.h5 066_0025.h5 066_0034.h5 078_0004.h5 087_0021.h5 088_0009.h5 093_0045.h5oina 093_0047.h5oina 093_0048.h5oina 093_0051.h5oina 093_0053.h5oina 093_0054.h5oina 093_0055.h5oina 093_0058.h5oina 093_0059.h5oina 093_0060.h5oina 093_0062.h5oina 093_0063.h5oina 101_0040.h5 110_0012.h5 114_0017.h5 116_0008.h5 116_0014.h5 116_0018.h5 116_0019.h5 116_0020.h5 116_0022.h5 116_0037.h5 116_0042.h5 124_0002.h5 124_0036.h5 125_0006.h5 126_0038.h5 130_0003.h5 130_2082.h5 130_2083.h5 130_2084.h5 130_2085.h5 130_2086.h5 130_2087.h5 130_2088.h5 130_2089.h5 130_2090.h5 130_2091.h5 130_2092.h5 130_2093.h5 130_2094.h5 132_0005.h5 144_0043.h5 173_0056.h5oina 173_0057.h5oina 174_0031.h5 207_2081.edaxh5 208_0061.h5oina 212_2095.h5oina 229_2096.oh5 229_2097.oh5"
+# Examples="026_0046.h5oina 026_0049.h5oina 026_0050.h5oina 026_0052.h5oina 066_0013.h5 066_0015.h5 066_0016.h5 066_0023.h5 066_0025.h5 066_0034.h5 078_0004.h5 087_0021.h5 088_0009.h5 093_0045.h5oina 093_0047.h5oina 093_0048.h5oina 093_0051.h5oina 093_0053.h5oina 093_0054.h5oina 093_0055.h5oina 093_0058.h5oina 093_0059.h5oina 093_0060.h5oina 093_0062.h5oina 093_0063.h5oina 101_0040.h5 110_0012.h5 114_0017.h5 116_0008.h5 116_0014.h5 116_0018.h5 116_0019.h5 116_0020.h5 116_0022.h5 116_0037.h5 116_0042.h5 124_0002.h5 124_0036.h5 125_0006.h5 126_0038.h5 130_0003.h5 130_2082.h5 130_2083.h5 130_2084.h5 130_2085.h5 130_2086.h5 130_2087.h5 130_2088.h5 130_2089.h5 130_2090.h5 130_2091.h5 130_2092.h5 130_2093.h5 130_2094.h5 132_0005.h5 144_0043.h5 173_0056.h5oina 173_0057.h5oina 174_0031.h5 207_2081.edaxh5 208_0061.h5oina 212_2095.h5oina 229_2096.oh5 229_2097.oh5"
 
 # Examples="207_2081.edaxh5"
 # Examples="173_0057.h5oina"
+Examples="229_2097.oh5"
 for example in $Examples; do
 	echo $example
 	dataconverter --reader em --nxdl NXroot --input-file $example --output debug.$example.nxs 1>stdout.$example.nxs.txt 2>stderr.$example.nxs.txt