Skip to content

Rename 'classification' field from ReconstructedContainer to 'particle_type' #2763

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/user-guide/data_format/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ output file, where ``<algorithm>`` is the identifier of the algorithm
* - /energy
- shower energy reconstruction
- :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ReconstructedEnergyContainer`
* - /classification
* - /particle_type
- shower classification parameters
- :py:class:`~ctapipe.containers.EventIndexContainer`, :py:class:`~ctapipe.containers.ParticleClassificationContainer`

Expand Down
27 changes: 26 additions & 1 deletion src/ctapipe/containers.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""

import enum
import warnings
from functools import partial

import numpy as np
Expand Down Expand Up @@ -1084,11 +1085,35 @@ class ReconstructedContainer(Container):
default_factory=partial(Map, ReconstructedEnergyContainer),
description="map of algorithm to reconstructed energy parameters",
)
classification = Field(
particle_type = Field(
default_factory=partial(Map, ParticleClassificationContainer),
description="map of algorithm to classification parameters",
)

@staticmethod
def _warn_classification_deprecated():
from ctapipe.utils.deprecation import CTAPipeDeprecationWarning

warnings.warn(
"The 'classification' field from 'ReconstructedContainer' was "
"renamed and will be deleted in future versions. Use "
"'particle_type' instead. "
"E.g. switch from 'event.dl2.{tel,stereo}.classification' to "
"'event.dl2.{tel,stereo}.particle_type'.",
CTAPipeDeprecationWarning,
stacklevel=3,
)

@property
def classification(self):
self._warn_classification_deprecated()
return self.particle_type

@classification.setter
def classification(self, value):
self._warn_classification_deprecated()
self.particle_type = value


class TelescopeReconstructedContainer(ReconstructedContainer):
"""Telescope-wise reconstructed quantities"""
Expand Down
6 changes: 3 additions & 3 deletions src/ctapipe/io/datawriter.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
Class to write DL1 (a,b) and DL2 (a) data from an event stream
"""


import pathlib
from collections import defaultdict

Expand Down Expand Up @@ -45,8 +44,9 @@ def _get_tel_index(event, tel_id):
# (meaning readers need to update scripts)
# - increase the minor number if new columns or datasets are added
# - increase the patch number if there is a small bugfix to the model.
DATA_MODEL_VERSION = "v7.1.0"
DATA_MODEL_VERSION = "v7.2.0"
DATA_MODEL_CHANGE_HISTORY = """
- v7.2.0: - Change field name in ``ReconstructedContainer`` from 'classification' to 'particle_type'.
- v7.1.0: - Two new fields for the hillas parameters, uncertainties on psi and the transversal cog coordinate.
- v7.0.0: - Use high resolution timestamps for times. CTAO high resolution times
are stored as two uint32: seconds and quarter nanoseconds since 1970-01-01T00:00:00 TAI.
Expand Down Expand Up @@ -705,7 +705,7 @@ def _write_dl2_telescope_events(self, event: ArrayEventContainer):
def _write_dl2_stereo_event(self, event: ArrayEventContainer):
"""
write per-telescope DL2 shower information to e.g.
`/dl2/event/stereo/{geometry,energy,classification}/<algorithm_name>`
`/dl2/event/stereo/{geometry,energy,particle_type}/<algorithm_name>`
"""
# pylint: disable=no-self-use
for container_name, algorithm_map in event.dl2.stereo.items():
Expand Down
9 changes: 8 additions & 1 deletion src/ctapipe/io/hdf5eventsource.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
DL2_CONTAINERS = {
"energy": ReconstructedEnergyContainer,
"geometry": ReconstructedGeometryContainer,
"classification": ParticleClassificationContainer,
"particle_type": ParticleClassificationContainer,
"impact": TelescopeImpactParameterContainer,
"disp": DispContainer,
}
Expand All @@ -79,6 +79,7 @@
"v6.0.0",
"v7.0.0",
"v7.1.0",
"v7.2.0",
]


Expand Down Expand Up @@ -456,7 +457,7 @@
one_telescope = parameters_group._v_children.values()[0]
return "camera_frame_hillas_intensity" in one_telescope.colnames

def _generator(self):

Check failure on line 460 in src/ctapipe/io/hdf5eventsource.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/io/hdf5eventsource.py#L460

Refactor this function to reduce its Cognitive Complexity from 106 to the 15 allowed.
"""
Yield ArrayEventContainer to iterate through events.
"""
Expand Down Expand Up @@ -569,6 +570,9 @@
dl2_group = self.file_.root[DL2_SUBARRAY_GROUP]

for kind, group in dl2_group._v_children.items():
# TODO: Delete if we drop support for older data model versions (< v7.2.0)

Check notice on line 573 in src/ctapipe/io/hdf5eventsource.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/io/hdf5eventsource.py#L573

Complete the task associated to this "TODO" comment.
if kind == "classification":
kind = "particle_type"
try:
container = DL2_CONTAINERS[kind]
except KeyError:
Expand All @@ -589,6 +593,9 @@
dl2_group = self.file_.root[DL2_TELESCOPE_GROUP]

for kind, group in dl2_group._v_children.items():
# TODO: Delete if we drop support for older data model versions (< v7.2.0)

Check notice on line 596 in src/ctapipe/io/hdf5eventsource.py

View check run for this annotation

CTAO-DPPS-SonarQube / ctapipe Sonarqube Results

src/ctapipe/io/hdf5eventsource.py#L596

Complete the task associated to this "TODO" comment.
if kind == "classification":
kind = "particle_type"
try:
container = DL2_CONTAINERS[kind]
except KeyError:
Expand Down
6 changes: 2 additions & 4 deletions src/ctapipe/io/tests/test_datawriter.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@ def generate_dummy_dl2(event):
energy=10 * u.TeV,
prefix=f"{algo}_tel",
)
event.dl2.tel[tel_id].classification[
algo
] = ParticleClassificationContainer(
event.dl2.tel[tel_id].particle_type[algo] = ParticleClassificationContainer(
prediction=0.9,
prefix=f"{algo}_tel",
)
Expand All @@ -58,7 +56,7 @@ def generate_dummy_dl2(event):
telescopes=[1, 2, 4],
prefix=algo,
)
event.dl2.stereo.classification[algo] = ParticleClassificationContainer(
event.dl2.stereo.particle_type[algo] = ParticleClassificationContainer(
prediction=0.9,
telescopes=[1, 2, 4],
prefix=algo,
Expand Down
2 changes: 1 addition & 1 deletion src/ctapipe/io/tests/test_hdf5eventsource.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def test_is_compatible(compatible_file, request):
def test_metadata(dl1_file):
with HDF5EventSource(input_url=dl1_file) as source:
assert source.is_simulation
assert source.datamodel_version == (7, 1, 0)
assert source.datamodel_version == (7, 2, 0)
assert set(source.datalevels) == {
DataLevel.DL1_IMAGES,
DataLevel.DL1_PARAMETERS,
Expand Down
3 changes: 2 additions & 1 deletion src/ctapipe/reco/sklearn.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""
Component Wrappers around sklearn models
"""

import pathlib
from abc import abstractmethod
from collections import defaultdict
Expand Down Expand Up @@ -482,7 +483,7 @@ def __call__(self, event: ArrayEventContainer) -> None:
)

container.prefix = f"{self.prefix}_tel"
event.dl2.tel[tel_id].classification[self.prefix] = container
event.dl2.tel[tel_id].particle_type[self.prefix] = container

self.stereo_combiner(event)

Expand Down
7 changes: 3 additions & 4 deletions src/ctapipe/reco/stereo_combination.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ class StereoMeanCombiner(StereoCombiner):
["none", "intensity", "konrad"],
default_value="none",
help=(
"What kind of weights to use."
" Options: ``none``, ``intensity``, ``konrad``."
"What kind of weights to use. Options: ``none``, ``intensity``, ``konrad``."
),
).tag(config=True)

Expand Down Expand Up @@ -169,7 +168,7 @@ def _combine_classification(self, event):
weights = []

for tel_id, dl2 in event.dl2.tel.items():
mono = dl2.classification[self.prefix]
mono = dl2.particle_type[self.prefix]
if mono.is_valid:
values.append(mono.prediction)
dl1 = event.dl1.tel[tel_id].parameters
Expand All @@ -186,7 +185,7 @@ def _combine_classification(self, event):
container = ParticleClassificationContainer(
prediction=mean, telescopes=ids, is_valid=valid, prefix=self.prefix
)
event.dl2.stereo.classification[self.prefix] = container
event.dl2.stereo.particle_type[self.prefix] = container

def _combine_altaz(self, event):
ids = []
Expand Down
8 changes: 4 additions & 4 deletions src/ctapipe/reco/tests/test_stereo_combination.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def test_mean_prediction_single_event(weights):
energy={
"dummy": ReconstructedEnergyContainer(energy=10 * u.GeV, is_valid=True)
},
classification={
particle_type={
"dummy": ParticleClassificationContainer(prediction=1.0, is_valid=True)
},
geometry={
Expand All @@ -190,7 +190,7 @@ def test_mean_prediction_single_event(weights):
energy={
"dummy": ReconstructedEnergyContainer(energy=20 * u.GeV, is_valid=True)
},
classification={
particle_type={
"dummy": ParticleClassificationContainer(prediction=0.0, is_valid=True)
},
geometry={
Expand All @@ -203,7 +203,7 @@ def test_mean_prediction_single_event(weights):
energy={
"dummy": ReconstructedEnergyContainer(energy=0.04 * u.TeV, is_valid=True)
},
classification={
particle_type={
"dummy": ParticleClassificationContainer(prediction=0.8, is_valid=True)
},
geometry={
Expand Down Expand Up @@ -239,4 +239,4 @@ def test_mean_prediction_single_event(weights):
assert u.isclose(event.dl2.stereo.energy["dummy"].energy, 30 * u.GeV)
assert u.isclose(event.dl2.stereo.geometry["dummy"].alt, 60.9748605 * u.deg)
assert u.isclose(event.dl2.stereo.geometry["dummy"].az, 316.0365515 * u.deg)
assert event.dl2.stereo.classification["dummy"].prediction == 0.6
assert event.dl2.stereo.particle_type["dummy"].prediction == 0.6
4 changes: 2 additions & 2 deletions src/ctapipe/tools/tests/test_process_ml.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,13 @@ def test_process_apply_classification(
run_tool(tool, argv=argv, cwd=tmp_path, raises=True)

tel_events = read_table(
output, "/dl2/event/telescope/classification/ExtraTreesClassifier/tel_004"
output, "/dl2/event/telescope/particle_type/ExtraTreesClassifier/tel_004"
)
assert "ExtraTreesClassifier_tel_is_valid" in tel_events.colnames
assert "ExtraTreesClassifier_tel_prediction" in tel_events.colnames

events = read_table(
output, "/dl2/event/subarray/classification/ExtraTreesClassifier"
output, "/dl2/event/subarray/particle_type/ExtraTreesClassifier"
)
trigger = read_table(output, "/dl1/event/subarray/trigger")
assert len(events) == len(trigger)
Expand Down
Loading