From 8f0c0183a38be43a187c1941213d9c6fe3043c43 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Mon, 13 Jan 2025 07:28:10 -0600 Subject: [PATCH 01/24] Added a TODO to start implementation of HED support in annotations --- mne/annotations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/annotations.py b/mne/annotations.py index 629ee7b20cb..1c9ad85b1c9 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -757,6 +757,7 @@ def rename(self, mapping, verbose=None): self.description = np.array([str(mapping.get(d, d)) for d in self.description]) return self +# TODO: Add support for HED annotations for use in epoching. class EpochAnnotationsMixin: """Mixin class for Annotations in Epochs.""" From 6f6ccdceb240438906a643ab7cd51de8b5ebcfe4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:33:33 +0000 Subject: [PATCH 02/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne/annotations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mne/annotations.py b/mne/annotations.py index 1c9ad85b1c9..694836d8188 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -757,8 +757,10 @@ def rename(self, mapping, verbose=None): self.description = np.array([str(mapping.get(d, d)) for d in self.description]) return self + # TODO: Add support for HED annotations for use in epoching. + class EpochAnnotationsMixin: """Mixin class for Annotations in Epochs.""" From c31e837a7934b13267561e61e62ae4b403feee90 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Mon, 13 Jan 2025 16:42:30 -0600 Subject: [PATCH 03/24] add sketch of HEDAnnotations [ci skip] --- mne/__init__.pyi | 2 + mne/annotations.py | 139 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 139 insertions(+), 2 deletions(-) diff --git a/mne/__init__.pyi b/mne/__init__.pyi index d50b5209346..6560854402e 100644 --- a/mne/__init__.pyi +++ b/mne/__init__.pyi @@ -11,6 +11,7 @@ __all__ = [ "Evoked", "EvokedArray", "Forward", + "HEDAnnotations", "Info", "Label", "MixedSourceEstimate", @@ -260,6 +261,7 @@ from ._freesurfer import ( ) from .annotations import ( Annotations, + HEDAnnotations, annotations_from_events, count_annotations, events_from_annotations, diff --git a/mne/annotations.py b/mne/annotations.py index 694836d8188..a784c3ae143 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -52,6 +52,7 @@ verbose, warn, ) +from .utils.check import _soft_import # For testing windows_like_datetime, we monkeypatch "datetime" in this module. # Keep the true datetime object around for _validate_type use. @@ -151,6 +152,7 @@ class Annotations: -------- mne.annotations_from_events mne.events_from_annotations + mne.HEDAnnotations Notes ----- @@ -288,7 +290,7 @@ def orig_time(self): def __eq__(self, other): """Compare to another Annotations instance.""" - if not isinstance(other, Annotations): + if not isinstance(other, type(self)): return False return ( np.array_equal(self.onset, other.onset) @@ -567,6 +569,8 @@ def _sort(self): self.duration = self.duration[order] self.description = self.description[order] self.ch_names = self.ch_names[order] + if hasattr(self, "hed_tags"): + self.hed_tags = self.hed_tags[order] @verbose def crop( @@ -758,7 +762,138 @@ def rename(self, mapping, verbose=None): return self -# TODO: Add support for HED annotations for use in epoching. +class HEDAnnotations(Annotations): + """Annotations object for annotating segments of raw data with HED tags. + + Parameters + ---------- + onset : array of float, shape (n_annotations,) + The starting time of annotations in seconds after ``orig_time``. + duration : array of float, shape (n_annotations,) | float + Durations of the annotations in seconds. If a float, all the + annotations are given the same duration. + description : array of str, shape (n_annotations,) | str + Array of strings containing description for each annotation. If a + string, all the annotations are given the same description. To reject + epochs, use description starting with keyword 'bad'. See example above. + hed_tags : array of str, shape (n_annotations,) | str + Array of strings containing a HED tag for each annotation. If a single string + is provided, all annotations are given the same HED tag. + hed_version : str + The HED schema version against which to validate the HED tags. + orig_time : float | str | datetime | tuple of int | None + A POSIX Timestamp, datetime or a tuple containing the timestamp as the + first element and microseconds as the second element. Determines the + starting time of annotation acquisition. If None (default), + starting time is determined from beginning of raw data acquisition. + In general, ``raw.info['meas_date']`` (or None) can be used for syncing + the annotations with raw data if their acquisition is started at the + same time. If it is a string, it should conform to the ISO8601 format. + More precisely to this '%%Y-%%m-%%d %%H:%%M:%%S.%%f' particular case of + the ISO8601 format where the delimiter between date and time is ' '. + %(ch_names_annot)s + + See Also + -------- + mne.Annotations + + Notes + ----- + + .. versionadded:: 1.10 + """ + + def __init__( + self, + onset, + duration, + description, + hed_tags, + hed_version="latest", # TODO @VisLab what is a sensible default here? + orig_time=None, + ch_names=None, + ): + hed = _soft_import("hed", "validation of HED tags in annotations") # noqa + # TODO is some sort of initialization of the HED cache directory necessary? + super().__init__( + onset=onset, + duration=duration, + description=description, + orig_time=orig_time, + ch_names=ch_names, + ) + # TODO validate the HED version the user claims to be using. + self.hed_version = hed_version + self._update_hed_tags(hed_tags=hed_tags) + + def _update_hed_tags(self, hed_tags): + if len(hed_tags) != len(self): + raise ValueError( + f"Number of HED tags ({len(hed_tags)}) must match the number of " + f"annotations ({len(self)})." + ) + # TODO insert validation of HED tags here + self.hed_tags = hed_tags + + def __eq__(self, other): + """Compare to another HEDAnnotations instance.""" + return ( + super().__eq__(self, other) + and np.array_equal(self.hed_tags, other.hed_tags) + and self.hed_version == other.hed_version + ) + + def __repr__(self): + """Show a textual summary of the object.""" + counter = Counter(self.hed_tags) + kinds = ", ".join(["{} ({})".format(*k) for k in sorted(counter.items())]) + kinds = (": " if len(kinds) > 0 else "") + kinds + ch_specific = ", channel-specific" if self._any_ch_names() else "" + s = ( + f"HEDAnnotations | {len(self.onset)} segment" + f"{_pl(len(self.onset))}{ch_specific}{kinds}" + ) + return "<" + shorten(s, width=77, placeholder=" ...") + ">" + + def __getitem__(self, key, *, with_ch_names=None): + """Propagate indexing and slicing to the underlying numpy structure.""" + result = super().__getitem__(self, key, with_ch_names=with_ch_names) + if isinstance(result, OrderedDict): + result["hed_tags"] = self.hed_tags[key] + else: + key = list(key) if isinstance(key, tuple) else key + hed_tags = self.hed_tags[key] + return HEDAnnotations( + result.onset, + result.duration, + result.description, + hed_tags, + hed_version=self.hed_version, + orig_time=self.orig_time, + ch_names=result.ch_names, + ) + + def append(self, onset, duration, description, ch_names=None): + """TODO.""" + pass + + def count(self): + """TODO. Unlike Annotations.count, keys should be HED tags not descriptions.""" + pass + + def crop( + self, tmin=None, tmax=None, emit_warning=False, use_orig_time=True, verbose=None + ): + """TODO.""" + pass + + def delete(self, idx): + """TODO.""" + pass + + def to_data_frame(self, time_format="datetime"): + """TODO.""" + pass class EpochAnnotationsMixin: From b3183d353c32606bd96e54885a7f54d323183a1b Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 31 Jan 2025 14:15:59 -0600 Subject: [PATCH 04/24] rename hed_tags -> hed_strings --- mne/annotations.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index a784c3ae143..d74f37882cf 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -569,8 +569,8 @@ def _sort(self): self.duration = self.duration[order] self.description = self.description[order] self.ch_names = self.ch_names[order] - if hasattr(self, "hed_tags"): - self.hed_tags = self.hed_tags[order] + if hasattr(self, "hed_strings"): + self.hed_strings = self.hed_strings[order] @verbose def crop( @@ -776,11 +776,12 @@ class HEDAnnotations(Annotations): Array of strings containing description for each annotation. If a string, all the annotations are given the same description. To reject epochs, use description starting with keyword 'bad'. See example above. - hed_tags : array of str, shape (n_annotations,) | str - Array of strings containing a HED tag for each annotation. If a single string - is provided, all annotations are given the same HED tag. + hed_strings : array of str, shape (n_annotations,) | str + Sequence of strings containing a HED tag (or comma-separated list of HED tags) + for each annotation. If a single string is provided, all annotations are + assigned the same HED string. hed_version : str - The HED schema version against which to validate the HED tags. + The HED schema version against which to validate the HED strings. orig_time : float | str | datetime | tuple of int | None A POSIX Timestamp, datetime or a tuple containing the timestamp as the first element and microseconds as the second element. Determines the @@ -808,7 +809,7 @@ def __init__( onset, duration, description, - hed_tags, + hed_strings, hed_version="latest", # TODO @VisLab what is a sensible default here? orig_time=None, ch_names=None, @@ -824,28 +825,28 @@ def __init__( ) # TODO validate the HED version the user claims to be using. self.hed_version = hed_version - self._update_hed_tags(hed_tags=hed_tags) + self._update_hed_strings(hed_strings=hed_strings) - def _update_hed_tags(self, hed_tags): - if len(hed_tags) != len(self): + def _update_hed_strings(self, hed_strings): + if len(hed_strings) != len(self): raise ValueError( - f"Number of HED tags ({len(hed_tags)}) must match the number of " + f"Number of HED strings ({len(hed_strings)}) must match the number of " f"annotations ({len(self)})." ) - # TODO insert validation of HED tags here - self.hed_tags = hed_tags + # TODO insert validation of HED strings here + self.hed_strings = hed_strings def __eq__(self, other): """Compare to another HEDAnnotations instance.""" return ( super().__eq__(self, other) - and np.array_equal(self.hed_tags, other.hed_tags) + and np.array_equal(self.hed_strings, other.hed_strings) and self.hed_version == other.hed_version ) def __repr__(self): """Show a textual summary of the object.""" - counter = Counter(self.hed_tags) + counter = Counter(self.hed_strings) kinds = ", ".join(["{} ({})".format(*k) for k in sorted(counter.items())]) kinds = (": " if len(kinds) > 0 else "") + kinds ch_specific = ", channel-specific" if self._any_ch_names() else "" @@ -859,15 +860,15 @@ def __getitem__(self, key, *, with_ch_names=None): """Propagate indexing and slicing to the underlying numpy structure.""" result = super().__getitem__(self, key, with_ch_names=with_ch_names) if isinstance(result, OrderedDict): - result["hed_tags"] = self.hed_tags[key] + result["hed_strings"] = self.hed_strings[key] else: key = list(key) if isinstance(key, tuple) else key - hed_tags = self.hed_tags[key] + hed_strings = self.hed_strings[key] return HEDAnnotations( result.onset, result.duration, result.description, - hed_tags, + hed_strings, hed_version=self.hed_version, orig_time=self.orig_time, ch_names=result.ch_names, From 4065433c2bfb1066488b8bb0a939170cbf7df479 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 31 Jan 2025 14:16:22 -0600 Subject: [PATCH 05/24] remove unnecessary TODOs --- mne/annotations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index d74f37882cf..79a47e3d20b 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -815,7 +815,6 @@ def __init__( ch_names=None, ): hed = _soft_import("hed", "validation of HED tags in annotations") # noqa - # TODO is some sort of initialization of the HED cache directory necessary? super().__init__( onset=onset, duration=duration, @@ -823,7 +822,6 @@ def __init__( orig_time=orig_time, ch_names=ch_names, ) - # TODO validate the HED version the user claims to be using. self.hed_version = hed_version self._update_hed_strings(hed_strings=hed_strings) From 52400411f2634092e46ce15d2ccafe11ef0bdbe0 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 31 Jan 2025 14:49:44 -0600 Subject: [PATCH 06/24] add basic validation --- mne/annotations.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 79a47e3d20b..e798f60243c 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -810,11 +810,12 @@ def __init__( duration, description, hed_strings, - hed_version="latest", # TODO @VisLab what is a sensible default here? + hed_version="8.3.0", # TODO @VisLab what is a sensible default here? orig_time=None, ch_names=None, ): - hed = _soft_import("hed", "validation of HED tags in annotations") # noqa + self.hed = _soft_import("hed", "validation of HED tags in annotations") + super().__init__( onset=onset, duration=duration, @@ -826,14 +827,37 @@ def __init__( self._update_hed_strings(hed_strings=hed_strings) def _update_hed_strings(self, hed_strings): + # NB: must import; calling self.hed.validator.HedValidator doesn't work + from hed.validator import HedValidator + if len(hed_strings) != len(self): raise ValueError( f"Number of HED strings ({len(hed_strings)}) must match the number of " f"annotations ({len(self)})." ) - # TODO insert validation of HED strings here + # validation of HED strings + schema = self.hed.load_schema_version(self.hed_version) + validator = HedValidator(schema) + error_handler = self.hed.errors.ErrorHandler(check_for_warnings=False) + error_strs = [ + self._validate_one_hed_string(hs, schema, validator, error_handler) + for hs in hed_strings + ] + if any(map(len, error_strs)): + raise ValueError( + "Some HED strings in your annotations failed to validate:\n" + + "\n - ".join(error_strs) + ) self.hed_strings = hed_strings + def _validate_one_hed_string(self, hed_string, schema, validator, error_handler): + """Validate a user-provided HED string.""" + foo = self.hed.HedString(hed_string, schema) + issues = validator.validate( + foo, allow_placeholders=False, error_handler=error_handler + ) + return self.hed.get_printable_issue_string(issues) + def __eq__(self, other): """Compare to another HEDAnnotations instance.""" return ( From 40311f0557268af59116b84b2ee66eec70365e39 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 31 Jan 2025 14:52:28 -0600 Subject: [PATCH 07/24] fix err message indentation --- mne/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/annotations.py b/mne/annotations.py index e798f60243c..ac2d527826e 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -845,7 +845,7 @@ def _update_hed_strings(self, hed_strings): ] if any(map(len, error_strs)): raise ValueError( - "Some HED strings in your annotations failed to validate:\n" + "Some HED strings in your annotations failed to validate:\n - " + "\n - ".join(error_strs) ) self.hed_strings = hed_strings From 1485356e5c87a8ea2c53b843989544f0076f48d7 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 31 Jan 2025 14:53:26 -0600 Subject: [PATCH 08/24] don't call it foo --- mne/annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index ac2d527826e..09d907637a7 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -852,9 +852,9 @@ def _update_hed_strings(self, hed_strings): def _validate_one_hed_string(self, hed_string, schema, validator, error_handler): """Validate a user-provided HED string.""" - foo = self.hed.HedString(hed_string, schema) + hs = self.hed.HedString(hed_string, schema) issues = validator.validate( - foo, allow_placeholders=False, error_handler=error_handler + hs, allow_placeholders=False, error_handler=error_handler ) return self.hed.get_printable_issue_string(issues) From 75a4d4f4de10982b6b7eb0938652236f428256b5 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 9 Apr 2025 17:41:05 -0500 Subject: [PATCH 09/24] store HedString objects; use short form for repr --- mne/annotations.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 09d907637a7..fb5fdba0584 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -835,28 +835,27 @@ def _update_hed_strings(self, hed_strings): f"Number of HED strings ({len(hed_strings)}) must match the number of " f"annotations ({len(self)})." ) - # validation of HED strings + # create HedString objects schema = self.hed.load_schema_version(self.hed_version) + self._hed_strings = [self.hed.HedString(hs, schema) for hs in hed_strings] + # validation of HED strings validator = HedValidator(schema) error_handler = self.hed.errors.ErrorHandler(check_for_warnings=False) - error_strs = [ - self._validate_one_hed_string(hs, schema, validator, error_handler) - for hs in hed_strings + issues = [ + validator.validate( + hs, allow_placeholders=False, error_handler=error_handler + ) + for hs in self._hed_strings ] - if any(map(len, error_strs)): + error_strings = [self.hed.get_printable_issue_string(issue) for issue in issues] + if any(map(len, error_strings)): raise ValueError( "Some HED strings in your annotations failed to validate:\n - " - + "\n - ".join(error_strs) + + "\n - ".join(error_strings) ) - self.hed_strings = hed_strings - - def _validate_one_hed_string(self, hed_string, schema, validator, error_handler): - """Validate a user-provided HED string.""" - hs = self.hed.HedString(hed_string, schema) - issues = validator.validate( - hs, allow_placeholders=False, error_handler=error_handler + self.hed_strings = tuple( + hs.get_original_hed_string() for hs in self._hed_strings ) - return self.hed.get_printable_issue_string(issues) def __eq__(self, other): """Compare to another HEDAnnotations instance.""" @@ -868,7 +867,7 @@ def __eq__(self, other): def __repr__(self): """Show a textual summary of the object.""" - counter = Counter(self.hed_strings) + counter = Counter([hs.get_as_short() for hs in self._hed_strings]) kinds = ", ".join(["{} ({})".format(*k) for k in sorted(counter.items())]) kinds = (": " if len(kinds) > 0 else "") + kinds ch_specific = ", channel-specific" if self._any_ch_names() else "" From fd7933dff2920a8e99f0ed84999578149472bcec Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Thu, 10 Apr 2025 17:34:49 -0500 Subject: [PATCH 10/24] get validation working on __setitem__ --- mne/annotations.py | 87 ++++++++++++++++++----------------- mne/tests/test_annotations.py | 33 +++++++++++++ 2 files changed, 77 insertions(+), 43 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index fb5fdba0584..fb949554fae 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -762,6 +762,40 @@ def rename(self, mapping, verbose=None): return self +class _HEDStrings(list): + """Subclass of ndarray that will validate before setting.""" + + def __init__(self, *args, hed_version, **kwargs): + self._hed = _soft_import("hed", "validation of HED tags in annotations") + self._hed_schema_version = hed_version + super().__init__(*args, **kwargs) + for item in self: + self._validate_hed_string(item) + + def __setitem__(self, key, value): + """Validate value first, before assigning.""" + hs = self._validate_hed_string(value) + super().__setitem__(key, hs.get_as_original()) + + def _validate_hed_string(self, value): + # NB: must import; calling self._hed.validator.HedValidator doesn't work + from hed.validator import HedValidator + + # create HedString object and validate it + schema = self._hed.load_schema_version(self._hed_schema_version) + hs = self._hed.HedString(value, schema) + validator = HedValidator(schema) + # handle any errors + error_handler = self._hed.errors.ErrorHandler(check_for_warnings=False) + issues = validator.validate( + hs, allow_placeholders=False, error_handler=error_handler + ) + error_string = self._hed.get_printable_issue_string(issues) + if len(error_string): + raise ValueError(f"A HED string failed to validate:\n {error_string}") + return hs + + class HEDAnnotations(Annotations): """Annotations object for annotating segments of raw data with HED tags. @@ -810,12 +844,10 @@ def __init__( duration, description, hed_strings, - hed_version="8.3.0", # TODO @VisLab what is a sensible default here? + hed_version="8.3.0", orig_time=None, ch_names=None, ): - self.hed = _soft_import("hed", "validation of HED tags in annotations") - super().__init__( onset=onset, duration=duration, @@ -823,51 +855,20 @@ def __init__( orig_time=orig_time, ch_names=ch_names, ) - self.hed_version = hed_version - self._update_hed_strings(hed_strings=hed_strings) - - def _update_hed_strings(self, hed_strings): - # NB: must import; calling self.hed.validator.HedValidator doesn't work - from hed.validator import HedValidator - - if len(hed_strings) != len(self): - raise ValueError( - f"Number of HED strings ({len(hed_strings)}) must match the number of " - f"annotations ({len(self)})." - ) - # create HedString objects - schema = self.hed.load_schema_version(self.hed_version) - self._hed_strings = [self.hed.HedString(hs, schema) for hs in hed_strings] - # validation of HED strings - validator = HedValidator(schema) - error_handler = self.hed.errors.ErrorHandler(check_for_warnings=False) - issues = [ - validator.validate( - hs, allow_placeholders=False, error_handler=error_handler - ) - for hs in self._hed_strings - ] - error_strings = [self.hed.get_printable_issue_string(issue) for issue in issues] - if any(map(len, error_strings)): - raise ValueError( - "Some HED strings in your annotations failed to validate:\n - " - + "\n - ".join(error_strings) - ) - self.hed_strings = tuple( - hs.get_original_hed_string() for hs in self._hed_strings - ) + self.hed_strings = _HEDStrings(hed_strings, hed_version=hed_version) def __eq__(self, other): """Compare to another HEDAnnotations instance.""" return ( super().__eq__(self, other) + # TODO don't use array_equal, use HED validation approach? and np.array_equal(self.hed_strings, other.hed_strings) - and self.hed_version == other.hed_version + and self._hed_version == other._hed_version ) def __repr__(self): """Show a textual summary of the object.""" - counter = Counter([hs.get_as_short() for hs in self._hed_strings]) + counter = Counter(self.hed_strings) kinds = ", ".join(["{} ({})".format(*k) for k in sorted(counter.items())]) kinds = (": " if len(kinds) > 0 else "") + kinds ch_specific = ", channel-specific" if self._any_ch_names() else "" @@ -879,18 +880,18 @@ def __repr__(self): def __getitem__(self, key, *, with_ch_names=None): """Propagate indexing and slicing to the underlying numpy structure.""" - result = super().__getitem__(self, key, with_ch_names=with_ch_names) + result = super().__getitem__(key, with_ch_names=with_ch_names) if isinstance(result, OrderedDict): - result["hed_strings"] = self.hed_strings[key] + result._hed_strings = self.hed_strings[key] else: key = list(key) if isinstance(key, tuple) else key - hed_strings = self.hed_strings[key] + hed_strings = [self.hed_strings[key]] return HEDAnnotations( result.onset, result.duration, result.description, - hed_strings, - hed_version=self.hed_version, + hed_strings=hed_strings, + hed_version=self._hed_version, orig_time=self.orig_time, ch_names=result.ch_names, ) diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 4d0db170e2a..e6786e23c47 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -22,6 +22,7 @@ from mne import ( Annotations, Epochs, + HEDAnnotations, annotations_from_events, count_annotations, create_info, @@ -1825,3 +1826,35 @@ def test_append_splits_boundary(tmp_path, split_size): assert len(raw.annotations) == 2 assert raw.annotations.description[0] == "BAD boundary" assert_allclose(raw.annotations.onset, [onset] * 2) + + +def test_hed_annotations(): + """Test hed_strings validation.""" + ann = HEDAnnotations( + onset=[1, 2, 3], + duration=[0.1, 0.0, 0.3], + description=["a", "b", "c"], + hed_strings=[ + "Sensory-event, Experimental-stimulus, Visual-presentation, (Square, " + "DarkBlue, (Center-of, Computer-screen))", + "Sensory-event, Experimental-stimulus, Auditory-presentation, (Tone, " + "Frequency/550 Hz)", + "Agent-action, (Experiment-participant, (Press, Mouse-button))", + ], + ) + # test modifying + ann.hed_strings[0] = ( + "Sensory-event, (Word, Label/Word-look), Auditory-presentation, " + "Visual-presentation" + ) + # test modifying with bad value + with pytest.raises(ValueError, match="A HED string failed to validate"): + ann.hed_strings[0] = "foo" + # test initting with bad value + with pytest.raises(ValueError, match="A HED string failed to validate"): + ann = HEDAnnotations( + onset=[1], + duration=[0.1], + description=["a"], + hed_strings=["foo"], + ) From 456830a1ea92b90e21c186855669cff9e40c807a Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 11 Apr 2025 17:21:39 -0500 Subject: [PATCH 11/24] better repr, better test, use singular form --- mne/annotations.py | 51 ++++++++++++++++++++++++----------- mne/tests/test_annotations.py | 19 +++++++++---- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index fb949554fae..e2994f3fa3f 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -569,8 +569,8 @@ def _sort(self): self.duration = self.duration[order] self.description = self.description[order] self.ch_names = self.ch_names[order] - if hasattr(self, "hed_strings"): - self.hed_strings = self.hed_strings[order] + if hasattr(self, "hed_string"): + self.hed_string = self.hed_string[order] @verbose def crop( @@ -769,13 +769,13 @@ def __init__(self, *args, hed_version, **kwargs): self._hed = _soft_import("hed", "validation of HED tags in annotations") self._hed_schema_version = hed_version super().__init__(*args, **kwargs) - for item in self: - self._validate_hed_string(item) + self._objs = [self._validate_hed_string(item) for item in self] def __setitem__(self, key, value): """Validate value first, before assigning.""" hs = self._validate_hed_string(value) - super().__setitem__(key, hs.get_as_original()) + super().__setitem__(key, hs.get_original_hed_string()) + self._objs[key] = hs def _validate_hed_string(self, value): # NB: must import; calling self._hed.validator.HedValidator doesn't work @@ -793,6 +793,7 @@ def _validate_hed_string(self, value): error_string = self._hed.get_printable_issue_string(issues) if len(error_string): raise ValueError(f"A HED string failed to validate:\n {error_string}") + hs.sort() return hs @@ -810,7 +811,7 @@ class HEDAnnotations(Annotations): Array of strings containing description for each annotation. If a string, all the annotations are given the same description. To reject epochs, use description starting with keyword 'bad'. See example above. - hed_strings : array of str, shape (n_annotations,) | str + hed_string : array of str, shape (n_annotations,) | str Sequence of strings containing a HED tag (or comma-separated list of HED tags) for each annotation. If a single string is provided, all annotations are assigned the same HED string. @@ -843,7 +844,7 @@ def __init__( onset, duration, description, - hed_strings, + hed_string, hed_version="8.3.0", orig_time=None, ch_names=None, @@ -855,42 +856,60 @@ def __init__( orig_time=orig_time, ch_names=ch_names, ) - self.hed_strings = _HEDStrings(hed_strings, hed_version=hed_version) + self.hed_string = _HEDStrings(hed_string, hed_version=hed_version) def __eq__(self, other): """Compare to another HEDAnnotations instance.""" return ( super().__eq__(self, other) # TODO don't use array_equal, use HED validation approach? - and np.array_equal(self.hed_strings, other.hed_strings) + and np.array_equal(self.hed_string, other.hed_string) and self._hed_version == other._hed_version ) def __repr__(self): """Show a textual summary of the object.""" - counter = Counter(self.hed_strings) - kinds = ", ".join(["{} ({})".format(*k) for k in sorted(counter.items())]) - kinds = (": " if len(kinds) > 0 else "") + kinds + counter = Counter([hs.get_as_short() for hs in self.hed_string._objs]) + + def _shorten(text, width=74, placeholder=" ..."): + parts = text.split(",") + out = parts[0] + for part in parts[1:]: + # +1 for the comma ↓↓↓ + if width < len(out) + 1 + len(part) + len(placeholder): + break + out = f"{out},{part}" + return out + placeholder + + kinds = [ + f"{_shorten(k, width=74):<74} ({v})" for k, v in sorted(counter.items()) + ] + if len(kinds) > 5: + kinds = [*kinds[:5], f"... and {len(kinds) - 5} more"] + kinds = "\n ".join(kinds) + if len(kinds): + kinds = f":\n {kinds}\n" ch_specific = ", channel-specific" if self._any_ch_names() else "" s = ( f"HEDAnnotations | {len(self.onset)} segment" f"{_pl(len(self.onset))}{ch_specific}{kinds}" ) - return "<" + shorten(s, width=77, placeholder=" ...") + ">" + return f"<{s}>" def __getitem__(self, key, *, with_ch_names=None): """Propagate indexing and slicing to the underlying numpy structure.""" result = super().__getitem__(key, with_ch_names=with_ch_names) if isinstance(result, OrderedDict): - result._hed_strings = self.hed_strings[key] + result["hed_string"] = self.hed_string[key] + return result else: key = list(key) if isinstance(key, tuple) else key - hed_strings = [self.hed_strings[key]] + hed_string = [self.hed_string[key]] return HEDAnnotations( result.onset, result.duration, result.description, - hed_strings=hed_strings, + hed_string=hed_string, hed_version=self._hed_version, orig_time=self.orig_time, ch_names=result.ch_names, diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index e6786e23c47..7c57e25a891 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1834,7 +1834,7 @@ def test_hed_annotations(): onset=[1, 2, 3], duration=[0.1, 0.0, 0.3], description=["a", "b", "c"], - hed_strings=[ + hed_string=[ "Sensory-event, Experimental-stimulus, Visual-presentation, (Square, " "DarkBlue, (Center-of, Computer-screen))", "Sensory-event, Experimental-stimulus, Auditory-presentation, (Tone, " @@ -1843,18 +1843,27 @@ def test_hed_annotations(): ], ) # test modifying - ann.hed_strings[0] = ( + new_value = ( "Sensory-event, (Word, Label/Word-look), Auditory-presentation, " "Visual-presentation" ) + ann.hed_string[0] = new_value # test modifying with bad value with pytest.raises(ValueError, match="A HED string failed to validate"): - ann.hed_strings[0] = "foo" + ann.hed_string[0] = "foo" # test initting with bad value with pytest.raises(ValueError, match="A HED string failed to validate"): - ann = HEDAnnotations( + _ = HEDAnnotations( onset=[1], duration=[0.1], description=["a"], - hed_strings=["foo"], + hed_string=["foo"], ) + # test __getitem__ + first = ann[0] + assert first["hed_string"] == new_value + first["hed_string"] = "foo" # should not try to validate (is a copy) + assert ann.hed_string[0] == new_value + # test __repr__ + foo = repr(ann) + assert "Auditory-presentation,Experimental-stimulus,Sensory-event ..." in foo From 233f515df7b5662e812aa2b2c3842d66a26d0bc2 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Fri, 11 Apr 2025 17:26:02 -0500 Subject: [PATCH 12/24] simplify --- mne/annotations.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index e2994f3fa3f..e0be3ecf0b5 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -778,18 +778,12 @@ def __setitem__(self, key, value): self._objs[key] = hs def _validate_hed_string(self, value): - # NB: must import; calling self._hed.validator.HedValidator doesn't work - from hed.validator import HedValidator - # create HedString object and validate it schema = self._hed.load_schema_version(self._hed_schema_version) hs = self._hed.HedString(value, schema) - validator = HedValidator(schema) # handle any errors error_handler = self._hed.errors.ErrorHandler(check_for_warnings=False) - issues = validator.validate( - hs, allow_placeholders=False, error_handler=error_handler - ) + issues = hs.validate(allow_placeholders=False, error_handler=error_handler) error_string = self._hed.get_printable_issue_string(issues) if len(error_string): raise ValueError(f"A HED string failed to validate:\n {error_string}") From 5248e9ecbccde7ab2ca6e211cfee2c79fbc21c28 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 12:54:01 -0500 Subject: [PATCH 13/24] make it xrefable --- doc/api/events.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/events.rst b/doc/api/events.rst index 3f7159a22d5..88479eb8f3e 100644 --- a/doc/api/events.rst +++ b/doc/api/events.rst @@ -9,6 +9,7 @@ Events Annotations AcqParserFIF + HEDAnnotations concatenate_events count_events find_events From 7c55d59a1fcb9a39ff13e05cac2ee43477c36478 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 12:58:20 -0500 Subject: [PATCH 14/24] importorskip --- mne/tests/test_annotations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 7c57e25a891..88c3d4d14a5 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1830,6 +1830,7 @@ def test_append_splits_boundary(tmp_path, split_size): def test_hed_annotations(): """Test hed_strings validation.""" + pytest.importorskip("hed") ann = HEDAnnotations( onset=[1, 2, 3], duration=[0.1, 0.0, 0.3], From f99ac088ad227dafc325592b8ead88ea616e6cfa Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 15:23:11 -0500 Subject: [PATCH 15/24] docstring --- mne/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/annotations.py b/mne/annotations.py index e0be3ecf0b5..51205da4ad7 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -763,7 +763,7 @@ def rename(self, mapping, verbose=None): class _HEDStrings(list): - """Subclass of ndarray that will validate before setting.""" + """Subclass of list that will validate before __setitem__.""" def __init__(self, *args, hed_version, **kwargs): self._hed = _soft_import("hed", "validation of HED tags in annotations") From 15659569a197f672d9233b3c8250bb9ede6898f7 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 15:23:24 -0500 Subject: [PATCH 16/24] load schema only once --- mne/annotations.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 51205da4ad7..50110ed31b3 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -767,7 +767,7 @@ class _HEDStrings(list): def __init__(self, *args, hed_version, **kwargs): self._hed = _soft_import("hed", "validation of HED tags in annotations") - self._hed_schema_version = hed_version + self._schema = self._hed.load_schema_version(hed_version) super().__init__(*args, **kwargs) self._objs = [self._validate_hed_string(item) for item in self] @@ -779,8 +779,7 @@ def __setitem__(self, key, value): def _validate_hed_string(self, value): # create HedString object and validate it - schema = self._hed.load_schema_version(self._hed_schema_version) - hs = self._hed.HedString(value, schema) + hs = self._hed.HedString(value, self._schema) # handle any errors error_handler = self._hed.errors.ErrorHandler(check_for_warnings=False) issues = hs.validate(allow_placeholders=False, error_handler=error_handler) From aed0b69be8c6d9d861d71d839e2b1e3c304f9506 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 15:26:30 -0500 Subject: [PATCH 17/24] clean up test --- mne/tests/test_annotations.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 88c3d4d14a5..01e59f8155c 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1831,6 +1831,15 @@ def test_append_splits_boundary(tmp_path, split_size): def test_hed_annotations(): """Test hed_strings validation.""" pytest.importorskip("hed") + # test initting with bad value + with pytest.raises(ValueError, match="A HED string failed to validate"): + _ = HEDAnnotations( + onset=[1], + duration=[0.1], + description=["a"], + hed_string=["foo"], + ) + # test initting ann = HEDAnnotations( onset=[1, 2, 3], duration=[0.1, 0.0, 0.3], @@ -1843,28 +1852,20 @@ def test_hed_annotations(): "Agent-action, (Experiment-participant, (Press, Mouse-button))", ], ) + # test modifying with bad value + with pytest.raises(ValueError, match="A HED string failed to validate"): + ann.hed_string[0] = "foo" # test modifying new_value = ( "Sensory-event, (Word, Label/Word-look), Auditory-presentation, " "Visual-presentation" ) ann.hed_string[0] = new_value - # test modifying with bad value - with pytest.raises(ValueError, match="A HED string failed to validate"): - ann.hed_string[0] = "foo" - # test initting with bad value - with pytest.raises(ValueError, match="A HED string failed to validate"): - _ = HEDAnnotations( - onset=[1], - duration=[0.1], - description=["a"], - hed_string=["foo"], - ) # test __getitem__ first = ann[0] assert first["hed_string"] == new_value first["hed_string"] = "foo" # should not try to validate (is a copy) assert ann.hed_string[0] == new_value # test __repr__ - foo = repr(ann) - assert "Auditory-presentation,Experimental-stimulus,Sensory-event ..." in foo + _repr = repr(ann) + assert "Auditory-presentation,Experimental-stimulus,Sensory-event ..." in _repr From 5959c646b15f1ba324da8c345c20e4eca21f5fe9 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 15:27:40 -0500 Subject: [PATCH 18/24] fill doc --- mne/annotations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/annotations.py b/mne/annotations.py index 50110ed31b3..8a0a2148afc 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -790,6 +790,7 @@ def _validate_hed_string(self, value): return hs +@fill_doc class HEDAnnotations(Annotations): """Annotations object for annotating segments of raw data with HED tags. From 30aaade30ed75c9df8c10e2337d8236baf085564 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 15:30:24 -0500 Subject: [PATCH 19/24] codecomment --- mne/annotations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mne/annotations.py b/mne/annotations.py index 8a0a2148afc..d598cd66351 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -865,6 +865,7 @@ def __repr__(self): """Show a textual summary of the object.""" counter = Counter([hs.get_as_short() for hs in self.hed_string._objs]) + # textwrap.shorten won't work: we remove all spaces and shouldn't split on `-` def _shorten(text, width=74, placeholder=" ..."): parts = text.split(",") out = parts[0] From 0c7961892fe434733c226a20fef557bb31ee2cb5 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 16 Apr 2025 17:32:14 -0500 Subject: [PATCH 20/24] serialization --- mne/annotations.py | 51 ++++++++++++++++++++++++++--------- mne/tests/test_annotations.py | 7 ++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index d598cd66351..ebe479f51ce 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -13,6 +13,7 @@ from textwrap import shorten import numpy as np +from packaging.version import Version from scipy.io import loadmat from ._fiff.constants import FIFF @@ -769,17 +770,17 @@ def __init__(self, *args, hed_version, **kwargs): self._hed = _soft_import("hed", "validation of HED tags in annotations") self._schema = self._hed.load_schema_version(hed_version) super().__init__(*args, **kwargs) - self._objs = [self._validate_hed_string(item) for item in self] + self._objs = [self._validate_hed_string(item, self._schema) for item in self] def __setitem__(self, key, value): """Validate value first, before assigning.""" - hs = self._validate_hed_string(value) + hs = self._validate_hed_string(value, self._schema) super().__setitem__(key, hs.get_original_hed_string()) self._objs[key] = hs - def _validate_hed_string(self, value): + def _validate_hed_string(self, value, schema): # create HedString object and validate it - hs = self._hed.HedString(value, self._schema) + hs = self._hed.HedString(value, schema) # handle any errors error_handler = self._hed.errors.ErrorHandler(check_for_warnings=False) issues = hs.validate(allow_placeholders=False, error_handler=error_handler) @@ -850,16 +851,18 @@ def __init__( orig_time=orig_time, ch_names=ch_names, ) - self.hed_string = _HEDStrings(hed_string, hed_version=hed_version) + self._hed_version = hed_version + self.hed_string = _HEDStrings(hed_string, hed_version=self._hed_version) def __eq__(self, other): """Compare to another HEDAnnotations instance.""" - return ( - super().__eq__(self, other) - # TODO don't use array_equal, use HED validation approach? - and np.array_equal(self.hed_string, other.hed_string) - and self._hed_version == other._hed_version - ) + _slf = self.hed_string + _oth = other.hed_string + if Version(self._hed_version) < Version(other._hed_version): + _slf = [_slf._validate_hed_string(v, _oth._schema) for v in _slf._objs] + elif Version(self._hed_version) > Version(other._hed_version): + _oth = [_oth._validate_hed_string(v, _slf._schema) for v in _oth._objs] + return super().__eq__(other) and _slf == _oth def __repr__(self): """Show a textual summary of the object.""" @@ -892,7 +895,7 @@ def _shorten(text, width=74, placeholder=" ..."): return f"<{s}>" def __getitem__(self, key, *, with_ch_names=None): - """Propagate indexing and slicing to the underlying numpy structure.""" + """Propagate indexing and slicing to the underlying NumPy structure.""" result = super().__getitem__(key, with_ch_names=with_ch_names) if isinstance(result, OrderedDict): result["hed_string"] = self.hed_string[key] @@ -910,6 +913,30 @@ def __getitem__(self, key, *, with_ch_names=None): ch_names=result.ch_names, ) + def __getstate__(self): + """Make serialization work, by removing module reference.""" + return dict( + _orig_time=self._orig_time, + onset=self.onset, + duration=self.duration, + description=self.description, + ch_names=self.ch_names, + hed_string=list(self.hed_string), + _hed_version=self._hed_version, + ) + + def __setstate__(self, state): + """Unpack from serialized format.""" + self._orig_time = state["_orig_time"] + self.onset = state["onset"] + self.duration = state["duration"] + self.description = state["description"] + self.ch_names = state["ch_names"] + self._hed_version = state["_hed_version"] + self.hed_string = _HEDStrings( + state["hed_string"], hed_version=self._hed_version + ) + def append(self, onset, duration, description, ch_names=None): """TODO.""" pass diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 01e59f8155c..48d3cef34a1 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1855,12 +1855,17 @@ def test_hed_annotations(): # test modifying with bad value with pytest.raises(ValueError, match="A HED string failed to validate"): ann.hed_string[0] = "foo" - # test modifying + # test modifying and __eq__ + foo = ann.copy() + assert ann == foo new_value = ( "Sensory-event, (Word, Label/Word-look), Auditory-presentation, " "Visual-presentation" ) + foo.hed_string[0] = new_value + assert ann != foo ann.hed_string[0] = new_value + assert ann == foo # test __getitem__ first = ann[0] assert first["hed_string"] == new_value From a2c01fccdbaa38127fe44fbb1bf8f6dad7dbbc3f Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Mon, 21 Apr 2025 10:26:51 -0500 Subject: [PATCH 21/24] use internal version comparator --- mne/annotations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index ebe479f51ce..2ed73b516ec 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -13,7 +13,6 @@ from textwrap import shorten import numpy as np -from packaging.version import Version from scipy.io import loadmat from ._fiff.constants import FIFF @@ -30,6 +29,7 @@ write_name_list_sanitized, write_string, ) +from .fixes import _compare_version from .utils import ( _check_dict_keys, _check_dt, @@ -858,9 +858,10 @@ def __eq__(self, other): """Compare to another HEDAnnotations instance.""" _slf = self.hed_string _oth = other.hed_string - if Version(self._hed_version) < Version(other._hed_version): + + if _compare_version(self._hed_version, "<", other._hed_version): _slf = [_slf._validate_hed_string(v, _oth._schema) for v in _slf._objs] - elif Version(self._hed_version) > Version(other._hed_version): + elif _compare_version(self._hed_version, ">", other._hed_version): _oth = [_oth._validate_hed_string(v, _slf._schema) for v in _oth._objs] return super().__eq__(other) and _slf == _oth From ef9d563eaf09476b7586c172e6df575a99d3ba85 Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 29 Apr 2025 10:51:40 -0500 Subject: [PATCH 22/24] get append and sort working --- mne/annotations.py | 46 +++++++++++++++++++++++++--- mne/tests/test_annotations.py | 57 ++++++++++++++++++++++------------- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 2ed73b516ec..9f772b79553 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -571,7 +571,11 @@ def _sort(self): self.description = self.description[order] self.ch_names = self.ch_names[order] if hasattr(self, "hed_string"): - self.hed_string = self.hed_string[order] + self.hed_string._objs = [self.hed_string._objs[i] for i in order] + for i in order: + self.hed_string.__setitem__( + i, self.hed_string._objs[i].get_original_hed_string() + ) @verbose def crop( @@ -790,6 +794,12 @@ def _validate_hed_string(self, value, schema): hs.sort() return hs + def append(self, item): + """Append an item to the end of the HEDString list.""" + hs = self._validate_hed_string(item, self._schema) + super().append(hs.get_original_hed_string()) + self._objs.append(hs) + @fill_doc class HEDAnnotations(Annotations): @@ -896,7 +906,7 @@ def _shorten(text, width=74, placeholder=" ..."): return f"<{s}>" def __getitem__(self, key, *, with_ch_names=None): - """Propagate indexing and slicing to the underlying NumPy structure.""" + """Propagate indexing and slicing to the underlying structure.""" result = super().__getitem__(key, with_ch_names=with_ch_names) if isinstance(result, OrderedDict): result["hed_string"] = self.hed_string[key] @@ -938,9 +948,35 @@ def __setstate__(self, state): state["hed_string"], hed_version=self._hed_version ) - def append(self, onset, duration, description, ch_names=None): - """TODO.""" - pass + @fill_doc + def append(self, *, onset, duration, description, hed_string, ch_names=None): + """Add an annotated segment. Operates inplace. + + Parameters + ---------- + onset : float | array-like + Annotation time onset from the beginning of the recording in + seconds. + duration : float | array-like + Duration of the annotation in seconds. + description : str | array-like + Description for the annotation. To reject epochs, use description + starting with keyword 'bad'. + hed_string : array of str, shape (n_annotations,) | str + Sequence of strings containing a HED tag (or comma-separated list of HED + tags) for each annotation. If a single string is provided, all annotations + are assigned the same HED string. + %(ch_names_annot)s + + Returns + ------- + self : mne.HEDAnnotations + The modified HEDAnnotations object. + """ + self.hed_string.append(hed_string) + super().append( + onset=onset, duration=duration, description=description, ch_names=ch_names + ) def count(self): """TODO. Unlike Annotations.count, keys should be HED tags not descriptions.""" diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 48d3cef34a1..b52248a2510 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1832,45 +1832,60 @@ def test_hed_annotations(): """Test hed_strings validation.""" pytest.importorskip("hed") # test initting with bad value - with pytest.raises(ValueError, match="A HED string failed to validate"): + validation_fail_msg = "A HED string failed to validate" + with pytest.raises(ValueError, match=validation_fail_msg): _ = HEDAnnotations( onset=[1], duration=[0.1], description=["a"], hed_string=["foo"], ) - # test initting + # test initting with good values + good_values = dict( + square="Sensory-event, Experimental-stimulus, Visual-presentation, (Square, " + "DarkBlue, (Center-of, Computer-screen))", # extra spaces intentional + tone="Sensory-event, Experimental-stimulus, Auditory-presentation, (Tone, " + "Frequency/550 Hz)", + press="Agent-action, (Experiment-participant, (Press, Mouse-button))", + word="Sensory-event, (Word, Label/Word-look), Auditory-presentation, " + "Visual-presentation", + ) ann = HEDAnnotations( - onset=[1, 2, 3], + onset=[3, 2, 1], duration=[0.1, 0.0, 0.3], - description=["a", "b", "c"], - hed_string=[ - "Sensory-event, Experimental-stimulus, Visual-presentation, (Square, " - "DarkBlue, (Center-of, Computer-screen))", - "Sensory-event, Experimental-stimulus, Auditory-presentation, (Tone, " - "Frequency/550 Hz)", - "Agent-action, (Experiment-participant, (Press, Mouse-button))", - ], + description=["d", "c", "a"], + hed_string=[good_values["square"], good_values["tone"], good_values["press"]], ) + # test appending + foo = ann.copy() + ons_dur_desc = dict(onset=1.5, duration=0.2, description="b") + with pytest.raises(ValueError, match=validation_fail_msg): + foo.append(**ons_dur_desc, hed_string="foo") + foo.append(**ons_dur_desc, hed_string=good_values["word"]) + # make sure that internal sorting by onset didn't mess up the relative orders of + # self.hed_string and self.hed_string._objs + assert list(foo.hed_string) == [ + x.get_original_hed_string() for x in foo.hed_string._objs + ] + # make sure we didn't mess up the type of the HEDStrings + assert isinstance(foo.hed_string, mne.annotations._HEDStrings) # test modifying with bad value - with pytest.raises(ValueError, match="A HED string failed to validate"): + with pytest.raises(ValueError, match=validation_fail_msg): ann.hed_string[0] = "foo" # test modifying and __eq__ foo = ann.copy() assert ann == foo - new_value = ( - "Sensory-event, (Word, Label/Word-look), Auditory-presentation, " - "Visual-presentation" - ) - foo.hed_string[0] = new_value + foo.hed_string[0] = good_values["word"] assert ann != foo - ann.hed_string[0] = new_value + ann.hed_string[0] = good_values["word"] assert ann == foo # test __getitem__ first = ann[0] - assert first["hed_string"] == new_value - first["hed_string"] = "foo" # should not try to validate (is a copy) - assert ann.hed_string[0] == new_value + assert first["hed_string"] == good_values["word"] + # setting bad value on extracted OrderedDict won't try to validate: + first["hed_string"] = "foo" + # ...and won't affect the original object + assert ann.hed_string[0] == good_values["word"] # test __repr__ _repr = repr(ann) assert "Auditory-presentation,Experimental-stimulus,Sensory-event ..." in _repr From 4a5364e9d1d220828b0ff6c038a12bd5af5455bf Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 29 Apr 2025 11:43:15 -0500 Subject: [PATCH 23/24] get count working --- mne/annotations.py | 11 ++++------- mne/tests/test_annotations.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 9f772b79553..1f037b44310 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -854,6 +854,8 @@ def __init__( orig_time=None, ch_names=None, ): + self._hed_version = hed_version + self.hed_string = _HEDStrings(hed_string, hed_version=self._hed_version) super().__init__( onset=onset, duration=duration, @@ -861,8 +863,6 @@ def __init__( orig_time=orig_time, ch_names=ch_names, ) - self._hed_version = hed_version - self.hed_string = _HEDStrings(hed_string, hed_version=self._hed_version) def __eq__(self, other): """Compare to another HEDAnnotations instance.""" @@ -978,10 +978,6 @@ def append(self, *, onset, duration, description, hed_string, ch_names=None): onset=onset, duration=duration, description=description, ch_names=ch_names ) - def count(self): - """TODO. Unlike Annotations.count, keys should be HED tags not descriptions.""" - pass - def crop( self, tmin=None, tmax=None, emit_warning=False, use_orig_time=True, verbose=None ): @@ -1971,5 +1967,6 @@ def count_annotations(annotations): >>> count_annotations(annotations) {'T0': 2, 'T1': 1} """ - types, counts = np.unique(annotations.description, return_counts=True) + field = "hed_string" if isinstance(annotations, HEDAnnotations) else "description" + types, counts = np.unique(getattr(annotations, field), return_counts=True) return {str(t): int(count) for t, count in zip(types, counts)} diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index b52248a2510..e9fcb62a838 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1856,14 +1856,16 @@ def test_hed_annotations(): description=["d", "c", "a"], hed_string=[good_values["square"], good_values["tone"], good_values["press"]], ) + # make sure sorting by onset worked correctly + assert ann.hed_string[0] == good_values["press"] + assert ann.hed_string._objs[0].get_original_hed_string() == good_values["press"] # test appending foo = ann.copy() ons_dur_desc = dict(onset=1.5, duration=0.2, description="b") with pytest.raises(ValueError, match=validation_fail_msg): foo.append(**ons_dur_desc, hed_string="foo") foo.append(**ons_dur_desc, hed_string=good_values["word"]) - # make sure that internal sorting by onset didn't mess up the relative orders of - # self.hed_string and self.hed_string._objs + # make sure sorting by onset also works for .append() assert list(foo.hed_string) == [ x.get_original_hed_string() for x in foo.hed_string._objs ] @@ -1879,6 +1881,13 @@ def test_hed_annotations(): assert ann != foo ann.hed_string[0] = good_values["word"] assert ann == foo + # test .count() + want_counts = { + good_values["word"]: 1, + good_values["tone"]: 1, + good_values["square"]: 1, + } + assert ann.count() == want_counts # test __getitem__ first = ann[0] assert first["hed_string"] == good_values["word"] From 308b9337020e30d4361362cbee5fff03dc954b4d Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Tue, 29 Apr 2025 11:59:42 -0500 Subject: [PATCH 24/24] get delete() working --- mne/annotations.py | 13 +++++++++++-- mne/tests/test_annotations.py | 5 ++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 1f037b44310..71fa0267d65 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -985,8 +985,17 @@ def crop( pass def delete(self, idx): - """TODO.""" - pass + """Remove an annotation. Operates inplace. + + Parameters + ---------- + idx : int | array-like of int + Index of the annotation to remove. Can be array-like to remove multiple + indices. + """ + _ = self.hed_string._objs.pop(idx) + _ = self.hed_string.pop(idx) + super().delete(idx) def to_data_frame(self, time_format="datetime"): """TODO.""" diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index e9fcb62a838..823aed20556 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1874,13 +1874,16 @@ def test_hed_annotations(): # test modifying with bad value with pytest.raises(ValueError, match=validation_fail_msg): ann.hed_string[0] = "foo" - # test modifying and __eq__ + # test modifying, __eq__, and delete() foo = ann.copy() assert ann == foo foo.hed_string[0] = good_values["word"] assert ann != foo ann.hed_string[0] = good_values["word"] assert ann == foo + foo.delete(0) + assert ann != foo + assert foo.hed_string[0] == ann.hed_string[1] # test .count() want_counts = { good_values["word"]: 1,