diff --git a/examples/transforms/transforms.py b/examples/transforms/transforms.py deleted file mode 100644 index c03f62c1..00000000 --- a/examples/transforms/transforms.py +++ /dev/null @@ -1,78 +0,0 @@ -from dataclasses import dataclass -from typing import Any - -import numpy as np - -import nnbench -from nnbench.reporter.file import FileIO -from nnbench.transforms import OneToOneTransform -from nnbench.types import BenchmarkRecord - - -class MyModel: - def __init__(self, checksum: str): - self.checksum = checksum - - def apply(self, data: np.ndarray) -> float: - return data.mean() - - def to_json(self) -> dict[str, Any]: - return {"checksum": self.checksum} - - @classmethod - def from_json(cls, obj: dict[str, Any]) -> "MyModel": - # intentionally fail if no checksum is given. - return cls(checksum=obj["checksum"]) - - -@nnbench.benchmark -def accuracy(model: MyModel, data: np.ndarray) -> float: - return model.apply(data) - - -class MyTransform(OneToOneTransform): - def apply(self, record: BenchmarkRecord) -> BenchmarkRecord: - """Apply this transform on a record.""" - for b in record.benchmarks: - params: dict[str, Any] = b["parameters"] - b["parameters"] = { - "model": params["model"].to_json(), - "data": params["data"].tolist(), - } - return record - - def iapply(self, record: BenchmarkRecord) -> BenchmarkRecord: - """Apply the inverse of this transform.""" - for b in record.benchmarks: - params: dict[str, Any] = b["parameters"] - b["parameters"] = { - "model": MyModel.from_json(params["model"]), - "data": np.asarray(params["data"]), - } - return record - - -def main(): - @dataclass(frozen=True) - class MyParams(nnbench.Parameters): - model: MyModel - data: np.ndarray - - runner = nnbench.BenchmarkRunner() - - m = MyModel(checksum="12345") - data = np.random.random_sample((10,)) - params = MyParams(m, data) - record = runner.run(__name__, params=params) - - transform = MyTransform() - trecord = transform.apply(record) - f = FileIO() - f.write(trecord, "record.json") - - record2 = f.read("record.json") - new_record = transform.iapply(record2) - - -if __name__ == "__main__": - main() diff --git a/src/nnbench/transforms/__init__.py b/src/nnbench/transforms/__init__.py deleted file mode 100644 index 45d380f0..00000000 --- a/src/nnbench/transforms/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import ManyToManyTransform, ManyToOneTransform, OneToOneTransform diff --git a/src/nnbench/transforms/base.py b/src/nnbench/transforms/base.py deleted file mode 100644 index b21e8e69..00000000 --- a/src/nnbench/transforms/base.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Metaclasses for defining transforms acting on benchmark records.""" - -from abc import ABC, abstractmethod -from collections.abc import Sequence - -from nnbench.types import BenchmarkRecord - - -class Transform: - """The basic transform which every transform has to inherit from.""" - - invertible: bool = True - """ - Whether this transform is invertible, - i.e. records can be converted back and forth with no changes or data loss. - """ - pass - - -class OneToOneTransform(ABC, Transform): - @abstractmethod - def apply(self, record: BenchmarkRecord) -> BenchmarkRecord: - """Apply this transform to a benchmark record. - - Parameters - ---------- - record: BenchmarkRecord - Benchmark record to apply the transform on. - - Returns - ------- - BenchmarkRecord - The transformed benchmark record. - """ - - def iapply(self, record: BenchmarkRecord) -> BenchmarkRecord: - """Apply the inverse of this transform. - - In general, applying the inverse on a record not previously transformed - may yield unexpected results. - - Parameters - ---------- - record: BenchmarkRecord - Benchmark record to apply the inverse transform on. - - Returns - ------- - BenchmarkRecord - The inversely transformed benchmark record. - - Raises - ------ - RuntimeError - If the `Transform.invertible` attribute is set to `False`. - """ - if not self.invertible: - raise RuntimeError(f"{self.__class__.__name__}() is marked as not invertible") - raise NotImplementedError - - -class ManyToOneTransform(Transform): - """A many-to-one transform reducing a collection of records to a single record. - - This is useful for computing statistics on a collection of runs. - """ - - @abstractmethod - def apply(self, record: Sequence[BenchmarkRecord]) -> BenchmarkRecord: - """Apply this transform to a benchmark record. - - Parameters - ---------- - record: Sequence[BenchmarkRecord] - A sequence of benchmark record to apply the transform on, - yielding a single resulting record. - - Returns - ------- - BenchmarkRecord - The transformed (reduced) benchmark record. - """ - - def iapply(self, record: BenchmarkRecord) -> Sequence[BenchmarkRecord]: - """Apply the inverse of this transform. - - In general, applying the inverse on a record not previously transformed - may yield unexpected results. - - Parameters - ---------- - record: BenchmarkRecord - Benchmark record to apply the inverse transform on. - - Returns - ------- - Sequence[BenchmarkRecord] - The inversely transformed benchmark record sequence. - - Raises - ------ - RuntimeError - If the `Transform.invertible` attribute is set to `False`. - """ - if not self.invertible: - raise RuntimeError(f"{self.__class__.__name__}() is marked as not invertible") - raise NotImplementedError - - -class ManyToManyTransform(Transform): - """A many-to-many transform mapping an input record collection to an output collection. - - Use this to programmatically wrangle metadata or types in records, or to - convert parameters into database-ready representations. - """ - - length_invariant: bool = True - """ - Whether this transform preserves the number of records, i.e. no records are dropped. - """ - - @abstractmethod - def apply(self, record: Sequence[BenchmarkRecord]) -> Sequence[BenchmarkRecord]: - """Apply this transform to a benchmark record. - - Parameters - ---------- - record: Sequence[BenchmarkRecord] - A sequence of benchmark record to apply the transform on. - - Returns - ------- - Sequence[BenchmarkRecord] - The transformed benchmark record sequence. - """ - - def iapply(self, record: Sequence[BenchmarkRecord]) -> Sequence[BenchmarkRecord]: - """Apply the inverse of this transform. - - In general, applying the inverse on a record not previously transformed - may yield unexpected results. - - Parameters - ---------- - record: Sequence[BenchmarkRecord] - A sequence of benchmark record to apply the transform on. - - Returns - ------- - Sequence[BenchmarkRecord] - The inversely transformed benchmark record sequence. - - Raises - ------ - RuntimeError - If the `Transform.invertible` attribute is set to `False`. - """ - if not self.invertible: - raise RuntimeError(f"{self.__class__.__name__}() is marked as not invertible") - raise NotImplementedError diff --git a/src/nnbench/transforms/params.py b/src/nnbench/transforms/params.py deleted file mode 100644 index 82d7b102..00000000 --- a/src/nnbench/transforms/params.py +++ /dev/null @@ -1,49 +0,0 @@ -from collections.abc import Sequence -from typing import Any - -from nnbench.transforms import ManyToManyTransform, OneToOneTransform -from nnbench.types import BenchmarkRecord - - -class CompressionMixin: - def compress(self, params: dict[str, Any]) -> dict[str, Any]: - containers = (tuple, list, set, frozenset) - natives = (float, int, str, bool, bytes, complex) - compressed: dict[str, Any] = {} - - def _compress_impl(val): - if isinstance(val, natives): - # save native types without modification... - return val - else: - # ... or return the string repr. - # TODO: Allow custom representations for types with formatters. - return repr(val) - - for k, v in params.items(): - if isinstance(v, containers): - container_type = type(v) - compressed[k] = container_type(_compress_impl(vv) for vv in v) - elif isinstance(v, dict): - compressed[k] = self.compress(v) - else: - compressed[k] = _compress_impl(v) - - return compressed - - -class ParameterCompression1to1(OneToOneTransform, CompressionMixin): - def apply(self, record: BenchmarkRecord) -> BenchmarkRecord: - for bm in record.benchmarks: - bm["params"] = self.compress(bm["params"]) - - return record - - -class ParameterCompressionNtoN(ManyToManyTransform, CompressionMixin): - def apply(self, record: Sequence[BenchmarkRecord]) -> Sequence[BenchmarkRecord]: - for rec in record: - for bm in rec.benchmarks: - bm["params"] = self.compress(bm["params"]) - - return record