Skip to content

dqm_ml_core.models

Data model definitions for DQM-ML configuration and output schemas.

This module re-exports all public model classes for processor configurations, column mappings, dataloader settings, compute/storage/error handling, interface definitions, and output specifications.

ProcessorConfig = Annotated[ImageFeaturesProcessorConfig | FeaturesEmbeddingsProcessorConfig | CompletenessProcessorConfig | RepresentativenessProcessorConfig | DiversityProcessorConfig | DomainGapProcessorConfig, Field(discriminator='type')] module-attribute

__all__ = ['ColumnRename', 'ColumnsConfig', 'CompletenessProcessorConfig', 'ComputeConfig', 'DataLoaderConfig', 'DataLoadersConfig', 'DiversityProcessorConfig', 'DomainGapProcessorConfig', 'ErrorsConfig', 'FeaturesEmbeddingsProcessorConfig', 'FeaturesInterfaceConfig', 'FeaturesOutputsConfig', 'FilterConfig', 'GapInterfaceConfig', 'GapOutputsConfig', 'HistogramConfig', 'ImageErrorsConfig', 'ImageFeaturesProcessorConfig', 'JobConfig', 'MetricsInterfaceConfig', 'MetricsOutputsConfig', 'ProcessorConfig', 'RepresentativenessProcessorConfig', 'RetryConfig', 'SamplePathConfig', 'SplitConfig', 'StorageConfig', 'TabularErrorsConfig', 'TransformConfig', 'TransformType'] module-attribute

ColumnRename

Bases: BaseModel

Mapping from an original column name to a new name.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/columns.py 
10
11
12
13
14
15
16
class ColumnRename(BaseModel):
    """Mapping from an original column name to a new name."""

    model_config = ConfigDict(extra="forbid")

    from_: str = Field(alias="from", description="Original column name.")
    to: str = Field(description="New column name.")

from_: str = Field(alias='from', description='Original column name.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

to: str = Field(description='New column name.') class-attribute instance-attribute

ColumnsConfig

Bases: BaseModel

Column selection, exclusion, renaming, and prefix/suffix operations.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/columns.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
class ColumnsConfig(BaseModel):
    """Column selection, exclusion, renaming, and prefix/suffix operations."""

    model_config = ConfigDict(extra="forbid")

    input: list[str] = Field(
        default=[],
        description="Columns to read (fnmatch patterns supported). [] reads all.",
    )
    exclude: list[str] | None = Field(
        default=None,
        description="Columns to exclude (fnmatch patterns supported).",
    )
    rename: list[ColumnRename] | None = None
    prefix: str = Field(default="", description="Prefix prepended to each column name.")
    suffix: str = Field(default="", description="Suffix appended to each column name.")

exclude: list[str] | None = Field(default=None, description='Columns to exclude (fnmatch patterns supported).') class-attribute instance-attribute

input: list[str] = Field(default=[], description='Columns to read (fnmatch patterns supported). [] reads all.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

prefix: str = Field(default='', description='Prefix prepended to each column name.') class-attribute instance-attribute

rename: list[ColumnRename] | None = None class-attribute instance-attribute

suffix: str = Field(default='', description='Suffix appended to each column name.') class-attribute instance-attribute

CompletenessProcessorConfig

Bases: _ProcessorBase

Configuration for completeness metric computation.

Computes per-column and overall completeness (non-null) metrics.

Attributes:

Name Type Description
type Literal['completeness']

Processor type discriminator ("completeness").
include_per_column bool

Include per-column completeness scores in output.
include_overall bool

Include overall completeness score in output.
include_metadata bool

Include metadata (total rows, null counts) in output.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
class CompletenessProcessorConfig(_ProcessorBase):
    """Configuration for completeness metric computation.

    Computes per-column and overall completeness (non-null) metrics.

    Attributes:
        type: Processor type discriminator ("completeness").
        include_per_column: Include per-column completeness scores in output.
        include_overall: Include overall completeness score in output.
        include_metadata: Include metadata (total rows, null counts) in output.
    """

    type: Literal["completeness"] = "completeness"
    include_per_column: bool = Field(default=True, description="Include per-column completeness scores.")
    include_overall: bool = Field(default=True, description="Include overall completeness score.")
    include_metadata: bool = Field(default=False, description="Include metadata in output.")

include_metadata: bool = Field(default=False, description='Include metadata in output.') class-attribute instance-attribute

include_overall: bool = Field(default=True, description='Include overall completeness score.') class-attribute instance-attribute

include_per_column: bool = Field(default=True, description='Include per-column completeness scores.') class-attribute instance-attribute

type: Literal['completeness'] = 'completeness' class-attribute instance-attribute

ComputeConfig

Bases: BaseModel

Global compute / runtime settings.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
class ComputeConfig(BaseModel):
    """Global compute / runtime settings."""

    model_config = ConfigDict(extra="forbid")

    seed: int = Field(default=42, description="Random seed for reproducibility.")
    log_level: Literal["debug", "info", "warning", "error"] = Field(
        default="warning",
        description="Logging verbosity.",
    )
    max_memory: str | None = Field(default=None, description="Maximum memory per worker (e.g. '4Gi').")
    device: Literal["auto", "cpu", "cuda"] = Field(
        default="auto",
        description="Compute device: 'auto' picks cuda if available.",
    )
    progress_bar: bool = Field(default=True, description="Show tqdm progress bars.")
    threads: int = Field(default=4, gt=0, description="Number of worker threads.")

device: Literal['auto', 'cpu', 'cuda'] = Field(default='auto', description="Compute device: 'auto' picks cuda if available.") class-attribute instance-attribute

log_level: Literal['debug', 'info', 'warning', 'error'] = Field(default='warning', description='Logging verbosity.') class-attribute instance-attribute

max_memory: str | None = Field(default=None, description="Maximum memory per worker (e.g. '4Gi').") class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

progress_bar: bool = Field(default=True, description='Show tqdm progress bars.') class-attribute instance-attribute

seed: int = Field(default=42, description='Random seed for reproducibility.') class-attribute instance-attribute

threads: int = Field(default=4, gt=0, description='Number of worker threads.') class-attribute instance-attribute

DataLoaderConfig

Bases: BaseModel

Configuration for a single dataloader (Parquet or CSV).

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class DataLoaderConfig(BaseModel):
    """Configuration for a single dataloader (Parquet or CSV)."""

    model_config = ConfigDict(extra="forbid")

    name: str = Field(description="Unique name for this dataloader.")
    type: Literal["parquet", "csv"] = Field(description="Data file format.")
    path: str = Field(description="Glob pattern or path to data files.")
    id_column: str | None = Field(default=None, description="Column used as row identifier.")
    batch_size: int = Field(default=10000, description="Number of rows per batch.")
    filters: list[FilterConfig] | None = None
    sample_path: list[SamplePathConfig] | None = None
    split: SplitConfig | None = None
    transform: list[TransformConfig] | None = None
    storage: StorageConfig | None = None

batch_size: int = Field(default=10000, description='Number of rows per batch.') class-attribute instance-attribute

filters: list[FilterConfig] | None = None class-attribute instance-attribute

id_column: str | None = Field(default=None, description='Column used as row identifier.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

name: str = Field(description='Unique name for this dataloader.') class-attribute instance-attribute

path: str = Field(description='Glob pattern or path to data files.') class-attribute instance-attribute

sample_path: list[SamplePathConfig] | None = None class-attribute instance-attribute

split: SplitConfig | None = None class-attribute instance-attribute

storage: StorageConfig | None = None class-attribute instance-attribute

transform: list[TransformConfig] | None = None class-attribute instance-attribute

type: Literal['parquet', 'csv'] = Field(description='Data file format.') class-attribute instance-attribute

DataLoadersConfig

Bases: BaseModel

Collection of dataloaders for a job.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
90
91
92
93
94
95
96
97
98
99
class DataLoadersConfig(BaseModel):
    """Collection of dataloaders for a job."""

    model_config = ConfigDict(extra="forbid")

    storage: StorageConfig | None = Field(
        default=None,
        description="Default storage config inherited by all loaders.",
    )
    loaders: list[DataLoaderConfig] = Field(description="List of dataloader configurations.")

loaders: list[DataLoaderConfig] = Field(description='List of dataloader configurations.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

storage: StorageConfig | None = Field(default=None, description='Default storage config inherited by all loaders.') class-attribute instance-attribute

DiversityProcessorConfig

Bases: _ProcessorBase

Configuration for diversity metric computation.

Computes diversity metrics: Simpson, Gini, Shannon entropy, and richness.

Attributes:

Name Type Description
type Literal['diversity']

Processor type discriminator ("diversity").
metrics list[str]

List of diversity metrics to compute.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
class DiversityProcessorConfig(_ProcessorBase):
    """Configuration for diversity metric computation.

    Computes diversity metrics: Simpson, Gini, Shannon entropy, and richness.

    Attributes:
        type: Processor type discriminator ("diversity").
        metrics: List of diversity metrics to compute.
    """

    type: Literal["diversity"] = "diversity"
    metrics: list[str] = Field(
        default=["simpson", "gini", "shannon", "richness"],
        description="List of diversity metrics to compute.",
    )

metrics: list[str] = Field(default=['simpson', 'gini', 'shannon', 'richness'], description='List of diversity metrics to compute.') class-attribute instance-attribute

type: Literal['diversity'] = 'diversity' class-attribute instance-attribute

DomainGapProcessorConfig

Bases: _ProcessorBase

Configuration for domain-gap (distribution shift) measurement.

Measures distribution shift between datasets using a configured distance metric.

Attributes:

Name Type Description
type Literal['domain_gap']

Processor type discriminator ("domain_gap").
columns ColumnsConfig

Column input configuration (required).
distance DistanceConfig

Distance metric configuration.
summary SummaryConfig | None

Embedding summary configuration.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
class DomainGapProcessorConfig(_ProcessorBase):
    """Configuration for domain-gap (distribution shift) measurement.

    Measures distribution shift between datasets using a configured distance metric.

    Attributes:
        type: Processor type discriminator ("domain_gap").
        columns: Column input configuration (required).
        distance: Distance metric configuration.
        summary: Embedding summary configuration.
    """

    type: Literal["domain_gap"] = "domain_gap"
    columns: ColumnsConfig = Field(description="Column input configuration (required).")
    distance: DistanceConfig = Field(description="Distance metric configuration.")
    summary: SummaryConfig | None = None

    @model_validator(mode="after")
    def _validate_domain_gap(self) -> "DomainGapProcessorConfig":
        """Validate domain-gap configuration constraints."""
        if self.columns and not self.columns.input:
            raise ValueError("'columns.input' is required and must not be empty for domain_gap processors")
        if self.distance and self.distance.feature_weights is not None and self.columns and self.columns.input:
            n_cols = len(self.columns.input)
            if len(self.distance.feature_weights) != n_cols:
                raise ValueError(
                    f"feature_weights length ({len(self.distance.feature_weights)}) "
                    f"must match columns.input length ({n_cols})"
                )
        return self

columns: ColumnsConfig = Field(description='Column input configuration (required).') class-attribute instance-attribute

distance: DistanceConfig = Field(description='Distance metric configuration.') class-attribute instance-attribute

summary: SummaryConfig | None = None class-attribute instance-attribute

type: Literal['domain_gap'] = 'domain_gap' class-attribute instance-attribute

ErrorsConfig

Bases: BaseModel

Aggregate error-handling configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
class ErrorsConfig(BaseModel):
    """Aggregate error-handling configuration."""

    model_config = ConfigDict(extra="forbid")

    default: Literal["silent_fail", "fail_fast"] = Field(
        default="silent_fail",
        description="Default error action when no specific policy is set.",
    )
    images: ImageErrorsConfig | None = None
    tabular: TabularErrorsConfig | None = None
    max_failure_rate: float = Field(
        default=0.05,
        ge=0,
        le=1,
        description="Maximum tolerated failure rate before the job aborts (from 0.0 to 1.0).",
    )

default: Literal['silent_fail', 'fail_fast'] = Field(default='silent_fail', description='Default error action when no specific policy is set.') class-attribute instance-attribute

images: ImageErrorsConfig | None = None class-attribute instance-attribute

max_failure_rate: float = Field(default=0.05, ge=0, le=1, description='Maximum tolerated failure rate before the job aborts (from 0.0 to 1.0).') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

tabular: TabularErrorsConfig | None = None class-attribute instance-attribute

FeaturesEmbeddingsProcessorConfig

Bases: _ProcessorBase

Configuration for neural-network embedding feature extraction.

Extracts deep learning embeddings from images using a configured model.

Attributes:

Name Type Description
type Literal['features_embeddings']

Processor type discriminator ("features_embeddings").
model ModelConfig

Neural network model configuration.
infer InferConfig

Inference pre-processing settings.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
173
174
175
176
177
178
179
180
181
182
183
184
185
186
class FeaturesEmbeddingsProcessorConfig(_ProcessorBase):
    """Configuration for neural-network embedding feature extraction.

    Extracts deep learning embeddings from images using a configured model.

    Attributes:
        type: Processor type discriminator ("features_embeddings").
        model: Neural network model configuration.
        infer: Inference pre-processing settings.
    """

    type: Literal["features_embeddings"] = "features_embeddings"
    model: ModelConfig = Field(default_factory=ModelConfig)
    infer: InferConfig = Field(default_factory=InferConfig)

infer: InferConfig = Field(default_factory=InferConfig) class-attribute instance-attribute

model: ModelConfig = Field(default_factory=ModelConfig) class-attribute instance-attribute

type: Literal['features_embeddings'] = 'features_embeddings' class-attribute instance-attribute

FeaturesInterfaceConfig

Bases: _InterfaceBase

Feature-extraction pipeline configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/interfaces.py 
33
34
35
36
37
class FeaturesInterfaceConfig(_InterfaceBase):
    """Feature-extraction pipeline configuration."""

    outputs: FeaturesOutputsConfig | None = None
    processors: list[ProcessorConfig] = Field(default=[], description="Ordered list of feature processors.")

outputs: FeaturesOutputsConfig | None = None class-attribute instance-attribute

processors: list[ProcessorConfig] = Field(default=[], description='Ordered list of feature processors.') class-attribute instance-attribute

FeaturesOutputsConfig

Bases: BaseModel

Output configuration for computed features.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/outputs.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
class FeaturesOutputsConfig(BaseModel):
    """Output configuration for computed features."""

    model_config = ConfigDict(extra="forbid")

    path: str = Field(description="Destination path for feature output.")
    include: list[str] | None = Field(
        default=None,
        description="Feature columns to include (fnmatch patterns supported).",
    )
    exclude: list[str] | None = Field(
        default=None,
        description="Feature columns to exclude (fnmatch patterns supported).",
    )

exclude: list[str] | None = Field(default=None, description='Feature columns to exclude (fnmatch patterns supported).') class-attribute instance-attribute

include: list[str] | None = Field(default=None, description='Feature columns to include (fnmatch patterns supported).') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

path: str = Field(description='Destination path for feature output.') class-attribute instance-attribute

FilterConfig

Bases: BaseModel

Row-level filter applied during data loading.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
15
16
17
18
19
20
21
22
23
class FilterConfig(BaseModel):
    """Row-level filter applied during data loading."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name to filter on.")
    values: list[bool] | list[str] | list[int] | list[float] = Field(
        description="Value(s) to keep. Rows where column matches are included.",
    )

column: str = Field(description='Column name to filter on.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

values: list[bool] | list[str] | list[int] | list[float] = Field(description='Value(s) to keep. Rows where column matches are included.') class-attribute instance-attribute

GapInterfaceConfig

Bases: _InterfaceBase

Domain-gap computation pipeline configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/interfaces.py 
47
48
49
50
51
class GapInterfaceConfig(_InterfaceBase):
    """Domain-gap computation pipeline configuration."""

    outputs: GapOutputsConfig | None = None
    processors: list[ProcessorConfig] = Field(default=[], description="Ordered list of domain-gap processors.")

outputs: GapOutputsConfig | None = None class-attribute instance-attribute

processors: list[ProcessorConfig] = Field(default=[], description='Ordered list of domain-gap processors.') class-attribute instance-attribute

GapOutputsConfig

Bases: BaseModel

Output configuration for domain-gap results.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/outputs.py 
57
58
59
60
61
62
63
class GapOutputsConfig(BaseModel):
    """Output configuration for domain-gap results."""

    model_config = ConfigDict(extra="forbid")

    path: str = Field(description="Destination path for domain-gap output.")
    pairwise: bool = Field(default=True, description="Include pairwise (delta) results.")

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

pairwise: bool = Field(default=True, description='Include pairwise (delta) results.') class-attribute instance-attribute

path: str = Field(description='Destination path for domain-gap output.') class-attribute instance-attribute

HistogramConfig

Bases: BaseModel

Histogram parameters for feature extraction.

Attributes:

Name Type Description
bins int

Number of histogram bins (must be positive).
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
44
45
46
47
48
49
50
51
52
53
class HistogramConfig(BaseModel):
    """Histogram parameters for feature extraction.

    Attributes:
        bins: Number of histogram bins (must be positive).
    """

    model_config = ConfigDict(extra="forbid")

    bins: int = Field(default=256, gt=0, description="Number of histogram bins.")

bins: int = Field(default=256, gt=0, description='Number of histogram bins.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

ImageErrorsConfig

Bases: BaseModel

Error-handling policy for image-processing failures.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
class ImageErrorsConfig(BaseModel):
    """Error-handling policy for image-processing failures."""

    model_config = ConfigDict(extra="forbid")

    on_decode_failure: Literal["silent_fail", "fail_fast"] = Field(
        default="silent_fail",
        description="Action when an image cannot be decoded.",
    )
    on_transform_error: Literal["silent_fail", "fail_fast"] = Field(
        default="silent_fail",
        description="Action when an image transform fails.",
    )
    on_unsupported_format: Literal["silent_fail", "fail_fast"] = Field(
        default="fail_fast",
        description="Action on unsupported image format.",
    )

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

on_decode_failure: Literal['silent_fail', 'fail_fast'] = Field(default='silent_fail', description='Action when an image cannot be decoded.') class-attribute instance-attribute

on_transform_error: Literal['silent_fail', 'fail_fast'] = Field(default='silent_fail', description='Action when an image transform fails.') class-attribute instance-attribute

on_unsupported_format: Literal['silent_fail', 'fail_fast'] = Field(default='fail_fast', description='Action on unsupported image format.') class-attribute instance-attribute

ImageFeaturesProcessorConfig

Bases: _ProcessorBase

Configuration for low-level image feature extraction.

Extracts luminosity, contrast, blur, and entropy features from images.

Attributes:

Name Type Description
type Literal['image_features']

Processor type discriminator ("image_features").
features list[str]

List of image features to compute.
batch_size int

Batch size for image processing.
grayscale bool

Whether to convert images to grayscale.
normalize bool

Whether to normalize pixel values to [0, 1].
laplacian_kernel str

Laplacian kernel size for blur detection ("3x3" or "5x5").
clip_percentiles tuple[int, int] | None

Percentile clipping for extreme pixel values, e.g. (1, 99).
histogram HistogramConfig | None

Histogram configuration for feature computation.
luminosity_weights str | tuple[float, float, float] | None

Luminosity weights for grayscale conversion. Standard name ('bt601', 'bt709', 'bt2020') or [R, G, B] list/tuple. Defaults to BT.709 when None.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
class ImageFeaturesProcessorConfig(_ProcessorBase):
    """Configuration for low-level image feature extraction.

    Extracts luminosity, contrast, blur, and entropy features from images.

    Attributes:
        type: Processor type discriminator ("image_features").
        features: List of image features to compute.
        batch_size: Batch size for image processing.
        grayscale: Whether to convert images to grayscale.
        normalize: Whether to normalize pixel values to [0, 1].
        laplacian_kernel: Laplacian kernel size for blur detection ("3x3" or "5x5").
        clip_percentiles: Percentile clipping for extreme pixel values, e.g. (1, 99).
        histogram: Histogram configuration for feature computation.
        luminosity_weights: Luminosity weights for grayscale conversion.
            Standard name ('bt601', 'bt709', 'bt2020') or [R, G, B] list/tuple.
            Defaults to BT.709 when None.
    """

    type: Literal["image_features"] = "image_features"
    features: list[str] = Field(
        default=["luminosity", "contrast", "blur", "entropy"],
        description="List of image features to compute.",
    )
    batch_size: int = Field(default=64, gt=0, description="Batch size for image processing.")
    grayscale: bool = Field(default=True, description="Convert images to grayscale.")
    normalize: bool = Field(default=True, description="Normalise pixel values to [0, 1].")
    laplacian_kernel: str = Field(default="3x3", description="Laplacian kernel size for blur detection.")
    clip_percentiles: tuple[int, int] | None = Field(
        default=None,
        description="Percentile clipping for extreme pixel values, e.g. (1, 99).",
    )
    histogram: HistogramConfig | None = None
    luminosity_weights: str | tuple[float, float, float] | None = Field(
        default=None,
        description="Luminosity weights for grayscale conversion. "
        "Standard name ('bt601', 'bt709', 'bt2020') or [R, G, B] list. "
        "Defaults to BT.709 when None.",
    )

    @field_validator("luminosity_weights", mode="before")
    @classmethod
    def _normalize_luminosity_weights(cls, v: Any) -> Any:
        """Normalize luminosity weights input to standard key or tuple.

        Args:
            v: Input value - None, standard name string, or [R, G, B] list/tuple.

        Returns:
            Normalized value: None, standard key (e.g. "bt709"), or tuple of 3 floats.

        Raises:
            ValueError: If input is not a recognized standard, list/tuple of length 3, or None.
        """
        if v is None:
            return v
        if isinstance(v, str):
            key = v.lower().replace(".", "")
            if key not in _LUMINOSITY_STANDARDS:
                raise ValueError(f"Unknown luminosity standard '{v}'. Use one of {list(_LUMINOSITY_STANDARDS)}.")
            return key
        if isinstance(v, (list, tuple)):
            if len(v) != 3:
                raise ValueError(f"luminosity_weights must have exactly 3 elements, got {len(v)}.")
            return tuple(v)
        raise ValueError(f"luminosity_weights must be a standard name, [R,G,B] list, or None, got {type(v).__name__}.")

batch_size: int = Field(default=64, gt=0, description='Batch size for image processing.') class-attribute instance-attribute

clip_percentiles: tuple[int, int] | None = Field(default=None, description='Percentile clipping for extreme pixel values, e.g. (1, 99).') class-attribute instance-attribute

features: list[str] = Field(default=['luminosity', 'contrast', 'blur', 'entropy'], description='List of image features to compute.') class-attribute instance-attribute

grayscale: bool = Field(default=True, description='Convert images to grayscale.') class-attribute instance-attribute

histogram: HistogramConfig | None = None class-attribute instance-attribute

laplacian_kernel: str = Field(default='3x3', description='Laplacian kernel size for blur detection.') class-attribute instance-attribute

luminosity_weights: str | tuple[float, float, float] | None = Field(default=None, description="Luminosity weights for grayscale conversion. Standard name ('bt601', 'bt709', 'bt2020') or [R, G, B] list. Defaults to BT.709 when None.") class-attribute instance-attribute

normalize: bool = Field(default=True, description='Normalise pixel values to [0, 1].') class-attribute instance-attribute

type: Literal['image_features'] = 'image_features' class-attribute instance-attribute

JobConfig

Bases: BaseModel

Root configuration for a dqm-ml job. Each field maps to a pipeline stage.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/config.py 
14
15
16
17
18
19
20
21
22
23
24
25
class JobConfig(BaseModel):
    """Root configuration for a dqm-ml job. Each field maps to a pipeline stage."""

    model_config = ConfigDict(extra="forbid")

    storage: StorageConfig | None = None
    compute: ComputeConfig | None = None
    errors: ErrorsConfig | None = None
    dataloaders: DataLoadersConfig
    features: FeaturesInterfaceConfig | None = None
    metrics: MetricsInterfaceConfig | None = None
    gap: GapInterfaceConfig | None = None

compute: ComputeConfig | None = None class-attribute instance-attribute

dataloaders: DataLoadersConfig instance-attribute

errors: ErrorsConfig | None = None class-attribute instance-attribute

features: FeaturesInterfaceConfig | None = None class-attribute instance-attribute

gap: GapInterfaceConfig | None = None class-attribute instance-attribute

metrics: MetricsInterfaceConfig | None = None class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

storage: StorageConfig | None = None class-attribute instance-attribute

MetricsInterfaceConfig

Bases: _InterfaceBase

Metric-computation pipeline configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/interfaces.py 
40
41
42
43
44
class MetricsInterfaceConfig(_InterfaceBase):
    """Metric-computation pipeline configuration."""

    outputs: MetricsOutputsConfig | None = None
    processors: list[ProcessorConfig] = Field(default=[], description="Ordered list of metric processors.")

outputs: MetricsOutputsConfig | None = None class-attribute instance-attribute

processors: list[ProcessorConfig] = Field(default=[], description='Ordered list of metric processors.') class-attribute instance-attribute

MetricsOutputsConfig

Bases: BaseModel

Output configuration for computed metrics.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/outputs.py 
49
50
51
52
53
54
class MetricsOutputsConfig(BaseModel):
    """Output configuration for computed metrics."""

    model_config = ConfigDict(extra="forbid")

    path: str = Field(description="Destination path for metrics output.")

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

path: str = Field(description='Destination path for metrics output.') class-attribute instance-attribute

RepresentativenessProcessorConfig

Bases: _ProcessorBase

Configuration for representativeness evaluation against a reference distribution.

Evaluates how well a dataset represents a target distribution using multiple statistical metrics.

Attributes:

Name Type Description
type Literal['representativeness']

Processor type discriminator ("representativeness").
metrics list[str]

List of representativeness metrics to compute.
alpha float

Significance level for statistical tests.
epsilon float

Small constant to avoid division by zero.
distribution Literal['normal', 'uniform']

Expected reference distribution ("normal" or "uniform").
interpretation InterpretationConfig | None

Human-readable labels for results.
histogram HistogramsConfig | None

Histogram configuration for evaluation.
shannon ShannonConfig | None

Shannon entropy threshold configuration.
grte GrteConfig | None

GRTE configuration.
ks KsConfig | None

Kolmogorov-Smirnov test configuration.
Source code in packages/dqm-ml-core/src/dqm_ml_core/models/processors.py 
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
class RepresentativenessProcessorConfig(_ProcessorBase):
    """Configuration for representativeness evaluation against a reference distribution.

    Evaluates how well a dataset represents a target distribution using
    multiple statistical metrics.

    Attributes:
        type: Processor type discriminator ("representativeness").
        metrics: List of representativeness metrics to compute.
        alpha: Significance level for statistical tests.
        epsilon: Small constant to avoid division by zero.
        distribution: Expected reference distribution ("normal" or "uniform").
        interpretation: Human-readable labels for results.
        histogram: Histogram configuration for evaluation.
        shannon: Shannon entropy threshold configuration.
        grte: GRTE configuration.
        ks: Kolmogorov-Smirnov test configuration.
    """

    type: Literal["representativeness"] = "representativeness"
    metrics: list[str] = Field(
        default=["chi-square", "grte", "kolmogorov-smirnov", "shannon-entropy"],
        description="List of representativeness metrics to compute.",
    )
    alpha: float = Field(default=0.05, description="Significance level for statistical tests.")
    epsilon: float = Field(
        default=1e-9,
        gt=0,
        description="Small constant to avoid division by zero.",
    )
    distribution: Literal["normal", "uniform"] = Field(
        default="normal",
        description="Expected reference distribution.",
    )
    mean_std_estimation: Literal["from_first_batch", "per_batch", "from_all_data", "user_provided"] = Field(
        default="from_first_batch",
        description=(
            "How distribution parameters (mean/std for normal, min/max for uniform) are estimated. "
            "'from_first_batch' — estimate from the first batch and reuse (consistent with bin edges). "
            "'per_batch' — re-estimate on each batch (use with high-variance data, risks insufficient_bins). "
            "'user_provided' — use explicit parameters from distribution_params. "
            "'from_all_data' — estimate from full dataset (not yet implemented)."
        ),
    )
    expected_counts_method: Literal["cdf", "monte_carlo"] = Field(
        default="cdf",
        description=(
            "Method for computing expected bin counts. "
            "'cdf' — exact expected counts via CDF (deterministic). "
            "'monte_carlo' — Monte Carlo sampling via RNG (stochastic)."
        ),
    )
    distribution_params: list[ColumnDistributionParams] | None = Field(
        default=None,
        description=(
            "Per-column explicit distribution parameters for 'user_provided' strategy. "
            "Example: [{'column': 'col1', 'mean': 0.0, 'std': 1.0}]"
        ),
    )
    interpretation: InterpretationConfig | None = None
    histogram: HistogramsConfig | None = None
    shannon: ShannonConfig | None = None
    grte: GrteConfig | None = None
    ks: KsConfig | None = None

alpha: float = Field(default=0.05, description='Significance level for statistical tests.') class-attribute instance-attribute

distribution: Literal['normal', 'uniform'] = Field(default='normal', description='Expected reference distribution.') class-attribute instance-attribute

distribution_params: list[ColumnDistributionParams] | None = Field(default=None, description="Per-column explicit distribution parameters for 'user_provided' strategy. Example: [{'column': 'col1', 'mean': 0.0, 'std': 1.0}]") class-attribute instance-attribute

epsilon: float = Field(default=1e-09, gt=0, description='Small constant to avoid division by zero.') class-attribute instance-attribute

expected_counts_method: Literal['cdf', 'monte_carlo'] = Field(default='cdf', description="Method for computing expected bin counts. 'cdf' — exact expected counts via CDF (deterministic). 'monte_carlo' — Monte Carlo sampling via RNG (stochastic).") class-attribute instance-attribute

grte: GrteConfig | None = None class-attribute instance-attribute

histogram: HistogramsConfig | None = None class-attribute instance-attribute

interpretation: InterpretationConfig | None = None class-attribute instance-attribute

ks: KsConfig | None = None class-attribute instance-attribute

mean_std_estimation: Literal['from_first_batch', 'per_batch', 'from_all_data', 'user_provided'] = Field(default='from_first_batch', description="How distribution parameters (mean/std for normal, min/max for uniform) are estimated. 'from_first_batch' — estimate from the first batch and reuse (consistent with bin edges). 'per_batch' — re-estimate on each batch (use with high-variance data, risks insufficient_bins). 'user_provided' — use explicit parameters from distribution_params. 'from_all_data' — estimate from full dataset (not yet implemented).") class-attribute instance-attribute

metrics: list[str] = Field(default=['chi-square', 'grte', 'kolmogorov-smirnov', 'shannon-entropy'], description='List of representativeness metrics to compute.') class-attribute instance-attribute

shannon: ShannonConfig | None = None class-attribute instance-attribute

type: Literal['representativeness'] = 'representativeness' class-attribute instance-attribute

RetryConfig

Bases: BaseModel

Retry policy for storage operations.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
15
16
17
18
19
20
21
22
23
24
class RetryConfig(BaseModel):
    """Retry policy for storage operations."""

    model_config = ConfigDict(extra="forbid")

    mode: Literal["default", "standard"] = Field(
        default="standard",
        description="Retry mode: 'default' uses exponential backoff, 'standard' uses fixed intervals.",
    )
    max_attempts: int = Field(default=3, gt=0, description="Maximum number of retry attempts.")

max_attempts: int = Field(default=3, gt=0, description='Maximum number of retry attempts.') class-attribute instance-attribute

mode: Literal['default', 'standard'] = Field(default='standard', description="Retry mode: 'default' uses exponential backoff, 'standard' uses fixed intervals.") class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

SamplePathConfig

Bases: BaseModel

Per-column path prefix configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
26
27
28
29
30
31
32
class SamplePathConfig(BaseModel):
    """Per-column path prefix configuration."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name containing relative file paths.")
    prefix: str | None = Field(default=None, description="Base directory for resolving relative paths.")

column: str = Field(description='Column name containing relative file paths.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

prefix: str | None = Field(default=None, description='Base directory for resolving relative paths.') class-attribute instance-attribute

SplitConfig

Bases: BaseModel

How to split data into named groups (e.g. train / test).

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
class SplitConfig(BaseModel):
    """How to split data into named groups (e.g. train / test)."""

    model_config = ConfigDict(extra="forbid")

    by: str = Field(description="Column used to determine the split group.")
    values: list[str] | None = Field(
        default=None,
        description="Explicit list of split-group values to materialise. Auto-discovered if None.",
    )
    exclude: list[str] | None = Field(
        default=None,
        description="Split-group values to exclude (fnmatch patterns supported).",
    )

by: str = Field(description='Column used to determine the split group.') class-attribute instance-attribute

exclude: list[str] | None = Field(default=None, description='Split-group values to exclude (fnmatch patterns supported).') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

values: list[str] | None = Field(default=None, description='Explicit list of split-group values to materialise. Auto-discovered if None.') class-attribute instance-attribute

StorageConfig

Bases: BaseModel

Remote or local storage configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
class StorageConfig(BaseModel):
    """Remote or local storage configuration."""

    model_config = ConfigDict(extra="forbid")

    type: Literal["s3", "local"] = Field(description="Storage backend type.")
    bucket: str | None = Field(default=None, description="S3 bucket name (required when type='s3').")

    access_key: str | None = Field(default=None, description="AWS access key ID.")
    secret_key: str | None = Field(default=None, description="AWS secret access key.")
    session_token: str | None = Field(default=None, description="AWS session token.")
    anonymous: bool = Field(default=False, description="Use anonymous (unsigned) requests.")
    role_arn: str | None = Field(default=None, description="ARN of IAM role to assume.")
    session_name: str | None = Field(default=None, description="Name for the assumed role session.")
    external_id: str | None = Field(default=None, description="External ID for role assumption.")
    load_frequency: int = Field(default=900, gt=0, description="Frequency (seconds) to refresh credentials / role.")

    region: str | None = Field(default=None, description="AWS region (e.g. 'us-east-1').")
    endpoint: str | None = Field(default=None, description="Custom S3 endpoint URL.")
    request_timeout: float | None = Field(default=None, description="Request timeout in seconds.")
    connect_timeout: float | None = Field(default=None, description="Connection timeout in seconds.")
    scheme: str | None = Field(default=None, description="URI scheme (e.g. 'https').")
    proxy_options: dict[str, Any] | str | None = Field(
        default=None,
        description="Proxy configuration as a dict or URL string.",
    )
    tls_ca_file_path: str | None = Field(default=None, description="Path to a custom TLS CA bundle.")

    retry: RetryConfig | None = None

    checksum_validation: Literal["when_required", "always", "never"] = Field(
        default="when_required",
        description="Controls S3 checksum validation behaviour.",
    )

    @model_validator(mode="after")
    def _require_bucket_for_s3(self) -> "StorageConfig":
        """Validate that S3 storage configuration includes a bucket name.

        Returns:
            The validated StorageConfig instance.

        Raises:
            ValueError: If storage type is 's3' but no bucket is provided.
        """
        if self.type == "s3" and self.bucket is None:
            raise ValueError("StorageConfig with type='s3' requires a 'bucket'")
        return self

access_key: str | None = Field(default=None, description='AWS access key ID.') class-attribute instance-attribute

anonymous: bool = Field(default=False, description='Use anonymous (unsigned) requests.') class-attribute instance-attribute

bucket: str | None = Field(default=None, description="S3 bucket name (required when type='s3').") class-attribute instance-attribute

checksum_validation: Literal['when_required', 'always', 'never'] = Field(default='when_required', description='Controls S3 checksum validation behaviour.') class-attribute instance-attribute

connect_timeout: float | None = Field(default=None, description='Connection timeout in seconds.') class-attribute instance-attribute

endpoint: str | None = Field(default=None, description='Custom S3 endpoint URL.') class-attribute instance-attribute

external_id: str | None = Field(default=None, description='External ID for role assumption.') class-attribute instance-attribute

load_frequency: int = Field(default=900, gt=0, description='Frequency (seconds) to refresh credentials / role.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

proxy_options: dict[str, Any] | str | None = Field(default=None, description='Proxy configuration as a dict or URL string.') class-attribute instance-attribute

region: str | None = Field(default=None, description="AWS region (e.g. 'us-east-1').") class-attribute instance-attribute

request_timeout: float | None = Field(default=None, description='Request timeout in seconds.') class-attribute instance-attribute

retry: RetryConfig | None = None class-attribute instance-attribute

role_arn: str | None = Field(default=None, description='ARN of IAM role to assume.') class-attribute instance-attribute

scheme: str | None = Field(default=None, description="URI scheme (e.g. 'https').") class-attribute instance-attribute

secret_key: str | None = Field(default=None, description='AWS secret access key.') class-attribute instance-attribute

session_name: str | None = Field(default=None, description='Name for the assumed role session.') class-attribute instance-attribute

session_token: str | None = Field(default=None, description='AWS session token.') class-attribute instance-attribute

tls_ca_file_path: str | None = Field(default=None, description='Path to a custom TLS CA bundle.') class-attribute instance-attribute

type: Literal['s3', 'local'] = Field(description='Storage backend type.') class-attribute instance-attribute

TabularErrorsConfig

Bases: BaseModel

Error-handling policy for tabular-data failures.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/global_.py 
115
116
117
118
119
120
121
122
123
124
125
126
127
class TabularErrorsConfig(BaseModel):
    """Error-handling policy for tabular-data failures."""

    model_config = ConfigDict(extra="forbid")

    on_missing_column: Literal["silent_fail", "fail_fast"] = Field(
        default="fail_fast",
        description="Action when a required column is missing.",
    )
    on_file_not_found: Literal["silent_fail", "fail_fast"] = Field(
        default="fail_fast",
        description="Action when a data file is not found.",
    )

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

on_file_not_found: Literal['silent_fail', 'fail_fast'] = Field(default='fail_fast', description='Action when a data file is not found.') class-attribute instance-attribute

on_missing_column: Literal['silent_fail', 'fail_fast'] = Field(default='fail_fast', description='Action when a required column is missing.') class-attribute instance-attribute

TransformConfig

Bases: BaseModel

Column type-casting transformation.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
63
64
65
66
67
68
69
70
class TransformConfig(BaseModel):
    """Column type-casting transformation."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name to transform.")
    to_type: TransformType = Field(description="Target data type.")
    in_place: bool = Field(default=False, description="Overwrite the original column in place.")

column: str = Field(description='Column name to transform.') class-attribute instance-attribute

in_place: bool = Field(default=False, description='Overwrite the original column in place.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

to_type: TransformType = Field(description='Target data type.') class-attribute instance-attribute

TransformType

Bases: str, Enum

Target data type for column transformations.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
51
52
53
54
55
56
57
58
59
60
class TransformType(str, Enum):
    """Target data type for column transformations."""

    INT32 = "int32"
    INT64 = "int64"
    FLOAT32 = "float32"
    FLOAT64 = "float64"
    BOOL = "bool"
    STR = "str"
    CATEGORICAL = "categorical"

BOOL = 'bool' class-attribute instance-attribute

CATEGORICAL = 'categorical' class-attribute instance-attribute

FLOAT32 = 'float32' class-attribute instance-attribute

FLOAT64 = 'float64' class-attribute instance-attribute

INT32 = 'int32' class-attribute instance-attribute

INT64 = 'int64' class-attribute instance-attribute

STR = 'str' class-attribute instance-attribute