Skip to content

dqm_ml_core.models.dataloaders

Data loading and transformation configuration models.

Defines models for filters, path sampling, data splitting, column transformations, and dataloader configurations for Parquet and CSV sources.

DataLoaderConfig

Bases: BaseModel

Configuration for a single dataloader (Parquet or CSV).

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class DataLoaderConfig(BaseModel):
    """Configuration for a single dataloader (Parquet or CSV)."""

    model_config = ConfigDict(extra="forbid")

    name: str = Field(description="Unique name for this dataloader.")
    type: Literal["parquet", "csv"] = Field(description="Data file format.")
    path: str = Field(description="Glob pattern or path to data files.")
    id_column: str | None = Field(default=None, description="Column used as row identifier.")
    batch_size: int = Field(default=10000, description="Number of rows per batch.")
    filters: list[FilterConfig] | None = None
    sample_path: list[SamplePathConfig] | None = None
    split: SplitConfig | None = None
    transform: list[TransformConfig] | None = None
    storage: StorageConfig | None = None

batch_size: int = Field(default=10000, description='Number of rows per batch.') class-attribute instance-attribute

filters: list[FilterConfig] | None = None class-attribute instance-attribute

id_column: str | None = Field(default=None, description='Column used as row identifier.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

name: str = Field(description='Unique name for this dataloader.') class-attribute instance-attribute

path: str = Field(description='Glob pattern or path to data files.') class-attribute instance-attribute

sample_path: list[SamplePathConfig] | None = None class-attribute instance-attribute

split: SplitConfig | None = None class-attribute instance-attribute

storage: StorageConfig | None = None class-attribute instance-attribute

transform: list[TransformConfig] | None = None class-attribute instance-attribute

type: Literal['parquet', 'csv'] = Field(description='Data file format.') class-attribute instance-attribute

DataLoadersConfig

Bases: BaseModel

Collection of dataloaders for a job.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
90
91
92
93
94
95
96
97
98
99
class DataLoadersConfig(BaseModel):
    """Collection of dataloaders for a job."""

    model_config = ConfigDict(extra="forbid")

    storage: StorageConfig | None = Field(
        default=None,
        description="Default storage config inherited by all loaders.",
    )
    loaders: list[DataLoaderConfig] = Field(description="List of dataloader configurations.")

loaders: list[DataLoaderConfig] = Field(description='List of dataloader configurations.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

storage: StorageConfig | None = Field(default=None, description='Default storage config inherited by all loaders.') class-attribute instance-attribute

FilterConfig

Bases: BaseModel

Row-level filter applied during data loading.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
15
16
17
18
19
20
21
22
23
class FilterConfig(BaseModel):
    """Row-level filter applied during data loading."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name to filter on.")
    values: list[bool] | list[str] | list[int] | list[float] = Field(
        description="Value(s) to keep. Rows where column matches are included.",
    )

column: str = Field(description='Column name to filter on.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

values: list[bool] | list[str] | list[int] | list[float] = Field(description='Value(s) to keep. Rows where column matches are included.') class-attribute instance-attribute

SamplePathConfig

Bases: BaseModel

Per-column path prefix configuration.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
26
27
28
29
30
31
32
class SamplePathConfig(BaseModel):
    """Per-column path prefix configuration."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name containing relative file paths.")
    prefix: str | None = Field(default=None, description="Base directory for resolving relative paths.")

column: str = Field(description='Column name containing relative file paths.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

prefix: str | None = Field(default=None, description='Base directory for resolving relative paths.') class-attribute instance-attribute

SplitConfig

Bases: BaseModel

How to split data into named groups (e.g. train / test).

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
class SplitConfig(BaseModel):
    """How to split data into named groups (e.g. train / test)."""

    model_config = ConfigDict(extra="forbid")

    by: str = Field(description="Column used to determine the split group.")
    values: list[str] | None = Field(
        default=None,
        description="Explicit list of split-group values to materialise. Auto-discovered if None.",
    )
    exclude: list[str] | None = Field(
        default=None,
        description="Split-group values to exclude (fnmatch patterns supported).",
    )

by: str = Field(description='Column used to determine the split group.') class-attribute instance-attribute

exclude: list[str] | None = Field(default=None, description='Split-group values to exclude (fnmatch patterns supported).') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

values: list[str] | None = Field(default=None, description='Explicit list of split-group values to materialise. Auto-discovered if None.') class-attribute instance-attribute

TransformConfig

Bases: BaseModel

Column type-casting transformation.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
63
64
65
66
67
68
69
70
class TransformConfig(BaseModel):
    """Column type-casting transformation."""

    model_config = ConfigDict(extra="forbid")

    column: str = Field(description="Column name to transform.")
    to_type: TransformType = Field(description="Target data type.")
    in_place: bool = Field(default=False, description="Overwrite the original column in place.")

column: str = Field(description='Column name to transform.') class-attribute instance-attribute

in_place: bool = Field(default=False, description='Overwrite the original column in place.') class-attribute instance-attribute

model_config = ConfigDict(extra='forbid') class-attribute instance-attribute

to_type: TransformType = Field(description='Target data type.') class-attribute instance-attribute

TransformType

Bases: str, Enum

Target data type for column transformations.

Source code in packages/dqm-ml-core/src/dqm_ml_core/models/dataloaders.py 
51
52
53
54
55
56
57
58
59
60
class TransformType(str, Enum):
    """Target data type for column transformations."""

    INT32 = "int32"
    INT64 = "int64"
    FLOAT32 = "float32"
    FLOAT64 = "float64"
    BOOL = "bool"
    STR = "str"
    CATEGORICAL = "categorical"

BOOL = 'bool' class-attribute instance-attribute

CATEGORICAL = 'categorical' class-attribute instance-attribute

FLOAT32 = 'float32' class-attribute instance-attribute

FLOAT64 = 'float64' class-attribute instance-attribute

INT32 = 'int32' class-attribute instance-attribute

INT64 = 'int64' class-attribute instance-attribute

STR = 'str' class-attribute instance-attribute