Direct usage with Notebook¶

In [21]:

Copied!

# Common Libraries
import numpy as np
import pandas as pd
# Common Libraries
import numpy as np
import pandas as pd

Tests on challenge welding data¶

This dataset is described on the website of challenge welding : https://confianceai.github.io/Welding-Quality-Detection-Challenge/

In [22]:

Copied!





# upload data
# Dataset path
path = "../../tests/integration/fixtures/data/challenge_welding.csv"
data = pd.read_csv(path, sep=",")
data
# upload data
# Dataset path
path = "../../tests/integration/fixtures/data/challenge_welding.csv"
data = pd.read_csv(path, sep=",")
data

Out[22]:

	sample_id	class	timestamp	welding-seams	labelling_type	resolution	path	sha256	storage_type	data_origin	blur_level	blur_class	luminosity_level	external_path
0	data_37966	OK	08/07/2022 02:05	c33	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'0F\\\x12\xa1\x81\xcdL\xc3~\xbe\x03\xc90\x91\...	s3	real	1667.425860	clean	51.079433	http://minio-storage.apps.confianceai-public.i...
1	data_25403	OK	19/07/2022 00:51	c102	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'\tlM\xd7\x83\\\x7fm\xaeV\xc6\xbd\xd6S@\x17S\...	s3	real	868.574712	blur	32.825601	http://minio-storage.apps.confianceai-public.i...
2	data_27038	OK	28/07/2022 16:08	c102	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'\xeb\xba/\xe0-\xe6\x14\xd6\xe0\xf2oT\xd1\xa9...	s3	real	1078.402671	clean	35.192525	http://minio-storage.apps.confianceai-public.i...
3	data_8767	OK	31/01/20 09:41	c33	expert	[1920 1080]	challenge-welding/datasets/welding-detection-c...	b"\x9e@!\xd3\xdd\xaa'\xb7]\xeb\xf7B\x800\xf1\x...	s3	real	1240.198851	clean	38.510588	http://minio-storage.apps.confianceai-public.i...
4	data_4744	OK	25/11/19 21:57	c20	expert	[1920 1080]	challenge-welding/datasets/welding-detection-c...	b'\x0c\xd8\xcfb\x95\\\xb9\xa8w\xe7^\xea\rX,8\x...	s3	real	3371.897144	clean	46.001366	http://minio-storage.apps.confianceai-public.i...
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
22748	data_66456	OK	29/10/19 01:33	c102	expert	[1920 1080]	challenge-welding/datasets/welding-detection-c...	b'\x9fZ^\xc5j\xf1\xd8JRw\x03\xa4C\x04\xa6\xa1\...	s3	real	4463.079210	clean	41.554284	http://minio-storage.apps.confianceai-public.i...
22749	data_24294	OK	07/07/2022 01:40	c102	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'\x05\x07x\xa4\xb8\x01\xf1a\x18.\x13\xe4\xe6\...	s3	real	859.194594	blur	31.966749	http://minio-storage.apps.confianceai-public.i...
22750	data_25568	OK	19/07/2022 11:59	c102	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'O\x0eT\xa1\x8b\xa0\xf5\xaei\x81\xe8\x16\x96p...	s3	real	977.316444	clean	40.872106	http://minio-storage.apps.confianceai-public.i...
22751	data_39356	OK	13/07/2022 06:45	c33	operator	[960 540]	challenge-welding/datasets/welding-detection-c...	b'\x1bh\xd9\x8aK\x88\xc1\xcb\xea\xff\xe3]{\x0e...	s3	real	1653.447154	clean	50.429395	http://minio-storage.apps.confianceai-public.i...
22752	data_8988	OK	09/03/20 12:22	c33	expert	[1920 1080]	challenge-welding/datasets/welding-detection-c...	b'\xce\x0e\x87\xd7\x9b\x8d\xc4\x10[\xe9\x8fl\x...	s3	real	593.528212	blur	39.343171	http://minio-storage.apps.confianceai-public.i...

22753 rows × 14 columns

Completeness¶

you can see also the file: packages/dqm-ml-core/src/dqm_ml_core/metrics/completeness.py

In [23]:

Copied!

from dqm_ml_core import CompletenessProcessor, ProcessorRunner
from dqm_ml_core import CompletenessProcessor, ProcessorRunner

In [24]:

Copied!





# Compute completeness on the whole dataset
runner = ProcessorRunner()

metric = CompletenessProcessor(config={"include_per_column": False})
metrics_values = runner.run(data, [metric])
print(metrics_values)
# Compute completeness on the whole dataset
runner = ProcessorRunner()

metric = CompletenessProcessor(config={"include_per_column": False})
metrics_values = runner.run(data, [metric])
print(metrics_values)

{'completeness_overall': 1.0}

In [25]:

Copied!





# Compute completeness on a single column

# Configure column and metric metric name in outdict
metric = CompletenessProcessor(config={"columns": {"input": ["blur_level"]}})
metrics_values = runner.run(data, [metric])

print("Overall Data Completeness Score: ", metrics_values["completeness_overall"])
print("Completeness Score for Column: ", metrics_values["completeness_blur_level"])
# Compute completeness on a single column

# Configure column and metric metric name in outdict
metric = CompletenessProcessor(config={"columns": {"input": ["blur_level"]}})
metrics_values = runner.run(data, [metric])

print("Overall Data Completeness Score: ", metrics_values["completeness_overall"])
print("Completeness Score for Column: ", metrics_values["completeness_blur_level"])

Overall Data Completeness Score:  1.0
Completeness Score for Column:  1.0

Representativeness¶

In [26]:

Copied!





# Using the method chisquare_test
from dqm_ml_core import RepresentativenessProcessor

bins = 20
distribution = "normal"

metric = RepresentativenessProcessor(
    config={
        "metrics": ["chi-square"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Chi-Square Test: p-value = ", metrics_values["blur_level_chi-square_p_value"])
# Using the method chisquare_test
from dqm_ml_core import RepresentativenessProcessor

bins = 20
distribution = "normal"

metric = RepresentativenessProcessor(
    config={
        "metrics": ["chi-square"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Chi-Square Test: p-value = ", metrics_values["blur_level_chi-square_p_value"])

{'blur_level_chi-square_p_value': 0.0, 'blur_level_chi-square_statistic': 6393.917769085385, 'blur_level_chi-square_interpretation': 'does_not_follow_distribution', '_metadata': '{"bins": 20, "distribution": "normal", "metrics_computed": ["chi-square"], "total_samples": 22753, "columns_analyzed": ["blur_level"], "ks_sampling_enabled": false, "note": "KS test uses random sampling approximation for scalability"}'}
Chi-Square Test: p-value =  0.0

In [27]:

Copied!





metric = RepresentativenessProcessor(
    config={
        "metrics": ["kolmogorov-smirnov"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Kolmogorov-Smirnov Test: p-value = ", metrics_values["blur_level_kolmogorov-smirnov_p_value"])
metric = RepresentativenessProcessor(
    config={
        "metrics": ["kolmogorov-smirnov"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Kolmogorov-Smirnov Test: p-value = ", metrics_values["blur_level_kolmogorov-smirnov_p_value"])

{'blur_level_kolmogorov-smirnov_p_value': 0.0011053326070895135, 'blur_level_kolmogorov-smirnov_statistic': 0.08622146246049645, 'blur_level_kolmogorov-smirnov_interpretation': 'does_not_follow_distribution', 'blur_level_kolmogorov-smirnov_sample_size': 500, 'blur_level_kolmogorov-smirnov_note': 'approximated_from_random_samples', '_metadata': '{"bins": 20, "distribution": "normal", "metrics_computed": ["kolmogorov-smirnov"], "total_samples": 22753, "columns_analyzed": ["blur_level"], "ks_sampling_enabled": true, "note": "KS test uses random sampling approximation for scalability"}'}
Kolmogorov-Smirnov Test: p-value =  0.0011053326070895135

In [28]:

Copied!





metric = RepresentativenessProcessor(
    config={
        "metrics": ["shannon-entropy"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Shannon Entropy Test: p-value = ", metrics_values["blur_level_shannon-entropy_entropy"])
metric = RepresentativenessProcessor(
    config={
        "metrics": ["shannon-entropy"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("Shannon Entropy Test: p-value = ", metrics_values["blur_level_shannon-entropy_entropy"])

{'blur_level_shannon-entropy_entropy': 2.9957322735539904, 'blur_level_shannon-entropy_interpretation': 'high_diversity', '_metadata': '{"bins": 20, "distribution": "normal", "metrics_computed": ["shannon-entropy"], "total_samples": 22753, "columns_analyzed": ["blur_level"], "ks_sampling_enabled": false, "note": "KS test uses random sampling approximation for scalability"}'}
Shannon Entropy Test: p-value =  2.9957322735539904

In [29]:

Copied!





metric = RepresentativenessProcessor(
    config={
        "metrics": ["grte"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("GRTE = ", metrics_values["blur_level_grte_grte_value"])
metric = RepresentativenessProcessor(
    config={
        "metrics": ["grte"],
        "columns": {"input": ["blur_level"]},
        "histogram": {"bins": 20},
        "distribution": "normal",
    }
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

print("GRTE = ", metrics_values["blur_level_grte_grte_value"])

{'blur_level_grte_grte_value': 0.7468998932594301, 'blur_level_grte_interpretation': 'high_representativeness', '_metadata': '{"bins": 20, "distribution": "normal", "metrics_computed": ["grte"], "total_samples": 22753, "columns_analyzed": ["blur_level"], "ks_sampling_enabled": false, "note": "KS test uses random sampling approximation for scalability"}'}
GRTE =  0.7468998932594301

We can compute all metrics at once¶

In [30]:

Copied!





metric = RepresentativenessProcessor(
    config={"columns": {"input": ["blur_level"]}, "histogram": {"bins": 20}, "distribution": "normal"}
)
metrics_values = runner.run(data, [metric])
print(metrics_values)
metric = RepresentativenessProcessor(
    config={"columns": {"input": ["blur_level"]}, "histogram": {"bins": 20}, "distribution": "normal"}
)
metrics_values = runner.run(data, [metric])
print(metrics_values)

{'blur_level_chi-square_p_value': 0.0, 'blur_level_chi-square_statistic': 6393.917769085385, 'blur_level_chi-square_interpretation': 'does_not_follow_distribution', 'blur_level_kolmogorov-smirnov_p_value': 0.00012501262907099823, 'blur_level_kolmogorov-smirnov_statistic': 0.09797170571252506, 'blur_level_kolmogorov-smirnov_interpretation': 'does_not_follow_distribution', 'blur_level_kolmogorov-smirnov_sample_size': 500, 'blur_level_kolmogorov-smirnov_note': 'approximated_from_random_samples', 'blur_level_shannon-entropy_entropy': 2.9957322735539904, 'blur_level_shannon-entropy_interpretation': 'high_diversity', 'blur_level_grte_grte_value': 0.7468998932594301, 'blur_level_grte_interpretation': 'high_representativeness', '_metadata': '{"bins": 20, "distribution": "normal", "metrics_computed": ["chi-square", "grte", "kolmogorov-smirnov", "shannon-entropy"], "total_samples": 22753, "columns_analyzed": ["blur_level"], "ks_sampling_enabled": true, "note": "KS test uses random sampling approximation for scalability"}'}