26616170697

Committed 29 May 2026 03:28AM UTC coverage: 75.347%. First build

Build # 26616170697

Build Type

Pull #360

github

Committed by

web-flow

Commit Message

Merge 101146123 into 936e3dd5a

Pull Request Pull Request #360: feat(drift): add ``from_cli`` constructor and add docs

Coverage Stats

137 of 154 new or added lines in 27 files covered. (88.96%)

7228 of 9593 relevant lines covered (75.35%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.8

/tests/test_classifiers.py

import os
from contextlib import nullcontext
from dataclasses import dataclass
from functools import partial
from tempfile import TemporaryDirectory
from typing import Callable, Optional

import pytest
import torch
from java.lang import Exception as JException
from pytest_subtests import SubTests

from capymoa._cli import classifier_cli_str
from capymoa.ann import Perceptron
from capymoa.base import Classifier, MOAClassifier
from capymoa.classifier import (
    CSMOTE,
    EFDT,
    KNN,
    LAST,
    AdaptiveRandomForestClassifier,
    DynamicWeightedMajority,
    Finetune,
    HoeffdingAdaptiveTree,
    HoeffdingTree,
    LeveragingBagging,
    MajorityClass,
    NaiveBayes,
    NoChange,
    OnlineAdwinBagging,
    OnlineBagging,
    OnlineSmoothBoost,
    OzaBoost,
    PassiveAggressiveClassifier,
    SAMkNN,
    SGDClassifier,
    ShrubsClassifier,
    StreamingGradientBoostedTrees,
    StreamingRandomPatches,
    WeightedkNN,
    DynamicEnsembleMemberSelection,
    PLASTIC,
)
from capymoa.datasets import ElectricityTiny
from capymoa.evaluation import ClassificationEvaluator, prequential_evaluation
from capymoa.misc import load_model, save_model
from capymoa.splitcriteria import GiniSplitCriterion
from capymoa.stream import Schema, Stream
import numpy as np


@dataclass
class ClassifierTestCase:
    test_name: str
    """A unique name to identify your test case. Usually the name of the learner."""
    learner_constructor: Callable[[Schema], Classifier]
    """A function that returns a new instance of the learner."""
    accuracy: float
    """The expected accuracy of the learner."""
    win_accuracy: float
    """The expected windowed accuracy of the learner."""
    cli_string: Optional[str] = None
    """The expected CLI string of the learner."""
    is_serializable: bool = True
    """Whether the learner is serializable."""
    batch_size: int = 1

    skip_prediction_check_before_training: bool = False
    """Skip checking the prediction before training. If False, the test will fail if the
    learner does not output None before training."""

    skip_reason: Optional[str] = None
    """A reason to skip the test. If set, the test will be skipped with this reason."""


"""
Add your test cases here. Each test case is a `ClassifierTestCase` object

Notice how we use the `partial` function to creates a new function with
hyperparameters already set. This allows us to use the same test function
for different learners with different hyperparameters.
"""
test_cases = [
    ClassifierTestCase(
        "OnlineBagging",
        partial(OnlineBagging, ensemble_size=5),
        84.6,
        89.0,
    ),
    ClassifierTestCase(
        "AdaptiveRandomForestClassifier",
        partial(AdaptiveRandomForestClassifier),
        89.0,
        91.0,
    ),
    ClassifierTestCase(
        "HoeffdingTree",
        partial(HoeffdingTree),
        82.65,
        83.0,
    ),
    ClassifierTestCase(
        "LAST",
        partial(LAST),
        83.95,
        91.0,
    ),
    ClassifierTestCase(
        "EFDT",
        partial(EFDT),
        82.69,
        82.0,
    ),
    ClassifierTestCase(
        "EFDT-1",
        partial(
            EFDT,
            grace_period=10,
            split_criterion=GiniSplitCriterion(),
            leaf_prediction="NaiveBayes",
        ),
        87.35,
        84.0,
        cli_string="trees.EFDT -R 200 -m 33554433 -g 10 -s GiniSplitCriterion -c 0.001 -z -p -l NB",
        skip_reason="https://github.com/adaptive-machine-learning/backlog/issues/59",
    ),
    ClassifierTestCase("PLASTIC", PLASTIC, 83.25, 83.0),
    ClassifierTestCase(
        "NaiveBayes",
        partial(NaiveBayes),
        84.0,
        91.0,
    ),
    ClassifierTestCase(
        "KNN",
        partial(KNN),
        81.6,
        74.0,
    ),
    ClassifierTestCase(
        "PassiveAggressiveClassifier",
        partial(PassiveAggressiveClassifier),
        84.7,
        81.0,
    ),
    ClassifierTestCase(
        "SGDClassifier",
        # When loss="log_loss" predict_proba works
        partial(SGDClassifier, loss="log_loss"),
        85.75,
        82.0,
    ),
    ClassifierTestCase(
        "StreamingGradientBoostedTrees",
        partial(StreamingGradientBoostedTrees),
        88.75,
        88.0,
    ),
    ClassifierTestCase(
        "OzaBoost",
        partial(OzaBoost),
        89.95,
        89.0,
    ),
    ClassifierTestCase(
        "MajorityClass",
        partial(MajorityClass),
        60.199999999999996,
        66.0,
    ),
    ClassifierTestCase(
        "NoChange",
        partial(NoChange),
        85.95,
        81.0,
        skip_prediction_check_before_training=True,
    ),
    ClassifierTestCase(
        "OnlineSmoothBoost",
        partial(OnlineSmoothBoost),
        87.85,
        90.0,
    ),
    ClassifierTestCase(
        "StreamingRandomPatches",
        partial(StreamingRandomPatches),
        90.2,
        89.0,
        is_serializable=False,
    ),
    ClassifierTestCase(
        "HoeffdingAdaptiveTree",
        partial(HoeffdingAdaptiveTree),
        84.15,
        92.0,
    ),
    ClassifierTestCase(
        "SAMkNN",
        partial(SAMkNN),
        82.65,
        82.0,
    ),
    ClassifierTestCase(
        "DynamicWeightedMajority",
        partial(DynamicWeightedMajority),
        84.05,
        89.0,
        skip_prediction_check_before_training=True,
    ),
    ClassifierTestCase(
        "CSMOTE",
        partial(CSMOTE),
        80.55,
        79.0,
    ),
    ClassifierTestCase(
        "LeveragingBagging",
        partial(LeveragingBagging),
        86.7,
        91.0,
    ),
    ClassifierTestCase(
        "OnlineAdwinBagging",
        partial(OnlineAdwinBagging),
        85.25,
        92.0,
    ),
    ClassifierTestCase(
        "WeightedkNN",
        partial(WeightedkNN),
        78.15,
        70,
    ),
    ClassifierTestCase("ShrubsClassifier", partial(ShrubsClassifier), 89.6, 91),
    ClassifierTestCase(
        "Finetune",
        partial(
            Finetune,
            model=Perceptron,
            optimizer=partial(torch.optim.Adam, lr=0.001),
        ),
        60.4,
        66.0,
        batch_size=32,
        skip_prediction_check_before_training=True,
    ),
    ClassifierTestCase(
        "DynamicEnsembleMemberSelection",
        partial(DynamicEnsembleMemberSelection),
        92.7,
        90,
        is_serializable=False,
    ),
]


def _score(classifier: Classifier, stream: Stream, limit=100) -> float:
    """Eval without training the classifier."""
    evaluator = ClassificationEvaluator(schema=stream.get_schema())
    stream.restart()
    for i, instance in enumerate(stream):
        if i >= limit:
            break
        prediction = classifier.predict(instance)
        evaluator.update(instance.y_index, prediction)
    return evaluator.accuracy()


def subtest_save_and_load(
    classifier: Classifier,
    stream: Stream,
    is_serializable: bool,
):
    """A subtest to check if a classifier can be saved and loaded."""

    with TemporaryDirectory() as tmp_dir:
        tmp_file = os.path.join(tmp_dir, "model.pkl")
        with pytest.raises(JException) if not is_serializable else nullcontext():
            # Save and load the model
            with open(tmp_file, "wb") as f:
                save_model(classifier, f)

            with open(tmp_file, "rb") as f:
                loaded_classifier: Classifier = load_model(f)

            # Check that the saved and loaded model have the same accuracy
            expected_acc = _score(classifier, stream)
            loaded_acc = _score(loaded_classifier, stream)
            assert expected_acc == loaded_acc, (
                f"Original accuracy {expected_acc * 100:.2f} != loaded accuracy {loaded_acc * 100:.2f}"
            )

            # Check that the loaded model can still be trained
            loaded_classifier.train(next(stream))


@pytest.mark.parametrize(
    "test_case",
    test_cases,
    ids=[c.test_name for c in test_cases],
)
def test_classifiers(test_case: ClassifierTestCase, subtests: SubTests):
    """``test_classifiers`` is a fast running complex test that checks:

    * Did the classifier reach the expected accuracy?
    * Did the classifier reach the expected windowed accuracy?
    * Can the classifier be saved and loaded?
    * Does the CLI string match the expected value?
    """
    if test_case.skip_reason:
        pytest.skip(test_case.skip_reason)

    stream = ElectricityTiny()
    y_shape = (stream.schema.get_num_classes(),)
    learner: Classifier = test_case.learner_constructor(schema=stream.get_schema())

    if not test_case.skip_prediction_check_before_training:
        # The classifier should not error when asked to make predictions before
        # training. Instead it must return None.
        assert learner.predict(next(stream)) is None
        assert learner.predict_proba(next(stream)) is None

    # Main Loop
    stream.restart()
    results = prequential_evaluation(
        stream, learner, window_size=100, batch_size=test_case.batch_size
    )

    # Check if the learner can make predictions after training and that those
    # predictions are of the expected type
    stream.restart()
    y_proba = learner.predict_proba(next(stream))
    y = learner.predict(next(stream))
    assert isinstance(y_proba, np.ndarray), f"Expected numpy array, got {type(y_proba)}"
    assert y_proba.dtype == np.float64, f"Expected float64, got {y_proba.dtype}"
    assert y_proba.shape == y_shape, f"Expected shape {y_shape}, got {y_proba.shape}"
    assert isinstance(y, int), f"Expected int, got {type(y)}"
    assert sum(y_proba) == pytest.approx(1.0, abs=1e-5), "Probability sum != 1"

    # Check if the accuracy matches the expected value for both evaluator types
    actual_acc = results.cumulative.accuracy()
    actual_win_acc = results.windowed.accuracy()[-1]

    assert actual_acc == pytest.approx(test_case.accuracy, abs=0.1), (
        f"Basic Eval: Expected accuracy of {test_case.accuracy:0.1f} got {actual_acc: 0.1f}"
    )
    assert actual_win_acc == pytest.approx(test_case.win_accuracy, abs=0.1), (
        f"Windowed Eval: Expected accuracy of {test_case.win_accuracy:0.1f} got {actual_win_acc:0.1f}"
    )

    # Check if the classifier can be saved and loaded
    with subtests.test(msg="save_and_load"):
        subtest_save_and_load(learner, stream, test_case.is_serializable)

    # Optionally check the CLI string if it was provided
    if isinstance(learner, MOAClassifier) and test_case.cli_string is not None:
        cli_str = classifier_cli_str(learner).strip("()")
        assert cli_str == test_case.cli_string, "CLI does not match expected value"

1	import os	1✔
2	from contextlib import nullcontext	1✔
3	from dataclasses import dataclass	1✔
4	from functools import partial	1✔
5	from tempfile import TemporaryDirectory	1✔
6	from typing import Callable, Optional	1✔
7
8	import pytest	1✔
9	import torch	1✔
10	from java.lang import Exception as JException	1✔
11	from pytest_subtests import SubTests	1✔
12
13	from capymoa._cli import classifier_cli_str	1✔
14	from capymoa.ann import Perceptron	1✔
15	from capymoa.base import Classifier, MOAClassifier	1✔
16	from capymoa.classifier import (	1✔
17	CSMOTE,
18	EFDT,
19	KNN,
20	LAST,
21	AdaptiveRandomForestClassifier,
22	DynamicWeightedMajority,
23	Finetune,
24	HoeffdingAdaptiveTree,
25	HoeffdingTree,
26	LeveragingBagging,
27	MajorityClass,
28	NaiveBayes,
29	NoChange,
30	OnlineAdwinBagging,
31	OnlineBagging,
32	OnlineSmoothBoost,
33	OzaBoost,
34	PassiveAggressiveClassifier,
35	SAMkNN,
36	SGDClassifier,
37	ShrubsClassifier,
38	StreamingGradientBoostedTrees,
39	StreamingRandomPatches,
40	WeightedkNN,
41	DynamicEnsembleMemberSelection,
42	PLASTIC,
43	)
44	from capymoa.datasets import ElectricityTiny	1✔
45	from capymoa.evaluation import ClassificationEvaluator, prequential_evaluation	1✔
46	from capymoa.misc import load_model, save_model	1✔
47	from capymoa.splitcriteria import GiniSplitCriterion	1✔
48	from capymoa.stream import Schema, Stream	1✔
49	import numpy as np	1✔
50
51
52	@dataclass	1✔
53	class ClassifierTestCase:	1✔
54	test_name: str	1✔
55	"""A unique name to identify your test case. Usually the name of the learner."""	1✔
56	learner_constructor: Callable[[Schema], Classifier]	1✔
57	"""A function that returns a new instance of the learner."""	1✔
58	accuracy: float	1✔
59	"""The expected accuracy of the learner."""	1✔
60	win_accuracy: float	1✔
61	"""The expected windowed accuracy of the learner."""	1✔
62	cli_string: Optional[str] = None	1✔
63	"""The expected CLI string of the learner."""	1✔
64	is_serializable: bool = True	1✔
65	"""Whether the learner is serializable."""	1✔
66	batch_size: int = 1	1✔
67
68	skip_prediction_check_before_training: bool = False	1✔
69	"""Skip checking the prediction before training. If False, the test will fail if the	1✔
70	learner does not output None before training."""
71
72	skip_reason: Optional[str] = None	1✔
73	"""A reason to skip the test. If set, the test will be skipped with this reason."""	1✔
74
75
76	"""	1✔
77	Add your test cases here. Each test case is a `ClassifierTestCase` object
78
79	Notice how we use the `partial` function to creates a new function with
80	hyperparameters already set. This allows us to use the same test function
81	for different learners with different hyperparameters.
82	"""
83	test_cases = [	1✔
84	ClassifierTestCase(
85	"OnlineBagging",
86	partial(OnlineBagging, ensemble_size=5),
87	84.6,
88	89.0,
89	),
90	ClassifierTestCase(
91	"AdaptiveRandomForestClassifier",
92	partial(AdaptiveRandomForestClassifier),
93	89.0,
94	91.0,
95	),
96	ClassifierTestCase(
97	"HoeffdingTree",
98	partial(HoeffdingTree),
99	82.65,
100	83.0,
101	),
102	ClassifierTestCase(
103	"LAST",
104	partial(LAST),
105	83.95,
106	91.0,
107	),
108	ClassifierTestCase(
109	"EFDT",
110	partial(EFDT),
111	82.69,
112	82.0,
113	),
114	ClassifierTestCase(
115	"EFDT-1",
116	partial(
117	EFDT,
118	grace_period=10,
119	split_criterion=GiniSplitCriterion(),
120	leaf_prediction="NaiveBayes",
121	),
122	87.35,
123	84.0,
124	cli_string="trees.EFDT -R 200 -m 33554433 -g 10 -s GiniSplitCriterion -c 0.001 -z -p -l NB",
125	skip_reason="https://github.com/adaptive-machine-learning/backlog/issues/59",
126	),
127	ClassifierTestCase("PLASTIC", PLASTIC, 83.25, 83.0),
128	ClassifierTestCase(
129	"NaiveBayes",
130	partial(NaiveBayes),
131	84.0,
132	91.0,
133	),
134	ClassifierTestCase(
135	"KNN",
136	partial(KNN),
137	81.6,
138	74.0,
139	),
140	ClassifierTestCase(
141	"PassiveAggressiveClassifier",
142	partial(PassiveAggressiveClassifier),
143	84.7,
144	81.0,
145	),
146	ClassifierTestCase(
147	"SGDClassifier",
148	# When loss="log_loss" predict_proba works
149	partial(SGDClassifier, loss="log_loss"),
150	85.75,
151	82.0,
152	),
153	ClassifierTestCase(
154	"StreamingGradientBoostedTrees",
155	partial(StreamingGradientBoostedTrees),
156	88.75,
157	88.0,
158	),
159	ClassifierTestCase(
160	"OzaBoost",
161	partial(OzaBoost),
162	89.95,
163	89.0,
164	),
165	ClassifierTestCase(
166	"MajorityClass",
167	partial(MajorityClass),
168	60.199999999999996,
169	66.0,
170	),
171	ClassifierTestCase(
172	"NoChange",
173	partial(NoChange),
174	85.95,
175	81.0,
176	skip_prediction_check_before_training=True,
177	),
178	ClassifierTestCase(
179	"OnlineSmoothBoost",
180	partial(OnlineSmoothBoost),
181	87.85,
182	90.0,
183	),
184	ClassifierTestCase(
185	"StreamingRandomPatches",
186	partial(StreamingRandomPatches),
187	90.2,
188	89.0,
189	is_serializable=False,
190	),
191	ClassifierTestCase(
192	"HoeffdingAdaptiveTree",
193	partial(HoeffdingAdaptiveTree),
194	84.15,
195	92.0,
196	),
197	ClassifierTestCase(
198	"SAMkNN",
199	partial(SAMkNN),
200	82.65,
201	82.0,
202	),
203	ClassifierTestCase(
204	"DynamicWeightedMajority",
205	partial(DynamicWeightedMajority),
206	84.05,
207	89.0,
208	skip_prediction_check_before_training=True,
209	),
210	ClassifierTestCase(
211	"CSMOTE",
212	partial(CSMOTE),
213	80.55,
214	79.0,
215	),
216	ClassifierTestCase(
217	"LeveragingBagging",
218	partial(LeveragingBagging),
219	86.7,
220	91.0,
221	),
222	ClassifierTestCase(
223	"OnlineAdwinBagging",
224	partial(OnlineAdwinBagging),
225	85.25,
226	92.0,
227	),
228	ClassifierTestCase(
229	"WeightedkNN",
230	partial(WeightedkNN),
231	78.15,
232	70,
233	),
234	ClassifierTestCase("ShrubsClassifier", partial(ShrubsClassifier), 89.6, 91),
235	ClassifierTestCase(
236	"Finetune",
237	partial(
238	Finetune,
239	model=Perceptron,
240	optimizer=partial(torch.optim.Adam, lr=0.001),
241	),
242	60.4,
243	66.0,
244	batch_size=32,
245	skip_prediction_check_before_training=True,
246	),
247	ClassifierTestCase(
248	"DynamicEnsembleMemberSelection",
249	partial(DynamicEnsembleMemberSelection),
250	92.7,
251	90,
252	is_serializable=False,
253	),
254	]
255
256
257	def _score(classifier: Classifier, stream: Stream, limit=100) -> float:	1✔
258	"""Eval without training the classifier."""
259	evaluator = ClassificationEvaluator(schema=stream.get_schema())	1✔
260	stream.restart()	1✔
261	for i, instance in enumerate(stream):	1✔
262	if i >= limit:	1✔
263	break	1✔
264	prediction = classifier.predict(instance)	1✔
265	evaluator.update(instance.y_index, prediction)	1✔
266	return evaluator.accuracy()	1✔
267
268
269	def subtest_save_and_load(	1✔
270	classifier: Classifier,
271	stream: Stream,
272	is_serializable: bool,
273	):
274	"""A subtest to check if a classifier can be saved and loaded."""
275
276	with TemporaryDirectory() as tmp_dir:	1✔
277	tmp_file = os.path.join(tmp_dir, "model.pkl")	1✔
278	with pytest.raises(JException) if not is_serializable else nullcontext():	1✔
279	# Save and load the model
280	with open(tmp_file, "wb") as f:	1✔
281	save_model(classifier, f)	1✔
282
283	with open(tmp_file, "rb") as f:	1✔
284	loaded_classifier: Classifier = load_model(f)	1✔
285
286	# Check that the saved and loaded model have the same accuracy
287	expected_acc = _score(classifier, stream)	1✔
288	loaded_acc = _score(loaded_classifier, stream)	1✔
289	assert expected_acc == loaded_acc, (	1✔
290	f"Original accuracy {expected_acc * 100:.2f} != loaded accuracy {loaded_acc * 100:.2f}"
291	)
292
293	# Check that the loaded model can still be trained
294	loaded_classifier.train(next(stream))	1✔
295
296
297	@pytest.mark.parametrize(	1✔
298	"test_case",
299	test_cases,
300	ids=[c.test_name for c in test_cases],
301	)
302	def test_classifiers(test_case: ClassifierTestCase, subtests: SubTests):	1✔
303	"""``test_classifiers`` is a fast running complex test that checks:
304
305	* Did the classifier reach the expected accuracy?
306	* Did the classifier reach the expected windowed accuracy?
307	* Can the classifier be saved and loaded?
308	* Does the CLI string match the expected value?
309	"""
310	if test_case.skip_reason:	1✔
311	pytest.skip(test_case.skip_reason)	1✔
312
313	stream = ElectricityTiny()	1✔
314	y_shape = (stream.schema.get_num_classes(),)	1✔
315	learner: Classifier = test_case.learner_constructor(schema=stream.get_schema())	1✔
316
317	if not test_case.skip_prediction_check_before_training:	1✔
318	# The classifier should not error when asked to make predictions before
319	# training. Instead it must return None.
320	assert learner.predict(next(stream)) is None	1✔
321	assert learner.predict_proba(next(stream)) is None	1✔
322
323	# Main Loop
324	stream.restart()	1✔
325	results = prequential_evaluation(	1✔
326	stream, learner, window_size=100, batch_size=test_case.batch_size
327	)
328
329	# Check if the learner can make predictions after training and that those
330	# predictions are of the expected type
331	stream.restart()	1✔
332	y_proba = learner.predict_proba(next(stream))	1✔
333	y = learner.predict(next(stream))	1✔
334	assert isinstance(y_proba, np.ndarray), f"Expected numpy array, got {type(y_proba)}"	1✔
335	assert y_proba.dtype == np.float64, f"Expected float64, got {y_proba.dtype}"	1✔
336	assert y_proba.shape == y_shape, f"Expected shape {y_shape}, got {y_proba.shape}"	1✔
337	assert isinstance(y, int), f"Expected int, got {type(y)}"	1✔
338	assert sum(y_proba) == pytest.approx(1.0, abs=1e-5), "Probability sum != 1"	1✔
339
340	# Check if the accuracy matches the expected value for both evaluator types
341	actual_acc = results.cumulative.accuracy()	1✔
342	actual_win_acc = results.windowed.accuracy()[-1]	1✔
343
344	assert actual_acc == pytest.approx(test_case.accuracy, abs=0.1), (	1✔
345	f"Basic Eval: Expected accuracy of {test_case.accuracy:0.1f} got {actual_acc: 0.1f}"
346	)
347	assert actual_win_acc == pytest.approx(test_case.win_accuracy, abs=0.1), (	1✔
348	f"Windowed Eval: Expected accuracy of {test_case.win_accuracy:0.1f} got {actual_win_acc:0.1f}"
349	)
350
351	# Check if the classifier can be saved and loaded
352	with subtests.test(msg="save_and_load"):	1✔
353	subtest_save_and_load(learner, stream, test_case.is_serializable)	1✔
354
355	# Optionally check the CLI string if it was provided
356	if isinstance(learner, MOAClassifier) and test_case.cli_string is not None:	1✔
NEW 357	cli_str = classifier_cli_str(learner).strip("()")	×
358	assert cli_str == test_case.cli_string, "CLI does not match expected value"	×

adaptive-machine-learning / CapyMOA / 26616170697

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous