TestFactory is cocotb-y way for test sweep with different configuration (ie parameters). As part of Cocotb 2.0, TestFactory is deprecated and parameterize is recommended instead.
Starting with the deprecation messages for TestFactory
def __init__(self, test_function: F, *args: Any, **kwargs: Any) -> None:
warnings.warn(
"TestFactory is deprecated, use `@cocotb.parameterize` instead",
DeprecationWarning,
stacklevel=2,
)
Jumping to parameterize decorator where the documentation pretty much sums it up. The test takes the parameters which is cross combination of arg1 and arg2.
@cocotb.test(
skip=False,
)
@cocotb.parameterize(
arg1=[0, 1],
arg2=["a", "b"],
)
async def my_test(arg1: int, arg2: str) -> None: ...
def parameterize(
*options_by_tuple: Union[
Tuple[str, Sequence[Any]], Tuple[Sequence[str], Sequence[Sequence[Any]]]
],
**options_by_name: Sequence[Any],
) -> Callable[[F], _Parameterized[F]]:
"""Decorator to generate parameterized tests from a single test function.
Decorates a test function with named test parameters.
The call to ``parameterize`` should include the name of each test parameter and the possible values each parameter can hold.
This will generate a test for each of the Cartesian products of the parameters and their values.
.. code-block:: python3
@cocotb.test(
skip=False,
)
@cocotb.parameterize(
arg1=[0, 1],
arg2=["a", "b"],
)
async def my_test(arg1: int, arg2: str) -> None: ...
The above is equivalent to the following.
.. code-block:: python3
@cocotb.test(skip=False)
async def my_test_0_a() -> None:
arg1, arg2 = 0, "a"
...
@cocotb.test(skip=False)
async def my_test_0_b() -> None:
arg1, arg2 = 0, "b"
...
@cocotb.test(skip=False)
async def my_test_1_a() -> None:
arg1, arg2 = 1, "a"
...
@cocotb.test(skip=False)
async def my_test_1_b() -> None:
arg1, arg2 = 1, "b"
...
Options can also be specified in much the same way that :meth:`TestFactory.add_option <cocotb.regression.TestFactory.add_option>` can,
either by supplying tuples of the parameter name to values,
or a sequence of variable names and a sequence of values.
.. code-block:: python3
@cocotb.parameterize(
("arg1", [0, 1]),
(("arg2", arg3"), [(1, 2), (3, 4)])
)
Args:
options_by_tuple:
Tuple of parameter name to sequence of values for that parameter,
or tuple of sequence of parameter names to sequence of sequences of values for that pack of parameters.
options_by_name:
Mapping of parameter name to sequence of values for that parameter.
.. versionadded:: 2.0
"""
# check good inputs
for i, option_by_tuple in enumerate(options_by_tuple):
if len(option_by_tuple) != 2:
raise ValueError(
f"Invalid option tuple {i}, expected exactly two fields `(name, values)`"
)
name, values = option_by_tuple
if not isinstance(name, str):
for n in name:
if not n.isidentifier():
raise ValueError("Option names must be valid Python identifiers")
values = cast(Sequence[Sequence[Any]], values)
for value in values:
if len(name) != len(value):
raise ValueError(
f"Invalid option tuple {i}, mismatching number of parameters ({name}) and values ({value})"
)
elif not name.isidentifier():
raise ValueError("Option names must be valid Python identifiers")
options = [*options_by_tuple, *options_by_name.items()]
def wrapper(f: F) -> _Parameterized[F]:
return _Parameterized(f, options)
return wrapper
Digging deeper into how parameterize works, there is generate_tests that yields Test with test_kwargs created from options passed to parameterize
def generate_tests(
self,
*,
name: Optional[str] = None,
timeout_time: Optional[float] = None,
timeout_unit: str = "step",
expect_fail: bool = False,
expect_error: Union[Type[Exception], Sequence[Type[Exception]]] = (),
skip: bool = False,
stage: int = 0,
_expect_sim_failure: bool = False,
) -> Iterable[Test]:
test_func_name = self.test_function.__qualname__ if name is None else name
# this value is a list of ranges of the same length as each set of values in self.options for passing to itertools.product
option_indexes = [range(len(option[1])) for option in self.options]
# go through the cartesian product of all values of all options
for selected_options in product(*option_indexes):
test_kwargs: Dict[str, Sequence[Any]] = {}
test_name_pieces: List[str] = [test_func_name]
for option_idx, select_idx in enumerate(selected_options):
option_name, option_values = self.options[option_idx]
selected_value = option_values[select_idx]
if isinstance(option_name, str):
# single params per option
selected_value = cast(Sequence[Any], selected_value)
test_kwargs[option_name] = selected_value
test_name_pieces.append(
f"/{option_name}={self._option_reprs[option_name][select_idx]}"
)
else:
# multiple params per option
selected_value = cast(Sequence[Any], selected_value)
for n, v in zip(option_name, selected_value):
test_kwargs[n] = v
test_name_pieces.append(
f"/{n}={self._option_reprs[n][select_idx]}"
)
parameterized_test_name = "".join(test_name_pieces)
# create wrapper function to bind kwargs
@functools.wraps(self.test_function)
async def _my_test(dut, kwargs: Dict[str, Any] = test_kwargs) -> None:
await self.test_function(dut, **kwargs)
yield Test(
func=_my_test,
name=parameterized_test_name,
timeout_time=timeout_time,
timeout_unit=timeout_unit,
expect_fail=expect_fail,
expect_error=expect_error,
skip=skip,
stage=stage,
_expect_sim_failure=_expect_sim_failure,
)
So, where is generate_tests is called? Jumping to test discovery, where _add_tests is called to added the generated test to listed of discovered tests.
def _add_tests(module_name: str, *tests: Test) -> None:
mod = sys.modules[module_name]
if not hasattr(mod, "__cocotb_tests__"):
mod.__cocotb_tests__ = []
mod.__cocotb_tests__.extend(tests)
if _func is not None:
if isinstance(_func, _Parameterized):
test_func = _func.test_function
_add_tests(test_func.__module__, *_func.generate_tests())
return test_func
else:
_add_tests(_func.__module__, Test(func=_func))
return _func
Going one level higher to the test discovery where _add_tests is called to start the whole thing.
def wrapper(f: Union[F, _Parameterized[F]]) -> F:
if isinstance(f, _Parameterized):
test_func = f.test_function
_add_tests(
test_func.__module__,
*f.generate_tests(
name=name,
timeout_time=timeout_time,
timeout_unit=timeout_unit,
expect_fail=expect_fail,
expect_error=expect_error,
skip=skip,
stage=stage,
_expect_sim_failure=_expect_sim_failure,
),
)
return test_func
else:
_add_tests(
f.__module__,
Test(
func=f,
name=name,
timeout_time=timeout_time,
timeout_unit=timeout_unit,
expect_fail=expect_fail,
expect_error=expect_error,
skip=skip,
stage=stage,
_expect_sim_failure=_expect_sim_failure,
),
)
return f
return wrapper