From 0dbf80e44e73c9b92ae166bf80570936457c8b11 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 11:47:45 -0800 Subject: [PATCH 01/12] tighten up processing of annotations through normalization and better numpy support --- plugins/python/python/phlex/__init__.py | 111 +-------- plugins/python/python/phlex/_typing.py | 185 +++++++++++++++ plugins/python/python/phlex/_variant.py | 114 +++++++++ plugins/python/src/modulewrap.cpp | 224 ++++++++---------- test/python/CMakeLists.txt | 79 +++--- test/python/test_typing.py | 65 +++++ .../{unit_test_variant.py => test_variant.py} | 42 ++-- test/python/vectypes.py | 111 +++------ 8 files changed, 555 insertions(+), 376 deletions(-) create mode 100644 plugins/python/python/phlex/_typing.py create mode 100644 plugins/python/python/phlex/_variant.py create mode 100644 test/python/test_typing.py rename test/python/{unit_test_variant.py => test_variant.py} (68%) diff --git a/plugins/python/python/phlex/__init__.py b/plugins/python/python/phlex/__init__.py index b21384ff4..501afc841 100644 --- a/plugins/python/python/phlex/__init__.py +++ b/plugins/python/python/phlex/__init__.py @@ -1,109 +1,4 @@ -"""Annotation helper for C++ typing variants. +"""Phlex Python support module.""" -Python algorithms are generic, like C++ templates, but the Phlex registration -process requires a single unique signature. These helpers generate annotated -functions for registration with the proper C++ types. -""" - -import collections -import copy -import inspect -from typing import Any, Callable - - -class MissingAnnotation(Exception): - """Exception noting the missing of an argument in the provied annotations.""" - - def __init__(self, arg: str): - """Construct exception from the name of the argument without annotation.""" - self.arg = arg - - def __str__(self): - """Report the argument that is missing an annotation.""" - return "argument '%s' is not annotated" % self.arg - - -class Variant: - """Wrapper to associate custom annotations with a callable. - - This class wraps a callable and provides custom ``__annotations__`` and - ``__name__`` attributes, allowing the same underlying function or callable - object to be registered multiple times with different type annotations. - - By default, the provided callable is kept by reference, but can be cloned - (e.g. for callable instances) if requested. - - Phlex will recognize the "phlex_callable" data member, allowing an unwrap - and thus saving an indirection. To detect performance degradation, the - wrapper is not callable by default. - - Attributes: - phlex_callable (Callable): The underlying callable (public). - __annotations__ (dict): Type information of arguments and return product. - __name__ (str): The name associated with this variant. - - Examples: - >>> def add(i: Number, j: Number) -> Number: - ... return i + j - ... - >>> int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd") - """ - - def __init__( - self, - f: Callable, - annotations: dict[str, str | type | Any], - name: str, - clone: bool | str = False, - allow_call: bool = False, - ): - """Annotate the callable F. - - Args: - f (Callable): Annotable function. - annotations (dict): Type information of arguments and return product. - name (str): Name to assign to this variant. - clone (bool|str): If True (or "deep"), creates a shallow (deep) copy - of the callable. - allow_call (bool): Allow this wrapper to forward to the callable. - """ - if clone == "deep": - self.phlex_callable = copy.deepcopy(f) - elif clone: - self.phlex_callable = copy.copy(f) - else: - self.phlex_callable = f - - # annotions are expected as an ordinary dict and should be ordered, but - # we do not require it, so re-order based on the function's co_varnames - self.__annotations__ = collections.OrderedDict() - - sig = inspect.signature(self.phlex_callable) - for k, v in sig.parameters.items(): - try: - self.__annotations__[k] = annotations[k] - except KeyError as e: - if v.default is inspect.Parameter.empty: - raise MissingAnnotation(k) from e - - self.__annotations__["return"] = annotations.get("return", None) - - self.__name__ = name - self.__code__ = getattr(self.phlex_callable, "__code__", None) - self.__defaults__ = getattr(self.phlex_callable, "__defaults__", None) - self._allow_call = allow_call - - def __call__(self, *args, **kwargs): - """Raises an error if called directly. - - Variant instances should not be called directly. The framework should - extract ``phlex_callable`` instead and call that. - - Raises: - AssertionError: To indicate incorrect usage, unless overridden. - """ - assert self._allow_call, ( - f"Variant '{self.__name__}' was called directly. " - f"The framework should extract phlex_callable instead." - ) - return self.phlex_callable(*args, **kwargs) # type: ignore +from ._typing import * # noqa: F403 no names used in this file +from ._variant import * # noqa: F403 idem diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py new file mode 100644 index 000000000..de3bbb293 --- /dev/null +++ b/plugins/python/python/phlex/_typing.py @@ -0,0 +1,185 @@ +"""Tooling to simplify representation of C++ types in Python. + +There is no canonical form of Python annotations and this gets even more +complicated with the existence of C++ types that have no representation in +Python. This tooling provides helpers to simplify what is presented to the +C++ side, and thus simplifies the maintenance of that code. +""" + +import builtins +import ctypes +import typing +from types import UnionType +from typing import Any, Dict, Union + +import numpy as np + +__all__ = [ + "normalize_type", +] + +# ctypes and numpy types are likely candidates for use in annotations +# TODO: should users be allowed to add to these? +_PY2CPP: dict[type, str] = { + # numpy types + np.bool_: "bool", + np.int8: "int8_t", + np.int16: "int16_t", + np.int32: "int32_t", + np.int64: "int64_t", + np.uint8: "uint8_t", + np.uint16: "uint16_t", + np.uint32: "uint32_t", + np.uint64: "uint64_t", + np.float32: "float", + np.float64: "double", + np.complex64: "std::complex", + np.complex128: "std::complex", + # the following types are aliased in numpy, so ignore here + #np.intp: "ptrdiff_t", + #np.uintp: "size_t", +} + +# ctypes types that don't map cleanly to intN_t / uintN_t +_CTYPES_SPECIAL: dict[type, str] = { + ctypes.c_bool: "bool", + ctypes.c_char: "char", # signedness is implementation-defined in C + ctypes.c_wchar: "wchar_t", + ctypes.c_float: "float", # always IEEE 754 32-bit + ctypes.c_double: "double", # always IEEE 754 64-bit + ctypes.c_longdouble:"long double",# platform-dependent width, no stdint.h alias + # the following types are aliased in ctypes, so ignore here + #ctypes.c_size_t: "size_t", + #ctypes.c_ssize_t: "ssize_t", + ctypes.c_void_p: "void*", + ctypes.c_char_p: "const char*", + ctypes.c_wchar_p: "const wchar_t*", +} + +_CTYPES_INTEGER: list[type] = [ + ctypes.c_byte, ctypes.c_ubyte, + ctypes.c_short, ctypes.c_ushort, + ctypes.c_int, ctypes.c_uint, + ctypes.c_long, ctypes.c_ulong, +] + +# (unsigned) long long may be aliased +if ctypes.c_longlong is not ctypes.c_long: + _CTYPES_INTEGER.append(ctypes.c_longlong) + _CTYPES_INTEGER.append(ctypes.c_ulonglong) + +def _build_ctypes_map() -> dict[type, str]: + result = dict(_CTYPES_SPECIAL) + for tp in _CTYPES_INTEGER: + bits = ctypes.sizeof(tp) * 8 + # _type_ uses struct format chars: lowercase = signed, uppercase = unsigned + signed = tp._type_.islower() + result[tp] = f"{'int' if signed else 'uint'}{bits}_t" + return result + +_PY2CPP.update(_build_ctypes_map()) + +# use ctypes to construct a mapping from platform types to exact types +_C2C: dict[str, str] = { + "short": _PY2CPP[ctypes.c_short], + "unsigned short": _PY2CPP[ctypes.c_ushort], + "int": _PY2CPP[ctypes.c_int], + "unsigned int": _PY2CPP[ctypes.c_uint], + "long": _PY2CPP[ctypes.c_long], + "unsigned long": _PY2CPP[ctypes.c_ulong], + "long long": 'int64_t', + "unsigned long long": 'uint64_t', +} + + +def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) -> str: + """Recursively normalize any Python annotation to a canonical name. + + This normalization supports: + - raw types: int, str, float + - typing generics: List[int], Dict[str, int], etc. + - built-in generics (3.9+): list[int], dict[str, int] + - string annotations: "list[int]", "list['int']" + - NoneType + + Args: + tp (Any): Some type annotation to normalize. + globalns (dict): optional global namespace to resolve types. + localns (dict): optional local namespace to resolve types. + + Returns: + Canonical string representation of the type. + """ + # most common case of some string; resolve it to handle `list[int]`, `list['int']` + if isinstance(tp, str): + ns = {**vars(builtins), **vars(typing)} + if globalns: + ns.update(globalns) + if localns: + ns.update(localns) + try: + tp = eval(tp, ns) + except Exception: + return _C2C.get(tp, tp) # unresolvable + + # get the unsubscripted version of the type, or None if it's something unknown + origin = typing.get_origin(tp) + + # sanity check: we have to reject union syntax, because the chosen C++ type must + # be unambiguous (TODO: what with `Optional`?) + if isinstance(tp, UnionType) or origin is Union: + raise TypeError("To support C++, annotations passed to Phlex must be unambiguous") + + # TODO: debatable: maybe pointers should be considered arrays by default? + if isinstance(tp, type) and issubclass(tp, ctypes._Pointer): + raise TypeError("Pointers are ambiguous; declare an array if that is intended") + + # common case of forward references, from e.g. `List["double"]` + if isinstance(tp, typing.ForwardRef): + # ForwardRef.__forward_arg__ is the a string; recurse to resolve it + return normalize_type(tp.__forward_arg__, globalns, localns) + + # special case for NoneType + if tp is type(None): + return "None" + + # clean up generic aliases, such as typing.List[int], list[int], etc. + if origin is not None: + args = typing.get_args(tp) + + if origin is np.ndarray: # numpy arrays + dtype_args = typing.get_args(args[1]) if len(args) >= 2 else () + if dtype_args: + return "ndarray[" + normalize_type(dtype_args[0], globalns, localns) + "]" + else: + raise TypeError("Numpy array with unparameterized or no scalar type") + + if isinstance(origin, type): # regular python typing type + name = origin.__name__ + else: + # fallback for unexpected origins + name = getattr(origin, "__name__", repr(origin)) + + return name + "[" + ",".join([normalize_type(a) for a in args]) + "]" + + # ctypes (fixed-size) array types + try: + if issubclass(tp, ctypes.Array): + # TODO: tp._length_ may be useful as well + return "array["+normalize_type(tp._type_, globalns, localns)+"]" + except TypeError: + pass # tp is not a class + + # known builtin types representations from ctypes and numpy + try: + return _PY2CPP[tp] + except KeyError: + pass # not a known Python type + + # fallback for plain Python types + if isinstance(tp, type): + return _C2C.get(tp.__name__, tp.__name__) + + # fallback for everything else, expecting repr() to be unique and consistent + return repr(tp) + diff --git a/plugins/python/python/phlex/_variant.py b/plugins/python/python/phlex/_variant.py new file mode 100644 index 000000000..162939894 --- /dev/null +++ b/plugins/python/python/phlex/_variant.py @@ -0,0 +1,114 @@ +"""Annotation helper for C++ typing variants. + +Python algorithms are generic, like C++ templates, but the Phlex registration +process requires a single unique signature. These helpers generate annotated +functions for registration with the proper C++ types. +""" + +import collections +import copy +import inspect +from typing import Any, Callable + +__all__ = [ + "MissingAnnotation", + "Variant", +] + + +class MissingAnnotation(Exception): + """Exception noting the missing of an argument in the provied annotations.""" + + def __init__(self, arg: str): + """Construct exception from the name of the argument without annotation.""" + self.arg = arg + + def __str__(self): + """Report the argument that is missing an annotation.""" + return "argument '%s' is not annotated" % self.arg + + +class Variant: + """Wrapper to associate custom annotations with a callable. + + This class wraps a callable and provides custom ``__annotations__`` and + ``__name__`` attributes, allowing the same underlying function or callable + object to be registered multiple times with different type annotations. + + By default, the provided callable is kept by reference, but can be cloned + (e.g. for callable instances) if requested. + + Phlex will recognize the "phlex_callable" data member, allowing an unwrap + and thus saving an indirection. To detect performance degradation, the + wrapper is not callable by default. + + Attributes: + phlex_callable (Callable): The underlying callable (public). + __annotations__ (dict): Type information of arguments and return product. + __name__ (str): The name associated with this variant. + + Examples: + >>> def add(i: Number, j: Number) -> Number: + ... return i + j + ... + >>> int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd") + """ + + def __init__( + self, + f: Callable, + annotations: dict[str, str | type | Any], + name: str, + clone: bool | str = False, + allow_call: bool = False, + ): + """Annotate the callable F. + + Args: + f (Callable): Annotable function. + annotations (dict): Type information of arguments and return product. + name (str): Name to assign to this variant. + clone (bool|str): If True (or "deep"), creates a shallow (deep) copy + of the callable. + allow_call (bool): Allow this wrapper to forward to the callable. + """ + if clone == "deep": + self.phlex_callable = copy.deepcopy(f) + elif clone: + self.phlex_callable = copy.copy(f) + else: + self.phlex_callable = f + + # annotions are expected as an ordinary dict and should be ordered, but + # we do not require it, so re-order based on the function's co_varnames + self.__annotations__ = collections.OrderedDict() + + sig = inspect.signature(self.phlex_callable) + for k, v in sig.parameters.items(): + try: + self.__annotations__[k] = annotations[k] + except KeyError as e: + if v.default is inspect.Parameter.empty: + raise MissingAnnotation(k) from e + + self.__annotations__["return"] = annotations.get("return", None) + + self.__name__ = name + self.__code__ = getattr(self.phlex_callable, "__code__", None) + self.__defaults__ = getattr(self.phlex_callable, "__defaults__", None) + self._allow_call = allow_call + + def __call__(self, *args, **kwargs): + """Raises an error if called directly. + + Variant instances should not be called directly. The framework should + extract ``phlex_callable`` instead and call that. + + Raises: + AssertionError: To indicate incorrect usage, unless overridden. + """ + assert self._allow_call, ( + f"Variant '{self.__name__}' was called directly. " + f"The framework should extract phlex_callable instead." + ) + return self.phlex_callable(*args, **kwargs) # type: ignore diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index c48973621..8f4355028 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -298,54 +298,30 @@ namespace { static std::string annotation_as_text(PyObject* pyobj) { - std::string ann; - if (!PyUnicode_Check(pyobj)) { - PyObject* pystr = PyObject_GetAttrString(pyobj, "__name__"); // eg. for classes - - // generics like Union have a __name__ that is not useful for our purposes - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr && (strcmp(cstr, "Union") == 0 || strcmp(cstr, "Optional") == 0)) { - Py_DECREF(pystr); - pystr = nullptr; - } + static PyObject* normalizer = nullptr; + if (!normalizer) { + PyObject* phlexmod = PyImport_ImportModule("phlex"); + if (phlexmod) { + normalizer = PyObject_GetAttrString(phlexmod, "normalize_type"); + Py_DECREF(phlexmod); } - if (!pystr) { - PyErr_Clear(); - pystr = PyObject_Str(pyobj); - } - - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr) - ann = cstr; - Py_DECREF(pystr); + if (!normalizer) { + std::string msg; + if (msg_from_py_error(msg, false)) + throw std::runtime_error("unable to retrieve the phlex type normalizer: " + msg); } + } - // for numpy typing, there's no useful way of figuring out the type from the - // name of the type, only from its string representation, so fall through and - // let this method return str() - if (ann != "ndarray" && ann != "list") - return ann; - - // start over for numpy type using result from str() - pystr = PyObject_Str(pyobj); - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr) // if failed, ann will remain "ndarray" - ann = cstr; - Py_DECREF(pystr); - } - return ann; + PyObject* norm = PyObject_CallOneArg(normalizer, pyobj); + if (!norm) { + std::string msg; + if (msg_from_py_error(msg, false)) + throw std::runtime_error("normalization error: " + msg); } - // unicode object, i.e. string name of the type - char const* cstr = PyUnicode_AsUTF8(pyobj); - if (cstr) - ann = cstr; - else - PyErr_Clear(); + std::string ann = PyUnicode_AsUTF8(norm); + Py_DECREF(norm); return ann; } @@ -369,11 +345,23 @@ namespace { static long pylong_as_strictlong(PyObject* pyobject) { // convert to C++ long, don't allow truncation - if (!PyLong_Check(pyobject)) { - PyErr_SetString(PyExc_TypeError, "int/long conversion expects an integer object"); - return (long)-1; + if (PyLong_Check(pyobject)) { // native Python integer + return PyLong_AsLong(pyobject); + } + + // accept numpy signed integer scalars (int8, int16, int32, int64) + if (PyArray_IsScalar(pyobject, SignedInteger)) { + // convert to Python int first, then to C long, that way we get a Python + // OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); + if (!pylong) return (long)-1; + long result = PyLong_AsLong(pylong); + Py_DECREF(pylong); + return result; } - return (long)PyLong_AsLong(pyobject); // already does long range check + + PyErr_SetString(PyExc_TypeError, "int/long conversion expects a signed integer object"); + return (long)-1; } static unsigned long pylong_or_int_as_ulong(PyObject* pyobject) @@ -384,6 +372,17 @@ namespace { return (unsigned long)-1; } + // accept numpy unsigned integer scalars (uint8, uint16, uint32, uint64) + if (PyArray_IsScalar(pyobject, UnsignedInteger)) { + // convert to Python int first, then to C unsigned long, that way we get a + // Python OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); + if (!pylong) return (long)-1; + unsigned long result = PyLong_AsUnsignedLong(pylong); + Py_DECREF(pylong); + return result; + } + unsigned long ul = PyLong_AsUnsignedLong(pyobject); if (ul == (unsigned long)-1 && PyErr_Occurred() && PyLong_Check(pyobject)) { PyErr_Clear(); @@ -420,10 +419,10 @@ namespace { } BASIC_CONVERTER(bool, bool, PyBool_FromLong, pylong_as_bool) - BASIC_CONVERTER(int, int, PyLong_FromLong, PyLong_AsLong) - BASIC_CONVERTER(uint, unsigned int, PyLong_FromLong, pylong_or_int_as_ulong) - BASIC_CONVERTER(long, long, PyLong_FromLong, pylong_as_strictlong) - BASIC_CONVERTER(ulong, unsigned long, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) + BASIC_CONVERTER(int, std::int32_t, PyLong_FromLong, PyLong_AsLong) + BASIC_CONVERTER(uint, std::uint32_t, PyLong_FromLong, pylong_or_int_as_ulong) + BASIC_CONVERTER(long, std::int64_t, PyLong_FromLong, pylong_as_strictlong) + BASIC_CONVERTER(ulong, std::uint64_t, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) BASIC_CONVERTER(float, float, PyFloat_FromDouble, PyFloat_AsDouble) BASIC_CONVERTER(double, double, PyFloat_FromDouble, PyFloat_AsDouble) @@ -466,10 +465,10 @@ namespace { return (intptr_t)pyll; \ } - VECTOR_CONVERTER(vint, int, NPY_INT) - VECTOR_CONVERTER(vuint, unsigned int, NPY_UINT) - VECTOR_CONVERTER(vlong, long, NPY_LONG) - VECTOR_CONVERTER(vulong, unsigned long, NPY_ULONG) + VECTOR_CONVERTER(vint, std::int32_t, NPY_INT32) + VECTOR_CONVERTER(vuint, std::uint32_t, NPY_UINT32) + VECTOR_CONVERTER(vlong, std::int64_t, NPY_INT64) + VECTOR_CONVERTER(vulong, std::uint64_t, NPY_UINT64) VECTOR_CONVERTER(vfloat, float, NPY_FLOAT) VECTOR_CONVERTER(vdouble, double, NPY_DOUBLE) @@ -517,10 +516,10 @@ namespace { return vec; \ } - NUMPY_ARRAY_CONVERTER(vint, int, NPY_INT, PyLong_AsLong) - NUMPY_ARRAY_CONVERTER(vuint, unsigned int, NPY_UINT, pylong_or_int_as_ulong) - NUMPY_ARRAY_CONVERTER(vlong, long, NPY_LONG, pylong_as_strictlong) - NUMPY_ARRAY_CONVERTER(vulong, unsigned long, NPY_ULONG, pylong_or_int_as_ulong) + NUMPY_ARRAY_CONVERTER(vint, std::int32_t, NPY_INT32, PyLong_AsLong) + NUMPY_ARRAY_CONVERTER(vuint, std::uint32_t, NPY_UINT32, pylong_or_int_as_ulong) + NUMPY_ARRAY_CONVERTER(vlong, std::int64_t, NPY_INT64, pylong_as_strictlong) + NUMPY_ARRAY_CONVERTER(vulong, std::uint64_t, NPY_UINT64, pylong_or_int_as_ulong) NUMPY_ARRAY_CONVERTER(vfloat, float, NPY_FLOAT, PyFloat_AsDouble) NUMPY_ARRAY_CONVERTER(vdouble, double, NPY_DOUBLE, PyFloat_AsDouble) @@ -687,52 +686,34 @@ static bool insert_input_converters(py_phlex_module* mod, if (inp_type == "bool") insert_converter(mod, pyname, bool_to_py, inp_pq, output); - else if (inp_type == "int") + else if (inp_type == "int32_t") insert_converter(mod, pyname, int_to_py, inp_pq, output); - else if (inp_type == "unsigned int") + else if (inp_type == "uint32_t") insert_converter(mod, pyname, uint_to_py, inp_pq, output); - else if (inp_type == "long") + else if (inp_type == "int64_t") insert_converter(mod, pyname, long_to_py, inp_pq, output); - else if (inp_type == "unsigned long") + else if (inp_type == "uint64_t") insert_converter(mod, pyname, ulong_to_py, inp_pq, output); else if (inp_type == "float") insert_converter(mod, pyname, float_to_py, inp_pq, output); else if (inp_type == "double") insert_converter(mod, pyname, double_to_py, inp_pq, output); - else if (inp_type.compare(0, 13, "numpy.ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { + else if (inp_type.compare(0, 7, "ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { // TODO: these are hard-coded std::vector <-> numpy array mappings, which is // way too simplistic for real use. It only exists for demonstration purposes, // until we have an IDL - auto pos = inp_type.rfind("numpy.dtype"); - if (pos == std::string::npos) { - if (inp_type[0] == 'l') { - pos = 0; - } else { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of input type \"%s\"", inp_type.c_str()); - return false; - } - } else { - pos += 18; - } - - if (inp_type.compare(pos, std::string::npos, "uint32]]") == 0 || - inp_type == "list[unsigned int]" || inp_type == "list['unsigned int']") { - insert_converter(mod, pyname, vuint_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "int32]]") == 0 || - inp_type == "list[int]") { + std::string_view dtype{inp_type.begin() + inp_type.rfind('['), inp_type.end()}; + if (dtype == "[int32_t]") { insert_converter(mod, pyname, vint_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true - inp_type == "list[unsigned long]" || inp_type == "list['unsigned long']") { - insert_converter(mod, pyname, vulong_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. - inp_type == "list[long]" || inp_type == "list['long']") { + } else if (dtype == "[uint32_t]") { + insert_converter(mod, pyname, vuint_to_py, inp_pq, output); + } else if (dtype == "[int64_t]") { insert_converter(mod, pyname, vlong_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "float32]]") == 0 || - inp_type == "list[float]") { + } else if (dtype == "[uint64_t]") { + insert_converter(mod, pyname, vulong_to_py, inp_pq, output); + } else if (dtype == "[float]") { insert_converter(mod, pyname, vfloat_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "float64]]") == 0 || - inp_type == "list[double]" || inp_type == "list['double']") { + } else if (dtype == "[double]") { insert_converter(mod, pyname, vdouble_to_py, inp_pq, output); } else { PyErr_Format(PyExc_TypeError, "unsupported collection input type \"%s\"", inp_type.c_str()); @@ -797,8 +778,8 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw std::string suff0 = "py_" + std::string{static_cast(*pq0.suffix)}; switch (input_queries.size()) { - case 1: { - auto* pyc = new py_callback_1{callable}; // TODO: leaks, but has program lifetime + case 1: { + auto* pyc = new py_callback_1{callable}; // TODO: leaks, but has program lifetime mod->ph_module->transform(pyname, *pyc, concurrency::serial) .input_family( product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}) @@ -845,64 +826,45 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw auto out_pq = product_query{.creator = identifier(pyname), .layer = identifier(output_layer), .suffix = identifier(pyoutput)}; - std::string output_type = output_types[0]; + std::string out_type = output_types[0]; std::string output = output_labels[0]; - if (output_type == "bool") + if (out_type == "bool") insert_converter(mod, cname, py_to_bool, out_pq, output); - else if (output_type == "int") + else if (out_type == "int32_t") insert_converter(mod, cname, py_to_int, out_pq, output); - else if (output_type == "unsigned int") + else if (out_type == "uint32_t") insert_converter(mod, cname, py_to_uint, out_pq, output); - else if (output_type == "long") + else if (out_type == "int64_t") insert_converter(mod, cname, py_to_long, out_pq, output); - else if (output_type == "unsigned long") + else if (out_type == "uint64_t") insert_converter(mod, cname, py_to_ulong, out_pq, output); - else if (output_type == "float") + else if (out_type == "float") insert_converter(mod, cname, py_to_float, out_pq, output); - else if (output_type == "double") + else if (out_type == "double") insert_converter(mod, cname, py_to_double, out_pq, output); - else if (output_type.compare(0, 13, "numpy.ndarray") == 0 || - output_type.compare(0, 4, "list") == 0) { + else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { // TODO: just like for input types, these are hard-coded, but should be handled by // an IDL instead. - auto pos = output_type.rfind("numpy.dtype"); - if (pos == std::string::npos) { - if (output_type[0] == 'l') { - pos = 0; - } else { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of output type \"%s\"", output_type.c_str()); - return nullptr; - } - } else { - pos += 18; - } - - if (output_type.compare(pos, std::string::npos, "uint32]]") == 0 || - output_type == "list[unsigned int]" || output_type == "list['unsigned int']") { - insert_converter(mod, cname, py_to_vuint, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "int32]]") == 0 || - output_type == "list[int]") { + std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; + if (dtype == "[int32_t]") { insert_converter(mod, cname, py_to_vint, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true - output_type == "list[unsigned long]" || output_type == "list['unsigned long']") { - insert_converter(mod, cname, py_to_vulong, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. - output_type == "list[long]" || output_type == "list['long']") { + } else if (dtype == "[uint32_t]") { + insert_converter(mod, cname, py_to_vuint, out_pq, output); + } else if (dtype == "[int64_t]") { insert_converter(mod, cname, py_to_vlong, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "float32]]") == 0 || - output_type == "list[float]") { + } else if (dtype == "[uint64_t]") { + insert_converter(mod, cname, py_to_vulong, out_pq, output); + } else if (dtype == "[float]") { insert_converter(mod, cname, py_to_vfloat, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "float64]]") == 0 || - output_type == "list[double]" || output_type == "list['double']") { + } else if (dtype == "[double]") { insert_converter(mod, cname, py_to_vdouble, out_pq, output); } else { PyErr_Format( - PyExc_TypeError, "unsupported collection output type \"%s\"", output_type.c_str()); + PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); return nullptr; } } else { - PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", output_type.c_str()); + PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", out_type.c_str()); return nullptr; } diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 20fd97abd..3b0c7e369 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -56,11 +56,36 @@ check_python_module_version("numba" "0.61.0" HAS_NUMBA) check_python_module_version("numpy" "2.0.0" HAS_NUMPY) check_python_module_version("pytest_cov" "4.0.0" HAS_PYTEST_COV) -if(HAS_CPPYY) - # Explicitly define Python executable variable to ensure it is visible in - # the test environment - set(PYTHON_TEST_EXECUTABLE ${Python_EXECUTABLE}) +# Explicitly define Python executable variable to ensure it is visible in +# the test environment +set(PYTHON_TEST_EXECUTABLE ${Python_EXECUTABLE}) + +set(PYTHON_TEST_FILES test_typing.py test_variant.py) + +# Determine pytest command based on coverage support +if(HAS_PYTEST_COV AND ENABLE_COVERAGE) + set( + PYTEST_COMMAND + ${PYTHON_TEST_EXECUTABLE} + -m + pytest + --cov=${CMAKE_CURRENT_SOURCE_DIR} + --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing + --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc + ${PYTHON_TEST_FILES} + ) + message(STATUS "Python tests will run with coverage reporting (pytest-cov)") +else() + set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) + if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) + message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") + endif() +endif() +if(HAS_CPPYY) # Export the location of phlex include headers if(DEFINED ENV{PHLEX_INSTALL}) set(PYTHON_TEST_PHLEX_INSTALL $ENV{PHLEX_INSTALL}) @@ -68,35 +93,9 @@ if(HAS_CPPYY) set(PYTHON_TEST_PHLEX_INSTALL ${CMAKE_SOURCE_DIR}) endif() - set(PYTHON_TEST_FILES test_phlex.py unit_test_variant.py) - - # Determine pytest command based on coverage support - if(HAS_PYTEST_COV AND ENABLE_COVERAGE) - set( - PYTEST_COMMAND - ${PYTHON_TEST_EXECUTABLE} - -m - pytest - --cov=${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing - --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml - --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc - ${PYTHON_TEST_FILES} - ) - message(STATUS "Python tests will run with coverage reporting (pytest-cov)") - else() - set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) - if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) - message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") - endif() - endif() - - # tests of the python support modules (relies on cppyy) - add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - set_property(TEST py:phlex PROPERTY ENVIRONMENT "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}") + + set(PYTHON_TEST_FILES ${PYTHON_TEST_FILES} test_phlex.py) endif() set(ACTIVE_PY_CPHLEX_TESTS "") @@ -250,13 +249,6 @@ set_tests_properties( ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" ) -# Unit tests for the phlex python package -add_test( - NAME py:unit_variant - COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/unit_test_variant.py -) -set_tests_properties(py:unit_variant PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") - # Python coverage target if(HAS_PYTEST_COV AND ENABLE_COVERAGE) if(HAS_CPPYY) @@ -271,7 +263,7 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) ) else() # When cppyy is not available, run a simpler pytest command - # for standalone Python tests (like unit_test_variant.py) + # for standalone Python tests (like test_variant.py) set(PYTHON_TEST_PHLEX_INSTALL_FALLBACK ${CMAKE_SOURCE_DIR}) add_custom_target( coverage-python @@ -281,9 +273,14 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) --cov=${CMAKE_CURRENT_SOURCE_DIR} --cov=${PROJECT_SOURCE_DIR}/plugins/python/python --cov-report=term-missing --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc unit_test_variant.py + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report (standalone tests only)" ) endif() endif() + +# tests of the python support modules +add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +set_tests_properties(py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") + diff --git a/test/python/test_typing.py b/test/python/test_typing.py new file mode 100644 index 000000000..62fc25402 --- /dev/null +++ b/test/python/test_typing.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Unit tests for phlex typing normalization.""" + +import ctypes +from typing import List + +import numpy as np +import numpy.typing as npt +from phlex._typing import _C2C + +from phlex import normalize_type + + +class TestTYPING: + """Tests for phlex-specific typinh.""" + + def test_list_normalization(self): + """Normalization of various forms of list annotations.""" + for types in (["int", int, ctypes.c_int], + ["unsigned int", ctypes.c_uint], + ["long", ctypes.c_long], + ["unsigned long", ctypes.c_ulong], + ["long long", ctypes.c_longlong], + ["unsigned long long", ctypes.c_ulonglong], + ["float", float, ctypes.c_float], + ["double", ctypes.c_double],): + # TODO: the use of _C2C here is a bit circular + tn = _C2C.get(types[0], types[0]) + + if 0 < tn.find('_'): + npt = tn[:tn.find('_')] + elif tn == "float": + npt = "float32" + elif tn == "double": + npt = "float64" + npt = getattr(np, npt) + + norm = "list[" + tn + "]" + for t in types + [npt]: + assert normalize_type(norm) == norm + try: + assert normalize_type(List[t]) == norm + except SyntaxError as e: + # some of the above are not legal Python when used with List, + # but are fine when used with list; just ignore these as they + # will not show up in user code + if "Forward reference must be an expression" not in str(e): + raise + assert normalize_type(list[t]) == norm + + def test_numpy_array_normalization(self): + """Normalization of standard Numpy typing.""" + for t, s in ((np.bool_, "bool"), + (np.int8, "int8_t"), + (np.int16, "int16_t"), + (np.int32, "int32_t"), + (np.int64, "int64_t"), + (np.uint8, "uint8_t"), + (np.uint16, "uint16_t"), + (np.uint32, "uint32_t"), + (np.uint64, "uint64_t"), + (np.float32, "float"), + (np.float64, "double"),): + assert normalize_type(npt.NDArray[t]) == "ndarray["+s+"]" + diff --git a/test/python/unit_test_variant.py b/test/python/test_variant.py similarity index 68% rename from test/python/unit_test_variant.py rename to test/python/test_variant.py index 0731bb81a..27a204259 100644 --- a/test/python/unit_test_variant.py +++ b/test/python/test_variant.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Unit tests for the phlex.Variant class.""" -import unittest +from pytest import raises from phlex import MissingAnnotation, Variant @@ -14,31 +14,32 @@ def example_func(a, b=1): ann = {"a": int, "b": int, "return": int} -class TestVariant(unittest.TestCase): +class TestVariant: """Tests for Variant wrapper.""" def test_initialization(self): """Test proper initialization and attribute exposure.""" wrapper = Variant(example_func, ann, "example_wrapper") - self.assertEqual(wrapper.__name__, "example_wrapper") - self.assertEqual(wrapper.__annotations__, ann) - self.assertEqual(wrapper.phlex_callable, example_func) + assert wrapper.__name__ == "example_wrapper" + assert wrapper.__annotations__ == ann + assert wrapper.phlex_callable == example_func # Check introspection attributes are exposed - self.assertEqual(wrapper.__code__, example_func.__code__) - self.assertEqual(wrapper.__defaults__, example_func.__defaults__) + assert wrapper.__code__ == example_func.__code__ + assert wrapper.__defaults__ == example_func.__defaults__ def test_call_by_default_raises(self): """Test that calling the wrapper raises AssertionError by default.""" wrapper = Variant(example_func, ann, "no_call") - with self.assertRaises(AssertionError) as cm: + with raises(AssertionError) as cm: wrapper(1) - self.assertIn("was called directly", str(cm.exception)) + + assert "was called directly" in str(cm.value) def test_allow_call(self): """Test that calling is allowed when configured.""" wrapper = Variant(example_func, ann, "yes_call", allow_call=True) - self.assertEqual(wrapper(10, 20), 30) + assert wrapper(10, 20) == 30 def test_clone_shallow(self): """Test shallow cloning behavior.""" @@ -46,7 +47,7 @@ def test_clone_shallow(self): # but let's test the flag logic in Variant wrapper = Variant(example_func, ann, "clone_shallow", clone=True) # function copy is same object - self.assertEqual(wrapper.phlex_callable, example_func) + assert wrapper.phlex_callable == example_func # Test valid copy logic with a mutable callable class CallableObj: @@ -55,7 +56,7 @@ def __call__(self): obj = CallableObj() wrapper_obj = Variant(obj, {}, "obj_clone", clone=True) - self.assertNotEqual(id(wrapper_obj.phlex_callable), id(obj)) # copy was made? + assert id(wrapper_obj.phlex_callable) != id(obj) # copy was made? # copy.copy of a custom object usually creates a new instance if generic def test_clone_deep(self): @@ -70,8 +71,8 @@ def __call__(self): c = Container() wrapper = Variant(c, {}, "deep_clone", clone="deep") - self.assertNotEqual(id(wrapper.phlex_callable), id(c)) - self.assertNotEqual(id(wrapper.phlex_callable.data), id(c.data)) + assert id(wrapper.phlex_callable) != id(c) + assert id(wrapper.phlex_callable.data) != id(c.data) def test_missing_annotation_raises(self): """Test that MissingAnnotation is raised when a required argument is missing.""" @@ -81,11 +82,11 @@ def func(x, y): # Missing 'y' incomplete_ann = {"x": int, "return": int} - with self.assertRaises(MissingAnnotation) as cm: + with raises(MissingAnnotation) as cm: Variant(func, incomplete_ann, "missing_y") - self.assertEqual(str(cm.exception), "argument 'y' is not annotated") - self.assertEqual(cm.exception.arg, "y") + assert str(cm.value) == "argument 'y' is not annotated" + assert cm.value.arg == "y" def test_missing_optional_annotation_does_not_raise(self): """Test that MissingAnnotation is not raised for arguments with default values.""" @@ -96,9 +97,6 @@ def func(x, y=1): # Missing 'y', but it has a default value incomplete_ann = {"x": int, "return": int} wrapper = Variant(func, incomplete_ann, "missing_optional_y") - self.assertIn("x", wrapper.__annotations__) - self.assertNotIn("y", wrapper.__annotations__) - + assert "x" in wrapper.__annotations__ + assert "y" not in wrapper.__annotations__ -if __name__ == "__main__": - unittest.main() diff --git a/test/python/vectypes.py b/test/python/vectypes.py index de182aabb..fea621048 100644 --- a/test/python/vectypes.py +++ b/test/python/vectypes.py @@ -10,44 +10,6 @@ # Type aliases for C++ types that don't have Python equivalents # These are used by the C++ wrapper to identify the correct converter - -class _CppTypeMeta(type): - """Metaclass to allow overriding __name__ for C++ type identification.""" - - def __new__(mcs, name, bases, namespace, cpp_name=None): - cls = super().__new__(mcs, name, bases, namespace) - cls._cpp_name = cpp_name if cpp_name else name - return cls - - @property - def __name__(cls): - return cls._cpp_name - - -class unsigned_int(int, metaclass=_CppTypeMeta, cpp_name="unsigned int"): # noqa: N801 - """Type alias for C++ unsigned int.""" - - pass - - -class unsigned_long(int, metaclass=_CppTypeMeta, cpp_name="unsigned long"): # noqa: N801 - """Type alias for C++ unsigned long.""" - - pass - - -class long(int, metaclass=_CppTypeMeta, cpp_name="long"): # noqa: N801, A001 - """Type alias for C++ long.""" - - pass - - -class double(float, metaclass=_CppTypeMeta, cpp_name="double"): # noqa: N801 - """Type alias for C++ double.""" - - pass - - def collectify_int32(i: int, j: int) -> npt.NDArray[np.int32]: """Create an int32 array from two integers.""" return np.array([i, j], dtype=np.int32) @@ -59,39 +21,39 @@ def sum_array_int32(coll: npt.NDArray[np.int32]) -> int: def collectify_uint32( - u1: unsigned_int, - u2: unsigned_int, + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint32]: """Create a uint32 array from two integers.""" return np.array([u1, u2], dtype=np.uint32) -def sum_array_uint32(coll: npt.NDArray[np.uint32]) -> unsigned_int: +def sum_array_uint32(coll: npt.NDArray[np.uint32]) -> np.uint32: """Sum a uint32 array.""" - return unsigned_int(sum(int(x) for x in coll)) + return np.uint32(sum(int(x) for x in coll)) -def collectify_int64(l1: long, l2: long) -> npt.NDArray[np.int64]: +def collectify_int64(l1: "long", l2: "long") -> npt.NDArray[np.int64]: # type: ignore # noqa: F821 """Create an int64 array from two integers.""" return np.array([l1, l2], dtype=np.int64) -def sum_array_int64(coll: npt.NDArray[np.int64]) -> long: +def sum_array_int64(coll: npt.NDArray[np.int64]) -> "int64_t": # type: ignore # noqa: F821 """Sum an int64 array.""" - return long(sum(int(x) for x in coll)) + return np.int64(sum(int(x) for x in coll)) def collectify_uint64( - ul1: unsigned_long, - ul2: unsigned_long, + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint64]: """Create a uint64 array from two integers.""" return np.array([ul1, ul2], dtype=np.uint64) -def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> unsigned_long: +def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> "uint64_t": # type: ignore # noqa: F821 """Sum a uint64 array.""" - return unsigned_long(sum(int(x) for x in coll)) + return np.uint64(sum(int(x) for x in coll)) def collectify_float32(f1: float, f2: float) -> npt.NDArray[np.float32]: @@ -99,12 +61,12 @@ def collectify_float32(f1: float, f2: float) -> npt.NDArray[np.float32]: return np.array([f1, f2], dtype=np.float32) -def sum_array_float32(coll: npt.NDArray[np.float32]) -> float: +def sum_array_float32(coll: npt.NDArray[np.float32]) -> np.float32: """Sum a float32 array.""" - return float(sum(coll)) + return np.float32(sum(coll)) -def collectify_float64(d1: double, d2: double) -> npt.NDArray[np.float64]: +def collectify_float64(d1: "double", d2: "double") -> npt.NDArray[np.float64]: # type: ignore # noqa: F821 """Create a float64 array from two floats.""" return np.array([d1, d2], dtype=np.float64) @@ -114,14 +76,14 @@ def collectify_float32_list(f1: float, f2: float) -> list[float]: return [f1, f2] -def collectify_float64_list(d1: double, d2: double) -> list["double"]: +def collectify_float64_list(d1: "double", d2: "double") -> list["double"]: # type: ignore # noqa: F821 """Create a float64 list from two floats.""" return [d1, d2] -def sum_array_float64(coll: npt.NDArray[np.float64]) -> double: +def sum_array_float64(coll: npt.NDArray[np.float64]) -> "double": # type: ignore # noqa: F821 """Sum a float64 array.""" - return double(sum(coll)) + return float(sum(coll)) def collectify_int32_list(i: int, j: int) -> list[int]: @@ -130,24 +92,24 @@ def collectify_int32_list(i: int, j: int) -> list[int]: def collectify_uint32_list( - u1: unsigned_int, - u2: unsigned_int, -) -> "list[unsigned int]": # type: ignore # noqa: F722 + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 +) -> list["unsigned int"]: # type: ignore # noqa: F722 # noqa: F821 """Create a uint32 list from two integers.""" - return [unsigned_int(u1), unsigned_int(u2)] + return [np.uint32(u1), np.uint32(u2)] -def collectify_int64_list(l1: long, l2: long) -> "list[long]": # type: ignore # noqa: F722 +def collectify_int64_list(l1: "long", l2: "long") -> "list[int64_t]": # type: ignore # noqa: F821 """Create an int64 list from two integers.""" - return [long(l1), long(l2)] + return [l1, l2] def collectify_uint64_list( - ul1: unsigned_long, - ul2: unsigned_long, -) -> "list[unsigned long]": # type: ignore # noqa: F722 + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 +) -> "list[uint64_t]": # type: ignore # noqa: F821 """Create a uint64 list from two integers.""" - return [unsigned_long(ul1), unsigned_long(ul2)] + return [np.uint64(ul1), np.uint64(ul2)] def sum_list_int32(coll: list[int]) -> int: @@ -155,29 +117,29 @@ def sum_list_int32(coll: list[int]) -> int: return sum(coll) -def sum_list_uint32(coll: "list[unsigned int]") -> unsigned_int: # type: ignore # noqa: F722 +def sum_list_uint32(coll: "list[uint32_t]") -> "uint32_t": # type: ignore # noqa: F821 """Sum a list of uints.""" - return unsigned_int(sum(coll)) + return sum(coll) -def sum_list_int64(coll: "list[long]") -> long: # type: ignore # noqa: F722 +def sum_list_int64(coll: "list[int64_t]") -> "int64_t": # type: ignore # noqa: F821 """Sum a list of longs.""" - return long(sum(coll)) + return sum(coll) -def sum_list_uint64(coll: "list[unsigned long]") -> unsigned_long: # type: ignore # noqa: F722 +def sum_list_uint64(coll: "list[uint64_t]") -> "uint64_t": # type: ignore # noqa: F821 """Sum a list of ulongs.""" - return unsigned_long(sum(coll)) + return sum(coll) def sum_list_float(coll: list[float]) -> float: """Sum a list of floats.""" - return sum(coll) + return np.float32(sum(coll)) -def sum_list_double(coll: list["double"]) -> double: +def sum_list_double(coll: list["double"]) -> "double": # type: ignore # noqa: F821 """Sum a list of doubles.""" - return double(sum(coll)) + return sum(coll) def PHLEX_REGISTER_ALGORITHMS(m, config): @@ -270,3 +232,4 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): sum_kwargs["name"] = sum_name m.transform(list_sum if use_lists else arr_sum, **sum_kwargs) + From 487c8a0a5daf8c5c1c58f8abe6afa54873beba9b Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 11:48:37 -0800 Subject: [PATCH 02/12] clang-format fixes --- plugins/python/src/modulewrap.cpp | 43 ++++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index 8f4355028..d809ef26a 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -345,19 +345,20 @@ namespace { static long pylong_as_strictlong(PyObject* pyobject) { // convert to C++ long, don't allow truncation - if (PyLong_Check(pyobject)) { // native Python integer - return PyLong_AsLong(pyobject); + if (PyLong_Check(pyobject)) { // native Python integer + return PyLong_AsLong(pyobject); } // accept numpy signed integer scalars (int8, int16, int32, int64) if (PyArray_IsScalar(pyobject, SignedInteger)) { - // convert to Python int first, then to C long, that way we get a Python - // OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); - if (!pylong) return (long)-1; - long result = PyLong_AsLong(pylong); - Py_DECREF(pylong); - return result; + // convert to Python int first, then to C long, that way we get a Python + // OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); + if (!pylong) + return (long)-1; + long result = PyLong_AsLong(pylong); + Py_DECREF(pylong); + return result; } PyErr_SetString(PyExc_TypeError, "int/long conversion expects a signed integer object"); @@ -374,13 +375,14 @@ namespace { // accept numpy unsigned integer scalars (uint8, uint16, uint32, uint64) if (PyArray_IsScalar(pyobject, UnsignedInteger)) { - // convert to Python int first, then to C unsigned long, that way we get a - // Python OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); - if (!pylong) return (long)-1; - unsigned long result = PyLong_AsUnsignedLong(pylong); - Py_DECREF(pylong); - return result; + // convert to Python int first, then to C unsigned long, that way we get a + // Python OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); + if (!pylong) + return (long)-1; + unsigned long result = PyLong_AsUnsignedLong(pylong); + Py_DECREF(pylong); + return result; } unsigned long ul = PyLong_AsUnsignedLong(pyobject); @@ -778,8 +780,8 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw std::string suff0 = "py_" + std::string{static_cast(*pq0.suffix)}; switch (input_queries.size()) { - case 1: { - auto* pyc = new py_callback_1{callable}; // TODO: leaks, but has program lifetime + case 1: { + auto* pyc = new py_callback_1{callable}; // TODO: leaks, but has program lifetime mod->ph_module->transform(pyname, *pyc, concurrency::serial) .input_family( product_query{.creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}) @@ -848,7 +850,7 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; if (dtype == "[int32_t]") { insert_converter(mod, cname, py_to_vint, out_pq, output); - } else if (dtype == "[uint32_t]") { + } else if (dtype == "[uint32_t]") { insert_converter(mod, cname, py_to_vuint, out_pq, output); } else if (dtype == "[int64_t]") { insert_converter(mod, cname, py_to_vlong, out_pq, output); @@ -859,8 +861,7 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw } else if (dtype == "[double]") { insert_converter(mod, cname, py_to_vdouble, out_pq, output); } else { - PyErr_Format( - PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); + PyErr_Format(PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); return nullptr; } } else { From d883994512cfcd7f6135e62eb7d4db4ef39bbb52 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 11:51:56 -0800 Subject: [PATCH 03/12] cmake-format fixes and fix of error string that needs to be found --- test/python/CMakeLists.txt | 277 ++++++++++++++++++++++--------------- 1 file changed, 169 insertions(+), 108 deletions(-) diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 3b0c7e369..03bc22060 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -1,12 +1,15 @@ -find_package(Python 3.12 COMPONENTS Interpreter REQUIRED) +find_package( + Python 3.12 + COMPONENTS Interpreter + REQUIRED + ) # Verify installation of necessary python modules for specific tests function(check_python_module_version MODULE_NAME MIN_VERSION OUT_VAR) execute_process( COMMAND - ${Python_EXECUTABLE} -c - "import sys + ${Python_EXECUTABLE} -c "import sys try: import ${MODULE_NAME} installed_version = getattr(${MODULE_NAME}, '__version__', None) @@ -29,25 +32,39 @@ except Exception as e: RESULT_VARIABLE _module_check_result OUTPUT_VARIABLE _module_check_out ERROR_VARIABLE _module_check_err - ) + ) message( STATUS - "Check ${MODULE_NAME}: Res=${_module_check_result} Out=${_module_check_out} Err=${_module_check_err}" - ) + "Check ${MODULE_NAME}: Res=${_module_check_result} Out=${_module_check_out} Err=${_module_check_err}" + ) if(_module_check_result EQUAL 0) - set(${OUT_VAR} TRUE PARENT_SCOPE) + set(${OUT_VAR} + TRUE + PARENT_SCOPE + ) elseif(_module_check_result EQUAL 1) - set(${OUT_VAR} FALSE PARENT_SCOPE) # silent b/c common + set(${OUT_VAR} + FALSE + PARENT_SCOPE + ) # silent b/c common elseif(_module_check_result EQUAL 2) message( WARNING - "Python module '${MODULE_NAME}' found but version too low (min required: ${MIN_VERSION})." - ) - set(${OUT_VAR} FALSE PARENT_SCOPE) + "Python module '${MODULE_NAME}' found but version too low (min required: ${MIN_VERSION})." + ) + set(${OUT_VAR} + FALSE + PARENT_SCOPE + ) else() - message(WARNING "Unknown error while checking Python module '${MODULE_NAME}'.") - set(${OUT_VAR} FALSE PARENT_SCOPE) + message( + WARNING "Unknown error while checking Python module '${MODULE_NAME}'." + ) + set(${OUT_VAR} + FALSE + PARENT_SCOPE + ) endif() endfunction() @@ -56,32 +73,30 @@ check_python_module_version("numba" "0.61.0" HAS_NUMBA) check_python_module_version("numpy" "2.0.0" HAS_NUMPY) check_python_module_version("pytest_cov" "4.0.0" HAS_PYTEST_COV) -# Explicitly define Python executable variable to ensure it is visible in -# the test environment +# Explicitly define Python executable variable to ensure it is visible in the +# test environment set(PYTHON_TEST_EXECUTABLE ${Python_EXECUTABLE}) set(PYTHON_TEST_FILES test_typing.py test_variant.py) # Determine pytest command based on coverage support if(HAS_PYTEST_COV AND ENABLE_COVERAGE) - set( - PYTEST_COMMAND - ${PYTHON_TEST_EXECUTABLE} - -m - pytest - --cov=${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing - --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml - --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc - ${PYTHON_TEST_FILES} - ) + set(PYTEST_COMMAND + ${PYTHON_TEST_EXECUTABLE} -m pytest --cov=${CMAKE_CURRENT_SOURCE_DIR} + --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing + --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} + ) message(STATUS "Python tests will run with coverage reporting (pytest-cov)") else() set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) - message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") + message( + WARNING + "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled" + ) endif() endif() @@ -93,7 +108,10 @@ if(HAS_CPPYY) set(PYTHON_TEST_PHLEX_INSTALL ${CMAKE_SOURCE_DIR}) endif() - set_property(TEST py:phlex PROPERTY ENVIRONMENT "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}") + set_property( + TEST py:phlex PROPERTY ENVIRONMENT + "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" + ) set(PYTHON_TEST_FILES ${PYTHON_TEST_FILES} test_phlex.py) endif() @@ -103,68 +121,89 @@ set(ACTIVE_PY_CPHLEX_TESTS "") # numpy support if installed if(HAS_NUMPY) # phlex-based tests that require numpy support - add_test(NAME py:vec COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyvec.jsonnet) + add_test(NAME py:vec COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyvec.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:vec) - add_test(NAME py:vectypes COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyvectypes.jsonnet) + add_test(NAME py:vectypes + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyvectypes.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:vectypes) - add_test( - NAME py:callback3 - COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycallback3.jsonnet - ) + add_test(NAME py:callback3 + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pycallback3.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:callback3) - # Expect failure for these tests (check for error propagation and type checking) - add_test(NAME py:raise COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyraise.jsonnet) + # Expect failure for these tests (check for error propagation and type + # checking) + add_test(NAME py:raise COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyraise.jsonnet + ) set_tests_properties( - py:raise - PROPERTIES PASS_REGULAR_EXPRESSION "RuntimeError: Intentional failure" - ) + py:raise PROPERTIES PASS_REGULAR_EXPRESSION + "RuntimeError: Intentional failure" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:raise) - add_test(NAME py:badbool COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybadbool.jsonnet) + add_test(NAME py:badbool + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pybadbool.jsonnet + ) set_tests_properties( py:badbool PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type bool: boolean value should be bool, or integer 1 or 0" - ) + "Python conversion error for type bool: boolean value should be bool, or integer 1 or 0" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:badbool) - add_test(NAME py:badint COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybadint.jsonnet) + add_test(NAME py:badint COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pybadint.jsonnet + ) set_tests_properties( py:badint PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type long: int/long conversion expects an integer object" - ) + "Python conversion error for type long: int/long conversion expects a signed integer object" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:badint) - add_test(NAME py:baduint COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybaduint.jsonnet) + add_test(NAME py:baduint + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pybaduint.jsonnet + ) set_tests_properties( py:baduint PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type uint: can't convert negative value to unsigned long" - ) + "Python conversion error for type uint: can't convert negative value to unsigned long" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:baduint) - add_test( - NAME py:mismatch_variant - COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch_variant.jsonnet - ) + add_test(NAME py:mismatch_variant + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch_variant.jsonnet + ) set_tests_properties( py:mismatch_variant - PROPERTIES - PASS_REGULAR_EXPRESSION "number of inputs .* does not match number of annotation types" - ) + PROPERTIES PASS_REGULAR_EXPRESSION + "number of inputs .* does not match number of annotation types" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:mismatch_variant) - add_test(NAME py:veclists COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyveclists.jsonnet) + add_test(NAME py:veclists + COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyveclists.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:veclists) - add_test(NAME py:types COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pytypes.jsonnet) + add_test(NAME py:types COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pytypes.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:types) endif() @@ -173,47 +212,57 @@ add_library(cppsource4py MODULE source.cpp) target_link_libraries(cppsource4py PRIVATE phlex::module) # phlex-based tests (no cppyy dependency) -add_test(NAME py:add COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyadd.jsonnet) +add_test(NAME py:add COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyadd.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:add) -add_test(NAME py:config COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyconfig.jsonnet) +add_test(NAME py:config COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyconfig.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:config) -add_test(NAME py:reduce COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyreduce.jsonnet) +add_test(NAME py:reduce COMMAND phlex::phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyreduce.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:reduce) -add_test(NAME py:coverage COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycoverage.jsonnet) +add_test(NAME py:coverage + COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycoverage.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:coverage) -add_test( - NAME py:mismatch - COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch.jsonnet -) +add_test(NAME py:mismatch + COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch.jsonnet + ) set_tests_properties( py:mismatch - PROPERTIES - PASS_REGULAR_EXPRESSION "number of inputs .* does not match number of annotation types" -) + PROPERTIES PASS_REGULAR_EXPRESSION + "number of inputs .* does not match number of annotation types" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:mismatch) # "failing" tests for checking error paths -add_test( - NAME py:failure - COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyfailure.jsonnet -) +add_test(NAME py:failure COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c + ${CMAKE_CURRENT_SOURCE_DIR}/pyfailure.jsonnet + ) set_tests_properties( - py:failure - PROPERTIES PASS_REGULAR_EXPRESSION "failed to retrieve property \"input\"" -) + py:failure PROPERTIES PASS_REGULAR_EXPRESSION + "failed to retrieve property \"input\"" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:failure) set(TEST_PYTHONPATH ${CMAKE_CURRENT_SOURCE_DIR}) -# Add the python plugin source directory to PYTHONPATH so tests can use phlex package -set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:${PROJECT_SOURCE_DIR}/plugins/python/python) +# Add the python plugin source directory to PYTHONPATH so tests can use phlex +# package +set(TEST_PYTHONPATH + ${TEST_PYTHONPATH}:${PROJECT_SOURCE_DIR}/plugins/python/python + ) -# Always add site-packages to PYTHONPATH for tests, as embedded python might -# not find them especially in spack environments where they are in -# non-standard locations +# Always add site-packages to PYTHONPATH for tests, as embedded python might not +# find them especially in spack environments where they are in non-standard +# locations if(Python_SITELIB) set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:${Python_SITELIB}) endif() @@ -227,27 +276,32 @@ endif() set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:$ENV{PYTHONPATH}) # Environment variables required: -set( - PYTHON_TEST_ENVIRONMENT - "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" -) +set(PYTHON_TEST_ENVIRONMENT + "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" + ) -set_tests_properties(${ACTIVE_PY_CPHLEX_TESTS} PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") +set_tests_properties( + ${ACTIVE_PY_CPHLEX_TESTS} PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}" + ) # phlex-based test to ensure that sys.path is sanely set for virtual environment set(PY_VIRTUAL_ENV_DIR ${CMAKE_CURRENT_BINARY_DIR}/py_virtual_env) execute_process(COMMAND python -m venv ${PY_VIRTUAL_ENV_DIR}) configure_file(pysyspath.jsonnet.in pysyspath.jsonnet @ONLY) -add_test(NAME py:syspath COMMAND phlex::phlex -c ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet) +add_test(NAME py:syspath COMMAND phlex::phlex -c + ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet + ) -# Activate the Python virtual environment "by hand". Requires setting the VIRTUAL_ENV -# environment variable and prepending to the PATH environment variable. +# Activate the Python virtual environment "by hand". Requires setting the +# VIRTUAL_ENV environment variable and prepending to the PATH environment +# variable. set_tests_properties( py:syspath - PROPERTIES - ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT};VIRTUAL_ENV=${PY_VIRTUAL_ENV_DIR}" - ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" -) + PROPERTIES ENVIRONMENT + "${PYTHON_TEST_ENVIRONMENT};VIRTUAL_ENV=${PY_VIRTUAL_ENV_DIR}" + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" + ) # Python coverage target if(HAS_PYTEST_COV AND ENABLE_COVERAGE) @@ -255,32 +309,39 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) # When cppyy is available, use the full pytest command add_custom_target( coverage-python - COMMAND - ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} - PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL} ${PYTEST_COMMAND} + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} + PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL} ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report" - ) + ) else() - # When cppyy is not available, run a simpler pytest command - # for standalone Python tests (like test_variant.py) + # When cppyy is not available, run a simpler pytest command for standalone + # Python tests (like test_variant.py) set(PYTHON_TEST_PHLEX_INSTALL_FALLBACK ${CMAKE_SOURCE_DIR}) add_custom_target( coverage-python COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} - PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL_FALLBACK} ${Python_EXECUTABLE} -m pytest - --cov=${CMAKE_CURRENT_SOURCE_DIR} --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL_FALLBACK} ${Python_EXECUTABLE} + -m pytest --cov=${CMAKE_CURRENT_SOURCE_DIR} + --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing + --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc + ${PYTHON_TEST_FILES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report (standalone tests only)" - ) + ) endif() endif() # tests of the python support modules -add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) -set_tests_properties(py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") - +add_test( + NAME py:phlex + COMMAND ${PYTEST_COMMAND} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) +set_tests_properties( + py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}" + ) From 27388d5f1526a82cdd6c45e4dfb9337dda5e27c5 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 20:48:44 -0800 Subject: [PATCH 04/12] more tests of type annotation normalization --- test/python/test_typing.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/python/test_typing.py b/test/python/test_typing.py index 62fc25402..4a4de9540 100644 --- a/test/python/test_typing.py +++ b/test/python/test_typing.py @@ -2,11 +2,12 @@ """Unit tests for phlex typing normalization.""" import ctypes -from typing import List +from typing import Dict, List import numpy as np import numpy.typing as npt from phlex._typing import _C2C +from pytest import raises from phlex import normalize_type @@ -63,3 +64,32 @@ def test_numpy_array_normalization(self): (np.float64, "double"),): assert normalize_type(npt.NDArray[t]) == "ndarray["+s+"]" + def test_special_cases(self): + """Special cases.""" + # None type + assert normalize_type(None) == "None" + + # use of namespaces for evaluation + assert("foo") == "foo" + global foo + foo = np.int64 + assert normalize_type("foo", globals()) == "int64_t" + bar = np.int32 + assert normalize_type("bar", globals()) == "bar" + assert normalize_type("bar", localns=locals()) == "int32_t" + + # union types are not allowed + raises(TypeError, normalize_type, List[int] | Dict[int, int]) + + # pointer types are not supported yet + raises(TypeError, normalize_type, ctypes.POINTER(ctypes.c_int)) + + # ctypes array + assert normalize_type((ctypes.c_int * 42)) == "array[int32_t]" + + # something unknown + class SomeType: + def __repr__(self): + return "some type" + + assert normalize_type(SomeType()) == "some type" From a92c5604458fac30f809ea9a3350407a3511a0a4 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 20:56:15 -0800 Subject: [PATCH 05/12] remove error unreachable error checking code --- plugins/python/src/modulewrap.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index d809ef26a..f0132f933 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -353,9 +353,7 @@ namespace { if (PyArray_IsScalar(pyobject, SignedInteger)) { // convert to Python int first, then to C long, that way we get a Python // OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); - if (!pylong) - return (long)-1; + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check long result = PyLong_AsLong(pylong); Py_DECREF(pylong); return result; @@ -377,9 +375,7 @@ namespace { if (PyArray_IsScalar(pyobject, UnsignedInteger)) { // convert to Python int first, then to C unsigned long, that way we get a // Python OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); - if (!pylong) - return (long)-1; + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check unsigned long result = PyLong_AsUnsignedLong(pylong); Py_DECREF(pylong); return result; From e1b21e595c041f05f59791cb762a3e36fe4b1272 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 04:57:53 +0000 Subject: [PATCH 06/12] Apply ruff fixes --- plugins/python/python/phlex/_typing.py | 97 ++++++++++++++------------ test/python/all_config.py | 10 +-- test/python/reducer.py | 8 +-- test/python/sumit.py | 10 +-- test/python/test_coverage.py | 15 ++-- test/python/test_mismatch.py | 8 ++- test/python/test_typing.py | 49 +++++++------ test/python/test_variant.py | 3 +- test/python/vectypes.py | 44 ++++++------ 9 files changed, 126 insertions(+), 118 deletions(-) diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py index de3bbb293..249de6d87 100644 --- a/plugins/python/python/phlex/_typing.py +++ b/plugins/python/python/phlex/_typing.py @@ -22,45 +22,49 @@ # TODO: should users be allowed to add to these? _PY2CPP: dict[type, str] = { # numpy types - np.bool_: "bool", - np.int8: "int8_t", - np.int16: "int16_t", - np.int32: "int32_t", - np.int64: "int64_t", - np.uint8: "uint8_t", - np.uint16: "uint16_t", - np.uint32: "uint32_t", - np.uint64: "uint64_t", - np.float32: "float", - np.float64: "double", - np.complex64: "std::complex", - np.complex128: "std::complex", + np.bool_: "bool", + np.int8: "int8_t", + np.int16: "int16_t", + np.int32: "int32_t", + np.int64: "int64_t", + np.uint8: "uint8_t", + np.uint16: "uint16_t", + np.uint32: "uint32_t", + np.uint64: "uint64_t", + np.float32: "float", + np.float64: "double", + np.complex64: "std::complex", + np.complex128: "std::complex", # the following types are aliased in numpy, so ignore here - #np.intp: "ptrdiff_t", - #np.uintp: "size_t", + # np.intp: "ptrdiff_t", + # np.uintp: "size_t", } # ctypes types that don't map cleanly to intN_t / uintN_t _CTYPES_SPECIAL: dict[type, str] = { - ctypes.c_bool: "bool", - ctypes.c_char: "char", # signedness is implementation-defined in C - ctypes.c_wchar: "wchar_t", - ctypes.c_float: "float", # always IEEE 754 32-bit - ctypes.c_double: "double", # always IEEE 754 64-bit - ctypes.c_longdouble:"long double",# platform-dependent width, no stdint.h alias + ctypes.c_bool: "bool", + ctypes.c_char: "char", # signedness is implementation-defined in C + ctypes.c_wchar: "wchar_t", + ctypes.c_float: "float", # always IEEE 754 32-bit + ctypes.c_double: "double", # always IEEE 754 64-bit + ctypes.c_longdouble: "long double", # platform-dependent width, no stdint.h alias # the following types are aliased in ctypes, so ignore here - #ctypes.c_size_t: "size_t", - #ctypes.c_ssize_t: "ssize_t", - ctypes.c_void_p: "void*", - ctypes.c_char_p: "const char*", - ctypes.c_wchar_p: "const wchar_t*", + # ctypes.c_size_t: "size_t", + # ctypes.c_ssize_t: "ssize_t", + ctypes.c_void_p: "void*", + ctypes.c_char_p: "const char*", + ctypes.c_wchar_p: "const wchar_t*", } _CTYPES_INTEGER: list[type] = [ - ctypes.c_byte, ctypes.c_ubyte, - ctypes.c_short, ctypes.c_ushort, - ctypes.c_int, ctypes.c_uint, - ctypes.c_long, ctypes.c_ulong, + ctypes.c_byte, + ctypes.c_ubyte, + ctypes.c_short, + ctypes.c_ushort, + ctypes.c_int, + ctypes.c_uint, + ctypes.c_long, + ctypes.c_ulong, ] # (unsigned) long long may be aliased @@ -68,6 +72,7 @@ _CTYPES_INTEGER.append(ctypes.c_longlong) _CTYPES_INTEGER.append(ctypes.c_ulonglong) + def _build_ctypes_map() -> dict[type, str]: result = dict(_CTYPES_SPECIAL) for tp in _CTYPES_INTEGER: @@ -77,22 +82,23 @@ def _build_ctypes_map() -> dict[type, str]: result[tp] = f"{'int' if signed else 'uint'}{bits}_t" return result + _PY2CPP.update(_build_ctypes_map()) # use ctypes to construct a mapping from platform types to exact types _C2C: dict[str, str] = { - "short": _PY2CPP[ctypes.c_short], - "unsigned short": _PY2CPP[ctypes.c_ushort], - "int": _PY2CPP[ctypes.c_int], - "unsigned int": _PY2CPP[ctypes.c_uint], - "long": _PY2CPP[ctypes.c_long], - "unsigned long": _PY2CPP[ctypes.c_ulong], - "long long": 'int64_t', - "unsigned long long": 'uint64_t', + "short": _PY2CPP[ctypes.c_short], + "unsigned short": _PY2CPP[ctypes.c_ushort], + "int": _PY2CPP[ctypes.c_int], + "unsigned int": _PY2CPP[ctypes.c_uint], + "long": _PY2CPP[ctypes.c_long], + "unsigned long": _PY2CPP[ctypes.c_ulong], + "long long": "int64_t", + "unsigned long long": "uint64_t", } -def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) -> str: +def normalize_type(tp: Any, globalns: Dict | None = None, localns: Dict | None = None) -> str: """Recursively normalize any Python annotation to a canonical name. This normalization supports: @@ -120,7 +126,7 @@ def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) - try: tp = eval(tp, ns) except Exception: - return _C2C.get(tp, tp) # unresolvable + return _C2C.get(tp, tp) # unresolvable # get the unsubscripted version of the type, or None if it's something unknown origin = typing.get_origin(tp) @@ -147,14 +153,14 @@ def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) - if origin is not None: args = typing.get_args(tp) - if origin is np.ndarray: # numpy arrays + if origin is np.ndarray: # numpy arrays dtype_args = typing.get_args(args[1]) if len(args) >= 2 else () if dtype_args: return "ndarray[" + normalize_type(dtype_args[0], globalns, localns) + "]" else: raise TypeError("Numpy array with unparameterized or no scalar type") - if isinstance(origin, type): # regular python typing type + if isinstance(origin, type): # regular python typing type name = origin.__name__ else: # fallback for unexpected origins @@ -166,15 +172,15 @@ def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) - try: if issubclass(tp, ctypes.Array): # TODO: tp._length_ may be useful as well - return "array["+normalize_type(tp._type_, globalns, localns)+"]" + return "array[" + normalize_type(tp._type_, globalns, localns) + "]" except TypeError: - pass # tp is not a class + pass # tp is not a class # known builtin types representations from ctypes and numpy try: return _PY2CPP[tp] except KeyError: - pass # not a known Python type + pass # not a known Python type # fallback for plain Python types if isinstance(tp, type): @@ -182,4 +188,3 @@ def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) - # fallback for everything else, expecting repr() to be unique and consistent return repr(tp) - diff --git a/test/python/all_config.py b/test/python/all_config.py index aeee3b373..0a4c16310 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -49,9 +49,9 @@ def __init__(self, config): assert len(config["some_objects"]) == 3 expected = [ - {'a': 'b', 'c': 'd', 'e': 'f'}, - {'g': 'h', 'i': 'j', 'k': 'l'}, - {'m': 'n', 'o': 'p', 'q': 'r'}, + {"a": "b", "c": "d", "e": "f"}, + {"g": "h", "i": "j", "k": "l"}, + {"m": "n", "o": "p", "q": "r"}, ] for i in range(3): assert config["some_objects"][i] == expected[i] @@ -60,10 +60,10 @@ def __init__(self, config): assert config["empty"] == () try: - config[42] # should raise + config[42] # should raise assert not "did not raise TypeError" except TypeError: - pass # all good as exception was raised + pass # all good as exception was raised def __call__(self, i: int, j: int) -> None: """Dummy routine to do something. diff --git a/test/python/reducer.py b/test/python/reducer.py index b32fe0395..75b283e6d 100644 --- a/test/python/reducer.py +++ b/test/python/reducer.py @@ -74,17 +74,13 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): {"creator": "reduce0", "layer": "event", "suffix": "sum0"}, {"creator": "reduce1", "layer": "event", "suffix": "sum1"}, ] - m.transform( - add_sum01, name="reduce01", input_family=input_family01, output_products=["sum01"] - ) + m.transform(add_sum01, name="reduce01", input_family=input_family01, output_products=["sum01"]) input_family01 = [ {"creator": "reduce2", "layer": "event", "suffix": "sum2"}, {"creator": "reduce3", "layer": "event", "suffix": "sum3"}, ] - m.transform( - add_sum23, name="reduce23", input_family=input_family01, output_products=["sum23"] - ) + m.transform(add_sum23, name="reduce23", input_family=input_family01, output_products=["sum23"]) # once more (and the configuration will add a verifier) input_family_final = [ diff --git a/test/python/sumit.py b/test/python/sumit.py index 17eaef93c..ae43e5ffc 100644 --- a/test/python/sumit.py +++ b/test/python/sumit.py @@ -63,8 +63,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): None """ m.transform(collectify, input_family=config["input"], output_products=["my_pyarray"]) - m.transform(sum_array, - input_family=[ - {"creator" : "collectify", "layer" : "event", "suffix" : "my_pyarray"} - ], - output_products=config["output"]) + m.transform( + sum_array, + input_family=[{"creator": "collectify", "layer": "event", "suffix": "my_pyarray"}], + output_products=config["output"], + ) diff --git a/test/python/test_coverage.py b/test/python/test_coverage.py index 33f82d33b..52c844d89 100644 --- a/test/python/test_coverage.py +++ b/test/python/test_coverage.py @@ -42,13 +42,14 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # We need to transform scalar inputs to lists first # i, f1, d1 come from cppsource4py - tfs = ((collect_int, "input", "i", "l_int"), - (collect_float, "input", "f1", "l_float"), - (collect_double, "input", "d1", "l_double"), - (list_int_func, collect_int.__name__, "l_int", "sum_int"), - (list_float_func, collect_float.__name__, "l_float", "sum_float"), - (list_double_func, collect_double.__name__, "l_double", "sum_double") - ) + tfs = ( + (collect_int, "input", "i", "l_int"), + (collect_float, "input", "f1", "l_float"), + (collect_double, "input", "d1", "l_double"), + (list_int_func, collect_int.__name__, "l_int", "sum_int"), + (list_float_func, collect_float.__name__, "l_float", "sum_float"), + (list_double_func, collect_double.__name__, "l_double", "sum_double"), + ) for func, creator, suffix, output in tfs: input_family = [{"creator": creator, "layer": "event", "suffix": suffix}] diff --git a/test/python/test_mismatch.py b/test/python/test_mismatch.py index 2d188496c..55eca0e7c 100644 --- a/test/python/test_mismatch.py +++ b/test/python/test_mismatch.py @@ -10,6 +10,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # input_family has 1 element, but function takes 2 arguments # This should trigger the error in modulewrap.cpp - m.transform(mismatch_func, - input_family=[{"creator": "input", "layer": "event", "suffix": "a"}], - output_products=["sum"]) + m.transform( + mismatch_func, + input_family=[{"creator": "input", "layer": "event", "suffix": "a"}], + output_products=["sum"], + ) diff --git a/test/python/test_typing.py b/test/python/test_typing.py index 62fc25402..5d3a3e6c0 100644 --- a/test/python/test_typing.py +++ b/test/python/test_typing.py @@ -16,19 +16,21 @@ class TestTYPING: def test_list_normalization(self): """Normalization of various forms of list annotations.""" - for types in (["int", int, ctypes.c_int], - ["unsigned int", ctypes.c_uint], - ["long", ctypes.c_long], - ["unsigned long", ctypes.c_ulong], - ["long long", ctypes.c_longlong], - ["unsigned long long", ctypes.c_ulonglong], - ["float", float, ctypes.c_float], - ["double", ctypes.c_double],): + for types in ( + ["int", int, ctypes.c_int], + ["unsigned int", ctypes.c_uint], + ["long", ctypes.c_long], + ["unsigned long", ctypes.c_ulong], + ["long long", ctypes.c_longlong], + ["unsigned long long", ctypes.c_ulonglong], + ["float", float, ctypes.c_float], + ["double", ctypes.c_double], + ): # TODO: the use of _C2C here is a bit circular tn = _C2C.get(types[0], types[0]) - if 0 < tn.find('_'): - npt = tn[:tn.find('_')] + if 0 < tn.find("_"): + npt = tn[: tn.find("_")] elif tn == "float": npt = "float32" elif tn == "double": @@ -50,16 +52,17 @@ def test_list_normalization(self): def test_numpy_array_normalization(self): """Normalization of standard Numpy typing.""" - for t, s in ((np.bool_, "bool"), - (np.int8, "int8_t"), - (np.int16, "int16_t"), - (np.int32, "int32_t"), - (np.int64, "int64_t"), - (np.uint8, "uint8_t"), - (np.uint16, "uint16_t"), - (np.uint32, "uint32_t"), - (np.uint64, "uint64_t"), - (np.float32, "float"), - (np.float64, "double"),): - assert normalize_type(npt.NDArray[t]) == "ndarray["+s+"]" - + for t, s in ( + (np.bool_, "bool"), + (np.int8, "int8_t"), + (np.int16, "int16_t"), + (np.int32, "int32_t"), + (np.int64, "int64_t"), + (np.uint8, "uint8_t"), + (np.uint16, "uint16_t"), + (np.uint32, "uint32_t"), + (np.uint64, "uint64_t"), + (np.float32, "float"), + (np.float64, "double"), + ): + assert normalize_type(npt.NDArray[t]) == "ndarray[" + s + "]" diff --git a/test/python/test_variant.py b/test/python/test_variant.py index 27a204259..fe65cecf4 100644 --- a/test/python/test_variant.py +++ b/test/python/test_variant.py @@ -72,7 +72,7 @@ def __call__(self): c = Container() wrapper = Variant(c, {}, "deep_clone", clone="deep") assert id(wrapper.phlex_callable) != id(c) - assert id(wrapper.phlex_callable.data) != id(c.data) + assert id(wrapper.phlex_callable.data) != id(c.data) def test_missing_annotation_raises(self): """Test that MissingAnnotation is raised when a required argument is missing.""" @@ -99,4 +99,3 @@ def func(x, y=1): wrapper = Variant(func, incomplete_ann, "missing_optional_y") assert "x" in wrapper.__annotations__ assert "y" not in wrapper.__annotations__ - diff --git a/test/python/vectypes.py b/test/python/vectypes.py index fea621048..18fa0369f 100644 --- a/test/python/vectypes.py +++ b/test/python/vectypes.py @@ -10,6 +10,7 @@ # Type aliases for C++ types that don't have Python equivalents # These are used by the C++ wrapper to identify the correct converter + def collectify_int32(i: int, j: int) -> npt.NDArray[np.int32]: """Create an int32 array from two integers.""" return np.array([i, j], dtype=np.int32) @@ -21,8 +22,8 @@ def sum_array_int32(coll: npt.NDArray[np.int32]) -> int: def collectify_uint32( - u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 - u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint32]: """Create a uint32 array from two integers.""" return np.array([u1, u2], dtype=np.uint32) @@ -33,25 +34,25 @@ def sum_array_uint32(coll: npt.NDArray[np.uint32]) -> np.uint32: return np.uint32(sum(int(x) for x in coll)) -def collectify_int64(l1: "long", l2: "long") -> npt.NDArray[np.int64]: # type: ignore # noqa: F821 +def collectify_int64(l1: "long", l2: "long") -> npt.NDArray[np.int64]: # type: ignore # noqa: F821 """Create an int64 array from two integers.""" return np.array([l1, l2], dtype=np.int64) -def sum_array_int64(coll: npt.NDArray[np.int64]) -> "int64_t": # type: ignore # noqa: F821 +def sum_array_int64(coll: npt.NDArray[np.int64]) -> "int64_t": # type: ignore # noqa: F821 """Sum an int64 array.""" return np.int64(sum(int(x) for x in coll)) def collectify_uint64( - ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 - ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint64]: """Create a uint64 array from two integers.""" return np.array([ul1, ul2], dtype=np.uint64) -def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> "uint64_t": # type: ignore # noqa: F821 +def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> "uint64_t": # type: ignore # noqa: F821 """Sum a uint64 array.""" return np.uint64(sum(int(x) for x in coll)) @@ -66,7 +67,7 @@ def sum_array_float32(coll: npt.NDArray[np.float32]) -> np.float32: return np.float32(sum(coll)) -def collectify_float64(d1: "double", d2: "double") -> npt.NDArray[np.float64]: # type: ignore # noqa: F821 +def collectify_float64(d1: "double", d2: "double") -> npt.NDArray[np.float64]: # type: ignore # noqa: F821 """Create a float64 array from two floats.""" return np.array([d1, d2], dtype=np.float64) @@ -76,12 +77,12 @@ def collectify_float32_list(f1: float, f2: float) -> list[float]: return [f1, f2] -def collectify_float64_list(d1: "double", d2: "double") -> list["double"]: # type: ignore # noqa: F821 +def collectify_float64_list(d1: "double", d2: "double") -> list["double"]: # type: ignore # noqa: F821 """Create a float64 list from two floats.""" return [d1, d2] -def sum_array_float64(coll: npt.NDArray[np.float64]) -> "double": # type: ignore # noqa: F821 +def sum_array_float64(coll: npt.NDArray[np.float64]) -> "double": # type: ignore # noqa: F821 """Sum a float64 array.""" return float(sum(coll)) @@ -92,8 +93,8 @@ def collectify_int32_list(i: int, j: int) -> list[int]: def collectify_uint32_list( - u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 - u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 ) -> list["unsigned int"]: # type: ignore # noqa: F722 # noqa: F821 """Create a uint32 list from two integers.""" return [np.uint32(u1), np.uint32(u2)] @@ -105,8 +106,8 @@ def collectify_int64_list(l1: "long", l2: "long") -> "list[int64_t]": # type: i def collectify_uint64_list( - ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 - ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 ) -> "list[uint64_t]": # type: ignore # noqa: F821 """Create a uint64 list from two integers.""" return [np.uint64(ul1), np.uint64(ul2)] @@ -137,7 +138,7 @@ def sum_list_float(coll: list[float]) -> float: return np.float32(sum(coll)) -def sum_list_double(coll: list["double"]) -> "double": # type: ignore # noqa: F821 +def sum_list_double(coll: list["double"]) -> "double": # type: ignore # noqa: F821 """Sum a list of doubles.""" return sum(coll) @@ -221,15 +222,16 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): ) sum_kwargs = { - "input_family": [{ - "creator": list_collect.__name__ if use_lists else arr_collect.__name__, - "layer": "event", - "suffix": arr_name - }], + "input_family": [ + { + "creator": list_collect.__name__ if use_lists else arr_collect.__name__, + "layer": "event", + "suffix": arr_name, + } + ], "output_products": config[out_key], } if sum_name: sum_kwargs["name"] = sum_name m.transform(list_sum if use_lists else arr_sum, **sum_kwargs) - From 0bc921ec9024979a1e232fa94113d0a48efe602d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 04:57:53 +0000 Subject: [PATCH 07/12] Apply cmake-format fixes --- test/python/CMakeLists.txt | 251 ++++++++++++++----------------------- 1 file changed, 96 insertions(+), 155 deletions(-) diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 03bc22060..9bb6cbded 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -1,15 +1,12 @@ -find_package( - Python 3.12 - COMPONENTS Interpreter - REQUIRED - ) +find_package(Python 3.12 COMPONENTS Interpreter REQUIRED) # Verify installation of necessary python modules for specific tests function(check_python_module_version MODULE_NAME MIN_VERSION OUT_VAR) execute_process( COMMAND - ${Python_EXECUTABLE} -c "import sys + ${Python_EXECUTABLE} -c + "import sys try: import ${MODULE_NAME} installed_version = getattr(${MODULE_NAME}, '__version__', None) @@ -32,39 +29,25 @@ except Exception as e: RESULT_VARIABLE _module_check_result OUTPUT_VARIABLE _module_check_out ERROR_VARIABLE _module_check_err - ) + ) message( STATUS - "Check ${MODULE_NAME}: Res=${_module_check_result} Out=${_module_check_out} Err=${_module_check_err}" - ) + "Check ${MODULE_NAME}: Res=${_module_check_result} Out=${_module_check_out} Err=${_module_check_err}" + ) if(_module_check_result EQUAL 0) - set(${OUT_VAR} - TRUE - PARENT_SCOPE - ) + set(${OUT_VAR} TRUE PARENT_SCOPE) elseif(_module_check_result EQUAL 1) - set(${OUT_VAR} - FALSE - PARENT_SCOPE - ) # silent b/c common + set(${OUT_VAR} FALSE PARENT_SCOPE) # silent b/c common elseif(_module_check_result EQUAL 2) message( WARNING - "Python module '${MODULE_NAME}' found but version too low (min required: ${MIN_VERSION})." - ) - set(${OUT_VAR} - FALSE - PARENT_SCOPE - ) + "Python module '${MODULE_NAME}' found but version too low (min required: ${MIN_VERSION})." + ) + set(${OUT_VAR} FALSE PARENT_SCOPE) else() - message( - WARNING "Unknown error while checking Python module '${MODULE_NAME}'." - ) - set(${OUT_VAR} - FALSE - PARENT_SCOPE - ) + message(WARNING "Unknown error while checking Python module '${MODULE_NAME}'.") + set(${OUT_VAR} FALSE PARENT_SCOPE) endif() endfunction() @@ -81,22 +64,24 @@ set(PYTHON_TEST_FILES test_typing.py test_variant.py) # Determine pytest command based on coverage support if(HAS_PYTEST_COV AND ENABLE_COVERAGE) - set(PYTEST_COMMAND - ${PYTHON_TEST_EXECUTABLE} -m pytest --cov=${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing - --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml - --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} - ) + set( + PYTEST_COMMAND + ${PYTHON_TEST_EXECUTABLE} + -m + pytest + --cov=${CMAKE_CURRENT_SOURCE_DIR} + --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing + --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc + ${PYTHON_TEST_FILES} + ) message(STATUS "Python tests will run with coverage reporting (pytest-cov)") else() set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) - message( - WARNING - "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled" - ) + message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") endif() endif() @@ -108,10 +93,7 @@ if(HAS_CPPYY) set(PYTHON_TEST_PHLEX_INSTALL ${CMAKE_SOURCE_DIR}) endif() - set_property( - TEST py:phlex PROPERTY ENVIRONMENT - "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" - ) + set_property(TEST py:phlex PROPERTY ENVIRONMENT "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}") set(PYTHON_TEST_FILES ${PYTHON_TEST_FILES} test_phlex.py) endif() @@ -121,89 +103,69 @@ set(ACTIVE_PY_CPHLEX_TESTS "") # numpy support if installed if(HAS_NUMPY) # phlex-based tests that require numpy support - add_test(NAME py:vec COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyvec.jsonnet - ) + add_test(NAME py:vec COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyvec.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:vec) - add_test(NAME py:vectypes - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyvectypes.jsonnet - ) + add_test(NAME py:vectypes COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyvectypes.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:vectypes) - add_test(NAME py:callback3 - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pycallback3.jsonnet - ) + add_test( + NAME py:callback3 + COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycallback3.jsonnet + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:callback3) # Expect failure for these tests (check for error propagation and type # checking) - add_test(NAME py:raise COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyraise.jsonnet - ) + add_test(NAME py:raise COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyraise.jsonnet) set_tests_properties( - py:raise PROPERTIES PASS_REGULAR_EXPRESSION - "RuntimeError: Intentional failure" - ) + py:raise + PROPERTIES PASS_REGULAR_EXPRESSION "RuntimeError: Intentional failure" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:raise) - add_test(NAME py:badbool - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pybadbool.jsonnet - ) + add_test(NAME py:badbool COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybadbool.jsonnet) set_tests_properties( py:badbool PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type bool: boolean value should be bool, or integer 1 or 0" - ) + "Python conversion error for type bool: boolean value should be bool, or integer 1 or 0" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:badbool) - add_test(NAME py:badint COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pybadint.jsonnet - ) + add_test(NAME py:badint COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybadint.jsonnet) set_tests_properties( py:badint PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type long: int/long conversion expects a signed integer object" - ) + "Python conversion error for type long: int/long conversion expects a signed integer object" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:badint) - add_test(NAME py:baduint - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pybaduint.jsonnet - ) + add_test(NAME py:baduint COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pybaduint.jsonnet) set_tests_properties( py:baduint PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type uint: can't convert negative value to unsigned long" - ) + "Python conversion error for type uint: can't convert negative value to unsigned long" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:baduint) - add_test(NAME py:mismatch_variant - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch_variant.jsonnet - ) + add_test( + NAME py:mismatch_variant + COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch_variant.jsonnet + ) set_tests_properties( py:mismatch_variant - PROPERTIES PASS_REGULAR_EXPRESSION - "number of inputs .* does not match number of annotation types" - ) + PROPERTIES + PASS_REGULAR_EXPRESSION "number of inputs .* does not match number of annotation types" + ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:mismatch_variant) - add_test(NAME py:veclists - COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyveclists.jsonnet - ) + add_test(NAME py:veclists COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyveclists.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:veclists) - add_test(NAME py:types COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pytypes.jsonnet - ) + add_test(NAME py:types COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pytypes.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:types) endif() @@ -212,53 +174,44 @@ add_library(cppsource4py MODULE source.cpp) target_link_libraries(cppsource4py PRIVATE phlex::module) # phlex-based tests (no cppyy dependency) -add_test(NAME py:add COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyadd.jsonnet - ) +add_test(NAME py:add COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyadd.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:add) -add_test(NAME py:config COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyconfig.jsonnet - ) +add_test(NAME py:config COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyconfig.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:config) -add_test(NAME py:reduce COMMAND phlex::phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyreduce.jsonnet - ) +add_test(NAME py:reduce COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyreduce.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:reduce) -add_test(NAME py:coverage - COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycoverage.jsonnet - ) +add_test(NAME py:coverage COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycoverage.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:coverage) -add_test(NAME py:mismatch - COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch.jsonnet - ) +add_test( + NAME py:mismatch + COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch.jsonnet +) set_tests_properties( py:mismatch - PROPERTIES PASS_REGULAR_EXPRESSION - "number of inputs .* does not match number of annotation types" - ) + PROPERTIES + PASS_REGULAR_EXPRESSION "number of inputs .* does not match number of annotation types" +) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:mismatch) # "failing" tests for checking error paths -add_test(NAME py:failure COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c - ${CMAKE_CURRENT_SOURCE_DIR}/pyfailure.jsonnet - ) +add_test( + NAME py:failure + COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyfailure.jsonnet +) set_tests_properties( - py:failure PROPERTIES PASS_REGULAR_EXPRESSION - "failed to retrieve property \"input\"" - ) + py:failure + PROPERTIES PASS_REGULAR_EXPRESSION "failed to retrieve property \"input\"" +) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:failure) set(TEST_PYTHONPATH ${CMAKE_CURRENT_SOURCE_DIR}) # Add the python plugin source directory to PYTHONPATH so tests can use phlex # package -set(TEST_PYTHONPATH - ${TEST_PYTHONPATH}:${PROJECT_SOURCE_DIR}/plugins/python/python - ) +set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:${PROJECT_SOURCE_DIR}/plugins/python/python) # Always add site-packages to PYTHONPATH for tests, as embedded python might not # find them especially in spack environments where they are in non-standard @@ -276,32 +229,28 @@ endif() set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:$ENV{PYTHONPATH}) # Environment variables required: -set(PYTHON_TEST_ENVIRONMENT - "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" - ) +set( + PYTHON_TEST_ENVIRONMENT + "SPDLOG_LEVEL=debug;PHLEX_PLUGIN_PATH=${PROJECT_BINARY_DIR};PYTHONPATH=${TEST_PYTHONPATH};PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}" +) -set_tests_properties( - ${ACTIVE_PY_CPHLEX_TESTS} PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}" - ) +set_tests_properties(${ACTIVE_PY_CPHLEX_TESTS} PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") # phlex-based test to ensure that sys.path is sanely set for virtual environment set(PY_VIRTUAL_ENV_DIR ${CMAKE_CURRENT_BINARY_DIR}/py_virtual_env) execute_process(COMMAND python -m venv ${PY_VIRTUAL_ENV_DIR}) configure_file(pysyspath.jsonnet.in pysyspath.jsonnet @ONLY) -add_test(NAME py:syspath COMMAND phlex::phlex -c - ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet - ) +add_test(NAME py:syspath COMMAND phlex::phlex -c ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet) # Activate the Python virtual environment "by hand". Requires setting the # VIRTUAL_ENV environment variable and prepending to the PATH environment # variable. set_tests_properties( py:syspath - PROPERTIES ENVIRONMENT - "${PYTHON_TEST_ENVIRONMENT};VIRTUAL_ENV=${PY_VIRTUAL_ENV_DIR}" - ENVIRONMENT_MODIFICATION - "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" - ) + PROPERTIES + ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT};VIRTUAL_ENV=${PY_VIRTUAL_ENV_DIR}" + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" +) # Python coverage target if(HAS_PYTEST_COV AND ENABLE_COVERAGE) @@ -309,11 +258,12 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) # When cppyy is available, use the full pytest command add_custom_target( coverage-python - COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} - PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL} ${PYTEST_COMMAND} + COMMAND + ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} + PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL} ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report" - ) + ) else() # When cppyy is not available, run a simpler pytest command for standalone # Python tests (like test_variant.py) @@ -322,26 +272,17 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) coverage-python COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${TEST_PYTHONPATH} - PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL_FALLBACK} ${Python_EXECUTABLE} - -m pytest --cov=${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing - --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL_FALLBACK} ${Python_EXECUTABLE} -m pytest + --cov=${CMAKE_CURRENT_SOURCE_DIR} --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc - ${PYTHON_TEST_FILES} + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report (standalone tests only)" - ) + ) endif() endif() # tests of the python support modules -add_test( - NAME py:phlex - COMMAND ${PYTEST_COMMAND} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - ) -set_tests_properties( - py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}" - ) +add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +set_tests_properties(py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") From 3f775b5aade8a5f5b38bc97b883d58eed7499781 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 04:57:58 +0000 Subject: [PATCH 08/12] Apply YAML formatter fixes --- .github/workflows/clang-format-fix.yaml | 4 +++- .github/workflows/clang-tidy-fix.yaml | 4 +++- .github/workflows/cmake-format-fix.yaml | 4 +++- .github/workflows/codeql-analysis.yaml | 8 +------- .github/workflows/header-guards-fix.yaml | 4 +++- .github/workflows/jsonnet-format-fix.yaml | 4 +++- .github/workflows/markdown-fix.yaml | 4 +++- .github/workflows/python-fix.yaml | 4 +++- .github/workflows/yaml-fix.yaml | 4 +++- 9 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/clang-format-fix.yaml b/.github/workflows/clang-format-fix.yaml index 92cf938f6..2b83fbd69 100644 --- a/.github/workflows/clang-format-fix.yaml +++ b/.github/workflows/clang-format-fix.yaml @@ -8,7 +8,9 @@ run-name: "${{ github.actor }} fixing C++ code format" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string workflow_call: diff --git a/.github/workflows/clang-tidy-fix.yaml b/.github/workflows/clang-tidy-fix.yaml index 061bd9568..936db1169 100644 --- a/.github/workflows/clang-tidy-fix.yaml +++ b/.github/workflows/clang-tidy-fix.yaml @@ -8,7 +8,9 @@ name: Clang-Tidy Fix workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string tidy-checks: diff --git a/.github/workflows/cmake-format-fix.yaml b/.github/workflows/cmake-format-fix.yaml index c2c7d3acc..7803fbb37 100644 --- a/.github/workflows/cmake-format-fix.yaml +++ b/.github/workflows/cmake-format-fix.yaml @@ -8,7 +8,9 @@ run-name: "${{ github.actor }} fixing CMake format" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string workflow_call: diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index f9bfad776..322cf978d 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -218,13 +218,7 @@ jobs: fi determine-languages: - needs: - [ - pre-check, - detect-changes-cpp, - detect-changes-python, - detect-changes-actions, - ] + needs: [pre-check, detect-changes-cpp, detect-changes-python, detect-changes-actions] if: always() && needs.pre-check.result == 'success' runs-on: ubuntu-latest outputs: diff --git a/.github/workflows/header-guards-fix.yaml b/.github/workflows/header-guards-fix.yaml index 9cbdcb038..686ed6939 100644 --- a/.github/workflows/header-guards-fix.yaml +++ b/.github/workflows/header-guards-fix.yaml @@ -43,7 +43,9 @@ run-name: "${{ github.actor }} fixing header guards" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string diff --git a/.github/workflows/jsonnet-format-fix.yaml b/.github/workflows/jsonnet-format-fix.yaml index a75725aa7..c179a0ac1 100644 --- a/.github/workflows/jsonnet-format-fix.yaml +++ b/.github/workflows/jsonnet-format-fix.yaml @@ -8,7 +8,9 @@ run-name: "${{ github.actor }} fixing Jsonnet format" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string workflow_call: diff --git a/.github/workflows/markdown-fix.yaml b/.github/workflows/markdown-fix.yaml index 2793cfc37..f5ab467a4 100644 --- a/.github/workflows/markdown-fix.yaml +++ b/.github/workflows/markdown-fix.yaml @@ -43,7 +43,9 @@ run-name: "${{ github.actor }} fixing Markdown format" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string diff --git a/.github/workflows/python-fix.yaml b/.github/workflows/python-fix.yaml index 41d29dc84..02c9c987f 100644 --- a/.github/workflows/python-fix.yaml +++ b/.github/workflows/python-fix.yaml @@ -8,7 +8,9 @@ run-name: "${{ github.actor }} fixing Python code" workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string workflow_call: diff --git a/.github/workflows/yaml-fix.yaml b/.github/workflows/yaml-fix.yaml index 6c261ddc0..bb8a39a44 100644 --- a/.github/workflows/yaml-fix.yaml +++ b/.github/workflows/yaml-fix.yaml @@ -8,7 +8,9 @@ name: YAML Fix workflow_dispatch: inputs: ref: - description: "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the repository's default branch." + description: + "The branch name to checkout and push fixes to (must be a branch, not a commit SHA). Defaults to the + repository's default branch." required: false type: string workflow_call: From 83125ac07c709b6bffa75a63012e3ed242d60141 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 20:58:53 -0800 Subject: [PATCH 09/12] remove unreachable error checking code --- plugins/python/python/phlex/_typing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py index de3bbb293..f69190079 100644 --- a/plugins/python/python/phlex/_typing.py +++ b/plugins/python/python/phlex/_typing.py @@ -149,10 +149,7 @@ def normalize_type(tp: Any, globalns: Dict|None=None, localns: Dict|None=None) - if origin is np.ndarray: # numpy arrays dtype_args = typing.get_args(args[1]) if len(args) >= 2 else () - if dtype_args: - return "ndarray[" + normalize_type(dtype_args[0], globalns, localns) + "]" - else: - raise TypeError("Numpy array with unparameterized or no scalar type") + return "ndarray[" + normalize_type(dtype_args[0], globalns, localns) + "]" if isinstance(origin, type): # regular python typing type name = origin.__name__ From a2c4d0c8a5b491823849254e963bd0cd04b4e19c Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 21:14:06 -0800 Subject: [PATCH 10/12] clang-format fixes --- plugins/python/src/modulewrap.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index f0132f933..45bd5d55a 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -353,7 +353,7 @@ namespace { if (PyArray_IsScalar(pyobject, SignedInteger)) { // convert to Python int first, then to C long, that way we get a Python // OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check long result = PyLong_AsLong(pylong); Py_DECREF(pylong); return result; @@ -375,7 +375,7 @@ namespace { if (PyArray_IsScalar(pyobject, UnsignedInteger)) { // convert to Python int first, then to C unsigned long, that way we get a // Python OverflowError if out-of-range - PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check unsigned long result = PyLong_AsUnsignedLong(pylong); Py_DECREF(pylong); return result; From c633886a0b8fb122f6f40771b810381c68bb33c0 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 21:17:24 -0800 Subject: [PATCH 11/12] tell mypy to ignore a line where it gets confused for some reason --- plugins/python/python/phlex/_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py index 83e4568cb..dede90608 100644 --- a/plugins/python/python/phlex/_typing.py +++ b/plugins/python/python/phlex/_typing.py @@ -78,7 +78,7 @@ def _build_ctypes_map() -> dict[type, str]: for tp in _CTYPES_INTEGER: bits = ctypes.sizeof(tp) * 8 # _type_ uses struct format chars: lowercase = signed, uppercase = unsigned - signed = tp._type_.islower() + signed = tp._type_.islower() # type: ignore # somehow mypy is confused result[tp] = f"{'int' if signed else 'uint'}{bits}_t" return result From 71aba5a40ce10f0afe4415bec8ca5844ae39c382 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 6 Mar 2026 21:35:27 -0800 Subject: [PATCH 12/12] attempt to make CodeQL accept my test --- test/python/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/test_typing.py b/test/python/test_typing.py index 338bb64e3..12cf1898e 100644 --- a/test/python/test_typing.py +++ b/test/python/test_typing.py @@ -75,7 +75,7 @@ def test_special_cases(self): # use of namespaces for evaluation assert("foo") == "foo" - global foo + global foo # lgtm[py/unused-global-variable] foo = np.int64 assert normalize_type("foo", globals()) == "int64_t" bar = np.int32