diff --git a/plugins/python/python/phlex/__init__.py b/plugins/python/python/phlex/__init__.py index b21384ff4..501afc841 100644 --- a/plugins/python/python/phlex/__init__.py +++ b/plugins/python/python/phlex/__init__.py @@ -1,109 +1,4 @@ -"""Annotation helper for C++ typing variants. +"""Phlex Python support module.""" -Python algorithms are generic, like C++ templates, but the Phlex registration -process requires a single unique signature. These helpers generate annotated -functions for registration with the proper C++ types. -""" - -import collections -import copy -import inspect -from typing import Any, Callable - - -class MissingAnnotation(Exception): - """Exception noting the missing of an argument in the provied annotations.""" - - def __init__(self, arg: str): - """Construct exception from the name of the argument without annotation.""" - self.arg = arg - - def __str__(self): - """Report the argument that is missing an annotation.""" - return "argument '%s' is not annotated" % self.arg - - -class Variant: - """Wrapper to associate custom annotations with a callable. - - This class wraps a callable and provides custom ``__annotations__`` and - ``__name__`` attributes, allowing the same underlying function or callable - object to be registered multiple times with different type annotations. - - By default, the provided callable is kept by reference, but can be cloned - (e.g. for callable instances) if requested. - - Phlex will recognize the "phlex_callable" data member, allowing an unwrap - and thus saving an indirection. To detect performance degradation, the - wrapper is not callable by default. - - Attributes: - phlex_callable (Callable): The underlying callable (public). - __annotations__ (dict): Type information of arguments and return product. - __name__ (str): The name associated with this variant. - - Examples: - >>> def add(i: Number, j: Number) -> Number: - ... return i + j - ... - >>> int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd") - """ - - def __init__( - self, - f: Callable, - annotations: dict[str, str | type | Any], - name: str, - clone: bool | str = False, - allow_call: bool = False, - ): - """Annotate the callable F. - - Args: - f (Callable): Annotable function. - annotations (dict): Type information of arguments and return product. - name (str): Name to assign to this variant. - clone (bool|str): If True (or "deep"), creates a shallow (deep) copy - of the callable. - allow_call (bool): Allow this wrapper to forward to the callable. - """ - if clone == "deep": - self.phlex_callable = copy.deepcopy(f) - elif clone: - self.phlex_callable = copy.copy(f) - else: - self.phlex_callable = f - - # annotions are expected as an ordinary dict and should be ordered, but - # we do not require it, so re-order based on the function's co_varnames - self.__annotations__ = collections.OrderedDict() - - sig = inspect.signature(self.phlex_callable) - for k, v in sig.parameters.items(): - try: - self.__annotations__[k] = annotations[k] - except KeyError as e: - if v.default is inspect.Parameter.empty: - raise MissingAnnotation(k) from e - - self.__annotations__["return"] = annotations.get("return", None) - - self.__name__ = name - self.__code__ = getattr(self.phlex_callable, "__code__", None) - self.__defaults__ = getattr(self.phlex_callable, "__defaults__", None) - self._allow_call = allow_call - - def __call__(self, *args, **kwargs): - """Raises an error if called directly. - - Variant instances should not be called directly. The framework should - extract ``phlex_callable`` instead and call that. - - Raises: - AssertionError: To indicate incorrect usage, unless overridden. - """ - assert self._allow_call, ( - f"Variant '{self.__name__}' was called directly. " - f"The framework should extract phlex_callable instead." - ) - return self.phlex_callable(*args, **kwargs) # type: ignore +from ._typing import * # noqa: F403 no names used in this file +from ._variant import * # noqa: F403 idem diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py new file mode 100644 index 000000000..dede90608 --- /dev/null +++ b/plugins/python/python/phlex/_typing.py @@ -0,0 +1,187 @@ +"""Tooling to simplify representation of C++ types in Python. + +There is no canonical form of Python annotations and this gets even more +complicated with the existence of C++ types that have no representation in +Python. This tooling provides helpers to simplify what is presented to the +C++ side, and thus simplifies the maintenance of that code. +""" + +import builtins +import ctypes +import typing +from types import UnionType +from typing import Any, Dict, Union + +import numpy as np + +__all__ = [ + "normalize_type", +] + +# ctypes and numpy types are likely candidates for use in annotations +# TODO: should users be allowed to add to these? +_PY2CPP: dict[type, str] = { + # numpy types + np.bool_: "bool", + np.int8: "int8_t", + np.int16: "int16_t", + np.int32: "int32_t", + np.int64: "int64_t", + np.uint8: "uint8_t", + np.uint16: "uint16_t", + np.uint32: "uint32_t", + np.uint64: "uint64_t", + np.float32: "float", + np.float64: "double", + np.complex64: "std::complex", + np.complex128: "std::complex", + # the following types are aliased in numpy, so ignore here + # np.intp: "ptrdiff_t", + # np.uintp: "size_t", +} + +# ctypes types that don't map cleanly to intN_t / uintN_t +_CTYPES_SPECIAL: dict[type, str] = { + ctypes.c_bool: "bool", + ctypes.c_char: "char", # signedness is implementation-defined in C + ctypes.c_wchar: "wchar_t", + ctypes.c_float: "float", # always IEEE 754 32-bit + ctypes.c_double: "double", # always IEEE 754 64-bit + ctypes.c_longdouble: "long double", # platform-dependent width, no stdint.h alias + # the following types are aliased in ctypes, so ignore here + # ctypes.c_size_t: "size_t", + # ctypes.c_ssize_t: "ssize_t", + ctypes.c_void_p: "void*", + ctypes.c_char_p: "const char*", + ctypes.c_wchar_p: "const wchar_t*", +} + +_CTYPES_INTEGER: list[type] = [ + ctypes.c_byte, + ctypes.c_ubyte, + ctypes.c_short, + ctypes.c_ushort, + ctypes.c_int, + ctypes.c_uint, + ctypes.c_long, + ctypes.c_ulong, +] + +# (unsigned) long long may be aliased +if ctypes.c_longlong is not ctypes.c_long: + _CTYPES_INTEGER.append(ctypes.c_longlong) + _CTYPES_INTEGER.append(ctypes.c_ulonglong) + + +def _build_ctypes_map() -> dict[type, str]: + result = dict(_CTYPES_SPECIAL) + for tp in _CTYPES_INTEGER: + bits = ctypes.sizeof(tp) * 8 + # _type_ uses struct format chars: lowercase = signed, uppercase = unsigned + signed = tp._type_.islower() # type: ignore # somehow mypy is confused + result[tp] = f"{'int' if signed else 'uint'}{bits}_t" + return result + + +_PY2CPP.update(_build_ctypes_map()) + +# use ctypes to construct a mapping from platform types to exact types +_C2C: dict[str, str] = { + "short": _PY2CPP[ctypes.c_short], + "unsigned short": _PY2CPP[ctypes.c_ushort], + "int": _PY2CPP[ctypes.c_int], + "unsigned int": _PY2CPP[ctypes.c_uint], + "long": _PY2CPP[ctypes.c_long], + "unsigned long": _PY2CPP[ctypes.c_ulong], + "long long": "int64_t", + "unsigned long long": "uint64_t", +} + + +def normalize_type(tp: Any, globalns: Dict | None = None, localns: Dict | None = None) -> str: + """Recursively normalize any Python annotation to a canonical name. + + This normalization supports: + - raw types: int, str, float + - typing generics: List[int], Dict[str, int], etc. + - built-in generics (3.9+): list[int], dict[str, int] + - string annotations: "list[int]", "list['int']" + - NoneType + + Args: + tp (Any): Some type annotation to normalize. + globalns (dict): optional global namespace to resolve types. + localns (dict): optional local namespace to resolve types. + + Returns: + Canonical string representation of the type. + """ + # most common case of some string; resolve it to handle `list[int]`, `list['int']` + if isinstance(tp, str): + ns = {**vars(builtins), **vars(typing)} + if globalns: + ns.update(globalns) + if localns: + ns.update(localns) + try: + tp = eval(tp, ns) + except Exception: + return _C2C.get(tp, tp) # unresolvable + + # get the unsubscripted version of the type, or None if it's something unknown + origin = typing.get_origin(tp) + + # sanity check: we have to reject union syntax, because the chosen C++ type must + # be unambiguous (TODO: what with `Optional`?) + if isinstance(tp, UnionType) or origin is Union: + raise TypeError("To support C++, annotations passed to Phlex must be unambiguous") + + # TODO: debatable: maybe pointers should be considered arrays by default? + if isinstance(tp, type) and issubclass(tp, ctypes._Pointer): + raise TypeError("Pointers are ambiguous; declare an array if that is intended") + + # common case of forward references, from e.g. `List["double"]` + if isinstance(tp, typing.ForwardRef): + # ForwardRef.__forward_arg__ is the a string; recurse to resolve it + return normalize_type(tp.__forward_arg__, globalns, localns) + + # special case for NoneType + if tp is type(None): + return "None" + + # clean up generic aliases, such as typing.List[int], list[int], etc. + if origin is not None: + args = typing.get_args(tp) + + if origin is np.ndarray: # numpy arrays + dtype_args = typing.get_args(args[1]) if len(args) >= 2 else () + return "ndarray[" + normalize_type(dtype_args[0], globalns, localns) + "]" + + if isinstance(origin, type): # regular python typing type + name = origin.__name__ + else: + # fallback for unexpected origins + name = getattr(origin, "__name__", repr(origin)) + + return name + "[" + ",".join([normalize_type(a) for a in args]) + "]" + + # ctypes (fixed-size) array types + try: + if issubclass(tp, ctypes.Array): + # TODO: tp._length_ may be useful as well + return "array[" + normalize_type(tp._type_, globalns, localns) + "]" + except TypeError: + pass # tp is not a class + + # known builtin types representations from ctypes and numpy + try: + return _PY2CPP[tp] + except KeyError: + pass # not a known Python type + + # fallback for plain Python types + if isinstance(tp, type): + return _C2C.get(tp.__name__, tp.__name__) + + # fallback for everything else, expecting repr() to be unique and consistent + return repr(tp) diff --git a/plugins/python/python/phlex/_variant.py b/plugins/python/python/phlex/_variant.py new file mode 100644 index 000000000..162939894 --- /dev/null +++ b/plugins/python/python/phlex/_variant.py @@ -0,0 +1,114 @@ +"""Annotation helper for C++ typing variants. + +Python algorithms are generic, like C++ templates, but the Phlex registration +process requires a single unique signature. These helpers generate annotated +functions for registration with the proper C++ types. +""" + +import collections +import copy +import inspect +from typing import Any, Callable + +__all__ = [ + "MissingAnnotation", + "Variant", +] + + +class MissingAnnotation(Exception): + """Exception noting the missing of an argument in the provied annotations.""" + + def __init__(self, arg: str): + """Construct exception from the name of the argument without annotation.""" + self.arg = arg + + def __str__(self): + """Report the argument that is missing an annotation.""" + return "argument '%s' is not annotated" % self.arg + + +class Variant: + """Wrapper to associate custom annotations with a callable. + + This class wraps a callable and provides custom ``__annotations__`` and + ``__name__`` attributes, allowing the same underlying function or callable + object to be registered multiple times with different type annotations. + + By default, the provided callable is kept by reference, but can be cloned + (e.g. for callable instances) if requested. + + Phlex will recognize the "phlex_callable" data member, allowing an unwrap + and thus saving an indirection. To detect performance degradation, the + wrapper is not callable by default. + + Attributes: + phlex_callable (Callable): The underlying callable (public). + __annotations__ (dict): Type information of arguments and return product. + __name__ (str): The name associated with this variant. + + Examples: + >>> def add(i: Number, j: Number) -> Number: + ... return i + j + ... + >>> int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd") + """ + + def __init__( + self, + f: Callable, + annotations: dict[str, str | type | Any], + name: str, + clone: bool | str = False, + allow_call: bool = False, + ): + """Annotate the callable F. + + Args: + f (Callable): Annotable function. + annotations (dict): Type information of arguments and return product. + name (str): Name to assign to this variant. + clone (bool|str): If True (or "deep"), creates a shallow (deep) copy + of the callable. + allow_call (bool): Allow this wrapper to forward to the callable. + """ + if clone == "deep": + self.phlex_callable = copy.deepcopy(f) + elif clone: + self.phlex_callable = copy.copy(f) + else: + self.phlex_callable = f + + # annotions are expected as an ordinary dict and should be ordered, but + # we do not require it, so re-order based on the function's co_varnames + self.__annotations__ = collections.OrderedDict() + + sig = inspect.signature(self.phlex_callable) + for k, v in sig.parameters.items(): + try: + self.__annotations__[k] = annotations[k] + except KeyError as e: + if v.default is inspect.Parameter.empty: + raise MissingAnnotation(k) from e + + self.__annotations__["return"] = annotations.get("return", None) + + self.__name__ = name + self.__code__ = getattr(self.phlex_callable, "__code__", None) + self.__defaults__ = getattr(self.phlex_callable, "__defaults__", None) + self._allow_call = allow_call + + def __call__(self, *args, **kwargs): + """Raises an error if called directly. + + Variant instances should not be called directly. The framework should + extract ``phlex_callable`` instead and call that. + + Raises: + AssertionError: To indicate incorrect usage, unless overridden. + """ + assert self._allow_call, ( + f"Variant '{self.__name__}' was called directly. " + f"The framework should extract phlex_callable instead." + ) + return self.phlex_callable(*args, **kwargs) # type: ignore diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index c48973621..45bd5d55a 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -298,54 +298,30 @@ namespace { static std::string annotation_as_text(PyObject* pyobj) { - std::string ann; - if (!PyUnicode_Check(pyobj)) { - PyObject* pystr = PyObject_GetAttrString(pyobj, "__name__"); // eg. for classes - - // generics like Union have a __name__ that is not useful for our purposes - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr && (strcmp(cstr, "Union") == 0 || strcmp(cstr, "Optional") == 0)) { - Py_DECREF(pystr); - pystr = nullptr; - } + static PyObject* normalizer = nullptr; + if (!normalizer) { + PyObject* phlexmod = PyImport_ImportModule("phlex"); + if (phlexmod) { + normalizer = PyObject_GetAttrString(phlexmod, "normalize_type"); + Py_DECREF(phlexmod); } - if (!pystr) { - PyErr_Clear(); - pystr = PyObject_Str(pyobj); - } - - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr) - ann = cstr; - Py_DECREF(pystr); + if (!normalizer) { + std::string msg; + if (msg_from_py_error(msg, false)) + throw std::runtime_error("unable to retrieve the phlex type normalizer: " + msg); } + } - // for numpy typing, there's no useful way of figuring out the type from the - // name of the type, only from its string representation, so fall through and - // let this method return str() - if (ann != "ndarray" && ann != "list") - return ann; - - // start over for numpy type using result from str() - pystr = PyObject_Str(pyobj); - if (pystr) { - char const* cstr = PyUnicode_AsUTF8(pystr); - if (cstr) // if failed, ann will remain "ndarray" - ann = cstr; - Py_DECREF(pystr); - } - return ann; + PyObject* norm = PyObject_CallOneArg(normalizer, pyobj); + if (!norm) { + std::string msg; + if (msg_from_py_error(msg, false)) + throw std::runtime_error("normalization error: " + msg); } - // unicode object, i.e. string name of the type - char const* cstr = PyUnicode_AsUTF8(pyobj); - if (cstr) - ann = cstr; - else - PyErr_Clear(); + std::string ann = PyUnicode_AsUTF8(norm); + Py_DECREF(norm); return ann; } @@ -369,11 +345,22 @@ namespace { static long pylong_as_strictlong(PyObject* pyobject) { // convert to C++ long, don't allow truncation - if (!PyLong_Check(pyobject)) { - PyErr_SetString(PyExc_TypeError, "int/long conversion expects an integer object"); - return (long)-1; + if (PyLong_Check(pyobject)) { // native Python integer + return PyLong_AsLong(pyobject); + } + + // accept numpy signed integer scalars (int8, int16, int32, int64) + if (PyArray_IsScalar(pyobject, SignedInteger)) { + // convert to Python int first, then to C long, that way we get a Python + // OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check + long result = PyLong_AsLong(pylong); + Py_DECREF(pylong); + return result; } - return (long)PyLong_AsLong(pyobject); // already does long range check + + PyErr_SetString(PyExc_TypeError, "int/long conversion expects a signed integer object"); + return (long)-1; } static unsigned long pylong_or_int_as_ulong(PyObject* pyobject) @@ -384,6 +371,16 @@ namespace { return (unsigned long)-1; } + // accept numpy unsigned integer scalars (uint8, uint16, uint32, uint64) + if (PyArray_IsScalar(pyobject, UnsignedInteger)) { + // convert to Python int first, then to C unsigned long, that way we get a + // Python OverflowError if out-of-range + PyObject* pylong = PyNumber_Long(pyobject); // doesn't fail b/c of type check + unsigned long result = PyLong_AsUnsignedLong(pylong); + Py_DECREF(pylong); + return result; + } + unsigned long ul = PyLong_AsUnsignedLong(pyobject); if (ul == (unsigned long)-1 && PyErr_Occurred() && PyLong_Check(pyobject)) { PyErr_Clear(); @@ -420,10 +417,10 @@ namespace { } BASIC_CONVERTER(bool, bool, PyBool_FromLong, pylong_as_bool) - BASIC_CONVERTER(int, int, PyLong_FromLong, PyLong_AsLong) - BASIC_CONVERTER(uint, unsigned int, PyLong_FromLong, pylong_or_int_as_ulong) - BASIC_CONVERTER(long, long, PyLong_FromLong, pylong_as_strictlong) - BASIC_CONVERTER(ulong, unsigned long, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) + BASIC_CONVERTER(int, std::int32_t, PyLong_FromLong, PyLong_AsLong) + BASIC_CONVERTER(uint, std::uint32_t, PyLong_FromLong, pylong_or_int_as_ulong) + BASIC_CONVERTER(long, std::int64_t, PyLong_FromLong, pylong_as_strictlong) + BASIC_CONVERTER(ulong, std::uint64_t, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) BASIC_CONVERTER(float, float, PyFloat_FromDouble, PyFloat_AsDouble) BASIC_CONVERTER(double, double, PyFloat_FromDouble, PyFloat_AsDouble) @@ -466,10 +463,10 @@ namespace { return (intptr_t)pyll; \ } - VECTOR_CONVERTER(vint, int, NPY_INT) - VECTOR_CONVERTER(vuint, unsigned int, NPY_UINT) - VECTOR_CONVERTER(vlong, long, NPY_LONG) - VECTOR_CONVERTER(vulong, unsigned long, NPY_ULONG) + VECTOR_CONVERTER(vint, std::int32_t, NPY_INT32) + VECTOR_CONVERTER(vuint, std::uint32_t, NPY_UINT32) + VECTOR_CONVERTER(vlong, std::int64_t, NPY_INT64) + VECTOR_CONVERTER(vulong, std::uint64_t, NPY_UINT64) VECTOR_CONVERTER(vfloat, float, NPY_FLOAT) VECTOR_CONVERTER(vdouble, double, NPY_DOUBLE) @@ -517,10 +514,10 @@ namespace { return vec; \ } - NUMPY_ARRAY_CONVERTER(vint, int, NPY_INT, PyLong_AsLong) - NUMPY_ARRAY_CONVERTER(vuint, unsigned int, NPY_UINT, pylong_or_int_as_ulong) - NUMPY_ARRAY_CONVERTER(vlong, long, NPY_LONG, pylong_as_strictlong) - NUMPY_ARRAY_CONVERTER(vulong, unsigned long, NPY_ULONG, pylong_or_int_as_ulong) + NUMPY_ARRAY_CONVERTER(vint, std::int32_t, NPY_INT32, PyLong_AsLong) + NUMPY_ARRAY_CONVERTER(vuint, std::uint32_t, NPY_UINT32, pylong_or_int_as_ulong) + NUMPY_ARRAY_CONVERTER(vlong, std::int64_t, NPY_INT64, pylong_as_strictlong) + NUMPY_ARRAY_CONVERTER(vulong, std::uint64_t, NPY_UINT64, pylong_or_int_as_ulong) NUMPY_ARRAY_CONVERTER(vfloat, float, NPY_FLOAT, PyFloat_AsDouble) NUMPY_ARRAY_CONVERTER(vdouble, double, NPY_DOUBLE, PyFloat_AsDouble) @@ -687,52 +684,34 @@ static bool insert_input_converters(py_phlex_module* mod, if (inp_type == "bool") insert_converter(mod, pyname, bool_to_py, inp_pq, output); - else if (inp_type == "int") + else if (inp_type == "int32_t") insert_converter(mod, pyname, int_to_py, inp_pq, output); - else if (inp_type == "unsigned int") + else if (inp_type == "uint32_t") insert_converter(mod, pyname, uint_to_py, inp_pq, output); - else if (inp_type == "long") + else if (inp_type == "int64_t") insert_converter(mod, pyname, long_to_py, inp_pq, output); - else if (inp_type == "unsigned long") + else if (inp_type == "uint64_t") insert_converter(mod, pyname, ulong_to_py, inp_pq, output); else if (inp_type == "float") insert_converter(mod, pyname, float_to_py, inp_pq, output); else if (inp_type == "double") insert_converter(mod, pyname, double_to_py, inp_pq, output); - else if (inp_type.compare(0, 13, "numpy.ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { + else if (inp_type.compare(0, 7, "ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { // TODO: these are hard-coded std::vector <-> numpy array mappings, which is // way too simplistic for real use. It only exists for demonstration purposes, // until we have an IDL - auto pos = inp_type.rfind("numpy.dtype"); - if (pos == std::string::npos) { - if (inp_type[0] == 'l') { - pos = 0; - } else { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of input type \"%s\"", inp_type.c_str()); - return false; - } - } else { - pos += 18; - } - - if (inp_type.compare(pos, std::string::npos, "uint32]]") == 0 || - inp_type == "list[unsigned int]" || inp_type == "list['unsigned int']") { - insert_converter(mod, pyname, vuint_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "int32]]") == 0 || - inp_type == "list[int]") { + std::string_view dtype{inp_type.begin() + inp_type.rfind('['), inp_type.end()}; + if (dtype == "[int32_t]") { insert_converter(mod, pyname, vint_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true - inp_type == "list[unsigned long]" || inp_type == "list['unsigned long']") { - insert_converter(mod, pyname, vulong_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. - inp_type == "list[long]" || inp_type == "list['long']") { + } else if (dtype == "[uint32_t]") { + insert_converter(mod, pyname, vuint_to_py, inp_pq, output); + } else if (dtype == "[int64_t]") { insert_converter(mod, pyname, vlong_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "float32]]") == 0 || - inp_type == "list[float]") { + } else if (dtype == "[uint64_t]") { + insert_converter(mod, pyname, vulong_to_py, inp_pq, output); + } else if (dtype == "[float]") { insert_converter(mod, pyname, vfloat_to_py, inp_pq, output); - } else if (inp_type.compare(pos, std::string::npos, "float64]]") == 0 || - inp_type == "list[double]" || inp_type == "list['double']") { + } else if (dtype == "[double]") { insert_converter(mod, pyname, vdouble_to_py, inp_pq, output); } else { PyErr_Format(PyExc_TypeError, "unsupported collection input type \"%s\"", inp_type.c_str()); @@ -845,64 +824,44 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw auto out_pq = product_query{.creator = identifier(pyname), .layer = identifier(output_layer), .suffix = identifier(pyoutput)}; - std::string output_type = output_types[0]; + std::string out_type = output_types[0]; std::string output = output_labels[0]; - if (output_type == "bool") + if (out_type == "bool") insert_converter(mod, cname, py_to_bool, out_pq, output); - else if (output_type == "int") + else if (out_type == "int32_t") insert_converter(mod, cname, py_to_int, out_pq, output); - else if (output_type == "unsigned int") + else if (out_type == "uint32_t") insert_converter(mod, cname, py_to_uint, out_pq, output); - else if (output_type == "long") + else if (out_type == "int64_t") insert_converter(mod, cname, py_to_long, out_pq, output); - else if (output_type == "unsigned long") + else if (out_type == "uint64_t") insert_converter(mod, cname, py_to_ulong, out_pq, output); - else if (output_type == "float") + else if (out_type == "float") insert_converter(mod, cname, py_to_float, out_pq, output); - else if (output_type == "double") + else if (out_type == "double") insert_converter(mod, cname, py_to_double, out_pq, output); - else if (output_type.compare(0, 13, "numpy.ndarray") == 0 || - output_type.compare(0, 4, "list") == 0) { + else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { // TODO: just like for input types, these are hard-coded, but should be handled by // an IDL instead. - auto pos = output_type.rfind("numpy.dtype"); - if (pos == std::string::npos) { - if (output_type[0] == 'l') { - pos = 0; - } else { - PyErr_Format( - PyExc_TypeError, "could not determine dtype of output type \"%s\"", output_type.c_str()); - return nullptr; - } - } else { - pos += 18; - } - - if (output_type.compare(pos, std::string::npos, "uint32]]") == 0 || - output_type == "list[unsigned int]" || output_type == "list['unsigned int']") { - insert_converter(mod, cname, py_to_vuint, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "int32]]") == 0 || - output_type == "list[int]") { + std::string_view dtype{out_type.begin() + out_type.rfind('['), out_type.end()}; + if (dtype == "[int32_t]") { insert_converter(mod, cname, py_to_vint, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "uint64]]") == 0 || // need not be true - output_type == "list[unsigned long]" || output_type == "list['unsigned long']") { - insert_converter(mod, cname, py_to_vulong, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "int64]]") == 0 || // id. - output_type == "list[long]" || output_type == "list['long']") { + } else if (dtype == "[uint32_t]") { + insert_converter(mod, cname, py_to_vuint, out_pq, output); + } else if (dtype == "[int64_t]") { insert_converter(mod, cname, py_to_vlong, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "float32]]") == 0 || - output_type == "list[float]") { + } else if (dtype == "[uint64_t]") { + insert_converter(mod, cname, py_to_vulong, out_pq, output); + } else if (dtype == "[float]") { insert_converter(mod, cname, py_to_vfloat, out_pq, output); - } else if (output_type.compare(pos, std::string::npos, "float64]]") == 0 || - output_type == "list[double]" || output_type == "list['double']") { + } else if (dtype == "[double]") { insert_converter(mod, cname, py_to_vdouble, out_pq, output); } else { - PyErr_Format( - PyExc_TypeError, "unsupported collection output type \"%s\"", output_type.c_str()); + PyErr_Format(PyExc_TypeError, "unsupported collection output type \"%s\"", out_type.c_str()); return nullptr; } } else { - PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", output_type.c_str()); + PyErr_Format(PyExc_TypeError, "unsupported output type \"%s\"", out_type.c_str()); return nullptr; } diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 20fd97abd..9bb6cbded 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -56,11 +56,36 @@ check_python_module_version("numba" "0.61.0" HAS_NUMBA) check_python_module_version("numpy" "2.0.0" HAS_NUMPY) check_python_module_version("pytest_cov" "4.0.0" HAS_PYTEST_COV) -if(HAS_CPPYY) - # Explicitly define Python executable variable to ensure it is visible in - # the test environment - set(PYTHON_TEST_EXECUTABLE ${Python_EXECUTABLE}) +# Explicitly define Python executable variable to ensure it is visible in the +# test environment +set(PYTHON_TEST_EXECUTABLE ${Python_EXECUTABLE}) + +set(PYTHON_TEST_FILES test_typing.py test_variant.py) +# Determine pytest command based on coverage support +if(HAS_PYTEST_COV AND ENABLE_COVERAGE) + set( + PYTEST_COMMAND + ${PYTHON_TEST_EXECUTABLE} + -m + pytest + --cov=${CMAKE_CURRENT_SOURCE_DIR} + --cov=${PROJECT_SOURCE_DIR}/plugins/python/python + --cov-report=term-missing + --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml + --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc + ${PYTHON_TEST_FILES} + ) + message(STATUS "Python tests will run with coverage reporting (pytest-cov)") +else() + set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) + if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) + message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") + endif() +endif() + +if(HAS_CPPYY) # Export the location of phlex include headers if(DEFINED ENV{PHLEX_INSTALL}) set(PYTHON_TEST_PHLEX_INSTALL $ENV{PHLEX_INSTALL}) @@ -68,35 +93,9 @@ if(HAS_CPPYY) set(PYTHON_TEST_PHLEX_INSTALL ${CMAKE_SOURCE_DIR}) endif() - set(PYTHON_TEST_FILES test_phlex.py unit_test_variant.py) - - # Determine pytest command based on coverage support - if(HAS_PYTEST_COV AND ENABLE_COVERAGE) - set( - PYTEST_COMMAND - ${PYTHON_TEST_EXECUTABLE} - -m - pytest - --cov=${CMAKE_CURRENT_SOURCE_DIR} - --cov=${PROJECT_SOURCE_DIR}/plugins/python/python - --cov-report=term-missing - --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml - --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc - ${PYTHON_TEST_FILES} - ) - message(STATUS "Python tests will run with coverage reporting (pytest-cov)") - else() - set(PYTEST_COMMAND ${PYTHON_TEST_EXECUTABLE} -m pytest ${PYTHON_TEST_FILES}) - if(ENABLE_COVERAGE AND NOT HAS_PYTEST_COV) - message(WARNING "ENABLE_COVERAGE is ON but pytest-cov not found; Python coverage disabled") - endif() - endif() - - # tests of the python support modules (relies on cppyy) - add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - set_property(TEST py:phlex PROPERTY ENVIRONMENT "PHLEX_INSTALL=${PYTHON_TEST_PHLEX_INSTALL}") + + set(PYTHON_TEST_FILES ${PYTHON_TEST_FILES} test_phlex.py) endif() set(ACTIVE_PY_CPHLEX_TESTS "") @@ -116,7 +115,8 @@ if(HAS_NUMPY) ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:callback3) - # Expect failure for these tests (check for error propagation and type checking) + # Expect failure for these tests (check for error propagation and type + # checking) add_test(NAME py:raise COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyraise.jsonnet) set_tests_properties( py:raise @@ -138,7 +138,7 @@ if(HAS_NUMPY) py:badint PROPERTIES PASS_REGULAR_EXPRESSION - "Python conversion error for type long: int/long conversion expects an integer object" + "Python conversion error for type long: int/long conversion expects a signed integer object" ) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:badint) @@ -209,12 +209,13 @@ set_tests_properties( list(APPEND ACTIVE_PY_CPHLEX_TESTS py:failure) set(TEST_PYTHONPATH ${CMAKE_CURRENT_SOURCE_DIR}) -# Add the python plugin source directory to PYTHONPATH so tests can use phlex package +# Add the python plugin source directory to PYTHONPATH so tests can use phlex +# package set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:${PROJECT_SOURCE_DIR}/plugins/python/python) -# Always add site-packages to PYTHONPATH for tests, as embedded python might -# not find them especially in spack environments where they are in -# non-standard locations +# Always add site-packages to PYTHONPATH for tests, as embedded python might not +# find them especially in spack environments where they are in non-standard +# locations if(Python_SITELIB) set(TEST_PYTHONPATH ${TEST_PYTHONPATH}:${Python_SITELIB}) endif() @@ -241,8 +242,9 @@ execute_process(COMMAND python -m venv ${PY_VIRTUAL_ENV_DIR}) configure_file(pysyspath.jsonnet.in pysyspath.jsonnet @ONLY) add_test(NAME py:syspath COMMAND phlex::phlex -c ${CMAKE_CURRENT_BINARY_DIR}/pysyspath.jsonnet) -# Activate the Python virtual environment "by hand". Requires setting the VIRTUAL_ENV -# environment variable and prepending to the PATH environment variable. +# Activate the Python virtual environment "by hand". Requires setting the +# VIRTUAL_ENV environment variable and prepending to the PATH environment +# variable. set_tests_properties( py:syspath PROPERTIES @@ -250,13 +252,6 @@ set_tests_properties( ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:${PY_VIRTUAL_ENV_DIR}/bin" ) -# Unit tests for the phlex python package -add_test( - NAME py:unit_variant - COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/unit_test_variant.py -) -set_tests_properties(py:unit_variant PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") - # Python coverage target if(HAS_PYTEST_COV AND ENABLE_COVERAGE) if(HAS_CPPYY) @@ -270,8 +265,8 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) COMMENT "Running Python coverage report" ) else() - # When cppyy is not available, run a simpler pytest command - # for standalone Python tests (like unit_test_variant.py) + # When cppyy is not available, run a simpler pytest command for standalone + # Python tests (like test_variant.py) set(PYTHON_TEST_PHLEX_INSTALL_FALLBACK ${CMAKE_SOURCE_DIR}) add_custom_target( coverage-python @@ -281,9 +276,13 @@ if(HAS_PYTEST_COV AND ENABLE_COVERAGE) --cov=${CMAKE_CURRENT_SOURCE_DIR} --cov=${PROJECT_SOURCE_DIR}/plugins/python/python --cov-report=term-missing --cov-report=xml:${CMAKE_BINARY_DIR}/coverage-python.xml --cov-report=html:${CMAKE_BINARY_DIR}/coverage-python-html - --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc unit_test_variant.py + --cov-config=${CMAKE_CURRENT_SOURCE_DIR}/.coveragerc ${PYTHON_TEST_FILES} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Running Python coverage report (standalone tests only)" ) endif() endif() + +# tests of the python support modules +add_test(NAME py:phlex COMMAND ${PYTEST_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +set_tests_properties(py:phlex PROPERTIES ENVIRONMENT "${PYTHON_TEST_ENVIRONMENT}") diff --git a/test/python/all_config.py b/test/python/all_config.py index aeee3b373..0a4c16310 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -49,9 +49,9 @@ def __init__(self, config): assert len(config["some_objects"]) == 3 expected = [ - {'a': 'b', 'c': 'd', 'e': 'f'}, - {'g': 'h', 'i': 'j', 'k': 'l'}, - {'m': 'n', 'o': 'p', 'q': 'r'}, + {"a": "b", "c": "d", "e": "f"}, + {"g": "h", "i": "j", "k": "l"}, + {"m": "n", "o": "p", "q": "r"}, ] for i in range(3): assert config["some_objects"][i] == expected[i] @@ -60,10 +60,10 @@ def __init__(self, config): assert config["empty"] == () try: - config[42] # should raise + config[42] # should raise assert not "did not raise TypeError" except TypeError: - pass # all good as exception was raised + pass # all good as exception was raised def __call__(self, i: int, j: int) -> None: """Dummy routine to do something. diff --git a/test/python/reducer.py b/test/python/reducer.py index b32fe0395..75b283e6d 100644 --- a/test/python/reducer.py +++ b/test/python/reducer.py @@ -74,17 +74,13 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): {"creator": "reduce0", "layer": "event", "suffix": "sum0"}, {"creator": "reduce1", "layer": "event", "suffix": "sum1"}, ] - m.transform( - add_sum01, name="reduce01", input_family=input_family01, output_products=["sum01"] - ) + m.transform(add_sum01, name="reduce01", input_family=input_family01, output_products=["sum01"]) input_family01 = [ {"creator": "reduce2", "layer": "event", "suffix": "sum2"}, {"creator": "reduce3", "layer": "event", "suffix": "sum3"}, ] - m.transform( - add_sum23, name="reduce23", input_family=input_family01, output_products=["sum23"] - ) + m.transform(add_sum23, name="reduce23", input_family=input_family01, output_products=["sum23"]) # once more (and the configuration will add a verifier) input_family_final = [ diff --git a/test/python/sumit.py b/test/python/sumit.py index 17eaef93c..ae43e5ffc 100644 --- a/test/python/sumit.py +++ b/test/python/sumit.py @@ -63,8 +63,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): None """ m.transform(collectify, input_family=config["input"], output_products=["my_pyarray"]) - m.transform(sum_array, - input_family=[ - {"creator" : "collectify", "layer" : "event", "suffix" : "my_pyarray"} - ], - output_products=config["output"]) + m.transform( + sum_array, + input_family=[{"creator": "collectify", "layer": "event", "suffix": "my_pyarray"}], + output_products=config["output"], + ) diff --git a/test/python/test_coverage.py b/test/python/test_coverage.py index 33f82d33b..52c844d89 100644 --- a/test/python/test_coverage.py +++ b/test/python/test_coverage.py @@ -42,13 +42,14 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # We need to transform scalar inputs to lists first # i, f1, d1 come from cppsource4py - tfs = ((collect_int, "input", "i", "l_int"), - (collect_float, "input", "f1", "l_float"), - (collect_double, "input", "d1", "l_double"), - (list_int_func, collect_int.__name__, "l_int", "sum_int"), - (list_float_func, collect_float.__name__, "l_float", "sum_float"), - (list_double_func, collect_double.__name__, "l_double", "sum_double") - ) + tfs = ( + (collect_int, "input", "i", "l_int"), + (collect_float, "input", "f1", "l_float"), + (collect_double, "input", "d1", "l_double"), + (list_int_func, collect_int.__name__, "l_int", "sum_int"), + (list_float_func, collect_float.__name__, "l_float", "sum_float"), + (list_double_func, collect_double.__name__, "l_double", "sum_double"), + ) for func, creator, suffix, output in tfs: input_family = [{"creator": creator, "layer": "event", "suffix": suffix}] diff --git a/test/python/test_mismatch.py b/test/python/test_mismatch.py index 2d188496c..55eca0e7c 100644 --- a/test/python/test_mismatch.py +++ b/test/python/test_mismatch.py @@ -10,6 +10,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): """Register algorithms.""" # input_family has 1 element, but function takes 2 arguments # This should trigger the error in modulewrap.cpp - m.transform(mismatch_func, - input_family=[{"creator": "input", "layer": "event", "suffix": "a"}], - output_products=["sum"]) + m.transform( + mismatch_func, + input_family=[{"creator": "input", "layer": "event", "suffix": "a"}], + output_products=["sum"], + ) diff --git a/test/python/test_typing.py b/test/python/test_typing.py new file mode 100644 index 000000000..12cf1898e --- /dev/null +++ b/test/python/test_typing.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Unit tests for phlex typing normalization.""" + +import ctypes +from typing import Dict, List + +import numpy as np +import numpy.typing as npt +from phlex._typing import _C2C +from pytest import raises + +from phlex import normalize_type + + +class TestTYPING: + """Tests for phlex-specific typinh.""" + + def test_list_normalization(self): + """Normalization of various forms of list annotations.""" + for types in ( + ["int", int, ctypes.c_int], + ["unsigned int", ctypes.c_uint], + ["long", ctypes.c_long], + ["unsigned long", ctypes.c_ulong], + ["long long", ctypes.c_longlong], + ["unsigned long long", ctypes.c_ulonglong], + ["float", float, ctypes.c_float], + ["double", ctypes.c_double], + ): + # TODO: the use of _C2C here is a bit circular + tn = _C2C.get(types[0], types[0]) + + if 0 < tn.find("_"): + npt = tn[: tn.find("_")] + elif tn == "float": + npt = "float32" + elif tn == "double": + npt = "float64" + npt = getattr(np, npt) + + norm = "list[" + tn + "]" + for t in types + [npt]: + assert normalize_type(norm) == norm + try: + assert normalize_type(List[t]) == norm + except SyntaxError as e: + # some of the above are not legal Python when used with List, + # but are fine when used with list; just ignore these as they + # will not show up in user code + if "Forward reference must be an expression" not in str(e): + raise + assert normalize_type(list[t]) == norm + + def test_numpy_array_normalization(self): + """Normalization of standard Numpy typing.""" + for t, s in ( + (np.bool_, "bool"), + (np.int8, "int8_t"), + (np.int16, "int16_t"), + (np.int32, "int32_t"), + (np.int64, "int64_t"), + (np.uint8, "uint8_t"), + (np.uint16, "uint16_t"), + (np.uint32, "uint32_t"), + (np.uint64, "uint64_t"), + (np.float32, "float"), + (np.float64, "double"), + ): + assert normalize_type(npt.NDArray[t]) == "ndarray["+s+"]" + + def test_special_cases(self): + """Special cases.""" + # None type + assert normalize_type(None) == "None" + + # use of namespaces for evaluation + assert("foo") == "foo" + global foo # lgtm[py/unused-global-variable] + foo = np.int64 + assert normalize_type("foo", globals()) == "int64_t" + bar = np.int32 + assert normalize_type("bar", globals()) == "bar" + assert normalize_type("bar", localns=locals()) == "int32_t" + + # union types are not allowed + raises(TypeError, normalize_type, List[int] | Dict[int, int]) + + # pointer types are not supported yet + raises(TypeError, normalize_type, ctypes.POINTER(ctypes.c_int)) + + # ctypes array + assert normalize_type((ctypes.c_int * 42)) == "array[int32_t]" + + # something unknown + class SomeType: + def __repr__(self): + return "some type" + + assert normalize_type(SomeType()) == "some type" diff --git a/test/python/unit_test_variant.py b/test/python/test_variant.py similarity index 68% rename from test/python/unit_test_variant.py rename to test/python/test_variant.py index 0731bb81a..fe65cecf4 100644 --- a/test/python/unit_test_variant.py +++ b/test/python/test_variant.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Unit tests for the phlex.Variant class.""" -import unittest +from pytest import raises from phlex import MissingAnnotation, Variant @@ -14,31 +14,32 @@ def example_func(a, b=1): ann = {"a": int, "b": int, "return": int} -class TestVariant(unittest.TestCase): +class TestVariant: """Tests for Variant wrapper.""" def test_initialization(self): """Test proper initialization and attribute exposure.""" wrapper = Variant(example_func, ann, "example_wrapper") - self.assertEqual(wrapper.__name__, "example_wrapper") - self.assertEqual(wrapper.__annotations__, ann) - self.assertEqual(wrapper.phlex_callable, example_func) + assert wrapper.__name__ == "example_wrapper" + assert wrapper.__annotations__ == ann + assert wrapper.phlex_callable == example_func # Check introspection attributes are exposed - self.assertEqual(wrapper.__code__, example_func.__code__) - self.assertEqual(wrapper.__defaults__, example_func.__defaults__) + assert wrapper.__code__ == example_func.__code__ + assert wrapper.__defaults__ == example_func.__defaults__ def test_call_by_default_raises(self): """Test that calling the wrapper raises AssertionError by default.""" wrapper = Variant(example_func, ann, "no_call") - with self.assertRaises(AssertionError) as cm: + with raises(AssertionError) as cm: wrapper(1) - self.assertIn("was called directly", str(cm.exception)) + + assert "was called directly" in str(cm.value) def test_allow_call(self): """Test that calling is allowed when configured.""" wrapper = Variant(example_func, ann, "yes_call", allow_call=True) - self.assertEqual(wrapper(10, 20), 30) + assert wrapper(10, 20) == 30 def test_clone_shallow(self): """Test shallow cloning behavior.""" @@ -46,7 +47,7 @@ def test_clone_shallow(self): # but let's test the flag logic in Variant wrapper = Variant(example_func, ann, "clone_shallow", clone=True) # function copy is same object - self.assertEqual(wrapper.phlex_callable, example_func) + assert wrapper.phlex_callable == example_func # Test valid copy logic with a mutable callable class CallableObj: @@ -55,7 +56,7 @@ def __call__(self): obj = CallableObj() wrapper_obj = Variant(obj, {}, "obj_clone", clone=True) - self.assertNotEqual(id(wrapper_obj.phlex_callable), id(obj)) # copy was made? + assert id(wrapper_obj.phlex_callable) != id(obj) # copy was made? # copy.copy of a custom object usually creates a new instance if generic def test_clone_deep(self): @@ -70,8 +71,8 @@ def __call__(self): c = Container() wrapper = Variant(c, {}, "deep_clone", clone="deep") - self.assertNotEqual(id(wrapper.phlex_callable), id(c)) - self.assertNotEqual(id(wrapper.phlex_callable.data), id(c.data)) + assert id(wrapper.phlex_callable) != id(c) + assert id(wrapper.phlex_callable.data) != id(c.data) def test_missing_annotation_raises(self): """Test that MissingAnnotation is raised when a required argument is missing.""" @@ -81,11 +82,11 @@ def func(x, y): # Missing 'y' incomplete_ann = {"x": int, "return": int} - with self.assertRaises(MissingAnnotation) as cm: + with raises(MissingAnnotation) as cm: Variant(func, incomplete_ann, "missing_y") - self.assertEqual(str(cm.exception), "argument 'y' is not annotated") - self.assertEqual(cm.exception.arg, "y") + assert str(cm.value) == "argument 'y' is not annotated" + assert cm.value.arg == "y" def test_missing_optional_annotation_does_not_raise(self): """Test that MissingAnnotation is not raised for arguments with default values.""" @@ -96,9 +97,5 @@ def func(x, y=1): # Missing 'y', but it has a default value incomplete_ann = {"x": int, "return": int} wrapper = Variant(func, incomplete_ann, "missing_optional_y") - self.assertIn("x", wrapper.__annotations__) - self.assertNotIn("y", wrapper.__annotations__) - - -if __name__ == "__main__": - unittest.main() + assert "x" in wrapper.__annotations__ + assert "y" not in wrapper.__annotations__ diff --git a/test/python/vectypes.py b/test/python/vectypes.py index de182aabb..18fa0369f 100644 --- a/test/python/vectypes.py +++ b/test/python/vectypes.py @@ -11,43 +11,6 @@ # These are used by the C++ wrapper to identify the correct converter -class _CppTypeMeta(type): - """Metaclass to allow overriding __name__ for C++ type identification.""" - - def __new__(mcs, name, bases, namespace, cpp_name=None): - cls = super().__new__(mcs, name, bases, namespace) - cls._cpp_name = cpp_name if cpp_name else name - return cls - - @property - def __name__(cls): - return cls._cpp_name - - -class unsigned_int(int, metaclass=_CppTypeMeta, cpp_name="unsigned int"): # noqa: N801 - """Type alias for C++ unsigned int.""" - - pass - - -class unsigned_long(int, metaclass=_CppTypeMeta, cpp_name="unsigned long"): # noqa: N801 - """Type alias for C++ unsigned long.""" - - pass - - -class long(int, metaclass=_CppTypeMeta, cpp_name="long"): # noqa: N801, A001 - """Type alias for C++ long.""" - - pass - - -class double(float, metaclass=_CppTypeMeta, cpp_name="double"): # noqa: N801 - """Type alias for C++ double.""" - - pass - - def collectify_int32(i: int, j: int) -> npt.NDArray[np.int32]: """Create an int32 array from two integers.""" return np.array([i, j], dtype=np.int32) @@ -59,39 +22,39 @@ def sum_array_int32(coll: npt.NDArray[np.int32]) -> int: def collectify_uint32( - u1: unsigned_int, - u2: unsigned_int, + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint32]: """Create a uint32 array from two integers.""" return np.array([u1, u2], dtype=np.uint32) -def sum_array_uint32(coll: npt.NDArray[np.uint32]) -> unsigned_int: +def sum_array_uint32(coll: npt.NDArray[np.uint32]) -> np.uint32: """Sum a uint32 array.""" - return unsigned_int(sum(int(x) for x in coll)) + return np.uint32(sum(int(x) for x in coll)) -def collectify_int64(l1: long, l2: long) -> npt.NDArray[np.int64]: +def collectify_int64(l1: "long", l2: "long") -> npt.NDArray[np.int64]: # type: ignore # noqa: F821 """Create an int64 array from two integers.""" return np.array([l1, l2], dtype=np.int64) -def sum_array_int64(coll: npt.NDArray[np.int64]) -> long: +def sum_array_int64(coll: npt.NDArray[np.int64]) -> "int64_t": # type: ignore # noqa: F821 """Sum an int64 array.""" - return long(sum(int(x) for x in coll)) + return np.int64(sum(int(x) for x in coll)) def collectify_uint64( - ul1: unsigned_long, - ul2: unsigned_long, + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 ) -> npt.NDArray[np.uint64]: """Create a uint64 array from two integers.""" return np.array([ul1, ul2], dtype=np.uint64) -def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> unsigned_long: +def sum_array_uint64(coll: npt.NDArray[np.uint64]) -> "uint64_t": # type: ignore # noqa: F821 """Sum a uint64 array.""" - return unsigned_long(sum(int(x) for x in coll)) + return np.uint64(sum(int(x) for x in coll)) def collectify_float32(f1: float, f2: float) -> npt.NDArray[np.float32]: @@ -99,12 +62,12 @@ def collectify_float32(f1: float, f2: float) -> npt.NDArray[np.float32]: return np.array([f1, f2], dtype=np.float32) -def sum_array_float32(coll: npt.NDArray[np.float32]) -> float: +def sum_array_float32(coll: npt.NDArray[np.float32]) -> np.float32: """Sum a float32 array.""" - return float(sum(coll)) + return np.float32(sum(coll)) -def collectify_float64(d1: double, d2: double) -> npt.NDArray[np.float64]: +def collectify_float64(d1: "double", d2: "double") -> npt.NDArray[np.float64]: # type: ignore # noqa: F821 """Create a float64 array from two floats.""" return np.array([d1, d2], dtype=np.float64) @@ -114,14 +77,14 @@ def collectify_float32_list(f1: float, f2: float) -> list[float]: return [f1, f2] -def collectify_float64_list(d1: double, d2: double) -> list["double"]: +def collectify_float64_list(d1: "double", d2: "double") -> list["double"]: # type: ignore # noqa: F821 """Create a float64 list from two floats.""" return [d1, d2] -def sum_array_float64(coll: npt.NDArray[np.float64]) -> double: +def sum_array_float64(coll: npt.NDArray[np.float64]) -> "double": # type: ignore # noqa: F821 """Sum a float64 array.""" - return double(sum(coll)) + return float(sum(coll)) def collectify_int32_list(i: int, j: int) -> list[int]: @@ -130,24 +93,24 @@ def collectify_int32_list(i: int, j: int) -> list[int]: def collectify_uint32_list( - u1: unsigned_int, - u2: unsigned_int, -) -> "list[unsigned int]": # type: ignore # noqa: F722 + u1: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 + u2: "unsigned int", # type: ignore # noqa: F722 # noqa: F821 +) -> list["unsigned int"]: # type: ignore # noqa: F722 # noqa: F821 """Create a uint32 list from two integers.""" - return [unsigned_int(u1), unsigned_int(u2)] + return [np.uint32(u1), np.uint32(u2)] -def collectify_int64_list(l1: long, l2: long) -> "list[long]": # type: ignore # noqa: F722 +def collectify_int64_list(l1: "long", l2: "long") -> "list[int64_t]": # type: ignore # noqa: F821 """Create an int64 list from two integers.""" - return [long(l1), long(l2)] + return [l1, l2] def collectify_uint64_list( - ul1: unsigned_long, - ul2: unsigned_long, -) -> "list[unsigned long]": # type: ignore # noqa: F722 + ul1: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 + ul2: "unsigned long", # type: ignore # noqa: F722 # noqa: F821 +) -> "list[uint64_t]": # type: ignore # noqa: F821 """Create a uint64 list from two integers.""" - return [unsigned_long(ul1), unsigned_long(ul2)] + return [np.uint64(ul1), np.uint64(ul2)] def sum_list_int32(coll: list[int]) -> int: @@ -155,29 +118,29 @@ def sum_list_int32(coll: list[int]) -> int: return sum(coll) -def sum_list_uint32(coll: "list[unsigned int]") -> unsigned_int: # type: ignore # noqa: F722 +def sum_list_uint32(coll: "list[uint32_t]") -> "uint32_t": # type: ignore # noqa: F821 """Sum a list of uints.""" - return unsigned_int(sum(coll)) + return sum(coll) -def sum_list_int64(coll: "list[long]") -> long: # type: ignore # noqa: F722 +def sum_list_int64(coll: "list[int64_t]") -> "int64_t": # type: ignore # noqa: F821 """Sum a list of longs.""" - return long(sum(coll)) + return sum(coll) -def sum_list_uint64(coll: "list[unsigned long]") -> unsigned_long: # type: ignore # noqa: F722 +def sum_list_uint64(coll: "list[uint64_t]") -> "uint64_t": # type: ignore # noqa: F821 """Sum a list of ulongs.""" - return unsigned_long(sum(coll)) + return sum(coll) def sum_list_float(coll: list[float]) -> float: """Sum a list of floats.""" - return sum(coll) + return np.float32(sum(coll)) -def sum_list_double(coll: list["double"]) -> double: +def sum_list_double(coll: list["double"]) -> "double": # type: ignore # noqa: F821 """Sum a list of doubles.""" - return double(sum(coll)) + return sum(coll) def PHLEX_REGISTER_ALGORITHMS(m, config): @@ -259,11 +222,13 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): ) sum_kwargs = { - "input_family": [{ - "creator": list_collect.__name__ if use_lists else arr_collect.__name__, - "layer": "event", - "suffix": arr_name - }], + "input_family": [ + { + "creator": list_collect.__name__ if use_lists else arr_collect.__name__, + "layer": "event", + "suffix": arr_name, + } + ], "output_products": config[out_key], } if sum_name: