Skip to content

Commit

Permalink
Fix-array-shape (#18)
Browse files Browse the repository at this point in the history 
* added dtype tests and  max_depth warning

* add struct of empty arrays test

* remove print

* add empty array tests, and remove sring len warning

---------

Co-authored-by: Peter Braun <[email protected]>
  • Loading branch information
@Bilchreis
Bilchreis and Peter Braun authored Nov 20, 2024
1 parent 5fa87de commit 4779eb6
Show file tree
Hide file tree
Showing 3 changed files with 271 additions and 131 deletions.
28 changes: 24 additions & 4 deletions src/secop_ophyd/SECoPSignal.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
import warnings
from functools import wraps
from typing import Any, Callable, Dict, Optional

Expand Down Expand Up @@ -31,6 +32,9 @@
ArrayOf,
)

# max depth for datatypes supported by tiled/databroker
MAX_DEPTH = 1


class LocalBackend(SignalBackend):
"""Class for the 'argument' and 'result' Signal backends of a SECoP_CMD_Device.
Expand Down Expand Up @@ -77,7 +81,7 @@ def __init__(

self.describe_dict["source"] = self.source("")

self.describe_dict.update(self.SECoP_type_info.describe_dict)
self.describe_dict.update(self.SECoP_type_info.get_datakey())

for property_name, prop_val in self.datainfo.items():
if property_name == "type":
Expand Down Expand Up @@ -242,6 +246,14 @@ def __init__(self, path: Path, secclient: AsyncFrappyClient) -> None:

self.SECoP_type_info: SECoPdtype = SECoPdtype(self.SECoPdtype_obj)

if self.SECoP_type_info.max_depth > MAX_DEPTH:
warnings.warn(
f"The datatype of parameter '{path._accessible_name}' has a maximum"
f"depth of {self.SECoP_type_info.max_depth}. Tiled & Databroker only"
f"support a Depth upto {MAX_DEPTH}"
f"dtype_descr: {self.SECoP_type_info.dtype_descr}"
)

self.describe_dict: dict = {}

self.source_name = (
Expand All @@ -260,7 +272,7 @@ def __init__(self, path: Path, secclient: AsyncFrappyClient) -> None:
self.describe_dict["source"] = self.source_name

# add gathered keys from SECoPdtype:
self.describe_dict.update(self.SECoP_type_info.describe_dict)
self.describe_dict.update(self.SECoP_type_info.get_datakey())

for property_name, prop_val in self._param_description.items():
# skip datainfo (treated seperately)
Expand Down Expand Up @@ -303,7 +315,7 @@ async def get_datakey(self, source: str) -> DataKey:

# this ensures the datakey is updated to the latest cached value
SECoPReading(entry=dataset, secop_dt=self.SECoP_type_info)
self.describe_dict.update(self.SECoP_type_info.describe_dict)
self.describe_dict.update(self.SECoP_type_info.get_datakey())

return self.describe_dict

Expand Down Expand Up @@ -390,13 +402,21 @@ def __init__(
self.SECoPdtype_obj: DataType = secop_dtype_obj_from_json(self._prop_value)
self.SECoP_type_info: SECoPdtype = SECoPdtype(self.SECoPdtype_obj)

if self.SECoP_type_info.max_depth > MAX_DEPTH:
warnings.warn(
f"The datatype of parameter '{prop_key}' has a maximum"
f"depth of {self.SECoP_type_info.max_depth}. Tiled & Databroker only"
f"support a Depth upto {MAX_DEPTH}"
f"dtype_descr: {self.SECoP_type_info.dtype_descr}"
)

# SECoP metadata is static and can only change when connection is reset
self.describe_dict = {}
self.source_name = prop_key
self.describe_dict["source"] = self.source_name

# add gathered keys from SECoPdtype:
self.describe_dict.update(self.SECoP_type_info.describe_dict)
self.describe_dict.update(self.SECoP_type_info.get_datakey())

self._secclient: AsyncFrappyClient = secclient
# TODO full property path
Expand Down
110 changes: 67 additions & 43 deletions src/secop_ophyd/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import copy
import time
import warnings
from abc import ABC, abstractmethod
from functools import reduce
from itertools import chain
Expand Down Expand Up @@ -156,6 +157,7 @@ class DtypeNP(ABC):
secop_dtype: DataType
name: str | None
array_element: bool = False
max_depth: int = 0

@abstractmethod
def make_numpy_dtype(self) -> tuple:
Expand Down Expand Up @@ -333,10 +335,10 @@ def __init__(
self.array_element = array_element

if string_dt.maxchars == 1 << 64:
Warning(
"maxchars was not set, default max char lenght is set to: "
+ str(STR_LEN_DEFAULT)
)
# warnings.warn(
# "maxchars was not set, default max char lenght is set to: "
# + str(STR_LEN_DEFAULT)
# )
self.strlen = STR_LEN_DEFAULT

else:
Expand Down Expand Up @@ -367,6 +369,9 @@ def __init__(
for (name, member) in struct_dt.members.items()
}

max_depth = [member.max_depth for member in self.members.values()]
self.max_depth = 1 + max(max_depth)

def make_numpy_dtype(self) -> tuple:
dt_list = []
for member in self.members.values():
Expand Down Expand Up @@ -406,10 +411,15 @@ def __init__(
self.secop_dtype = tuple_dt
self.array_element = array_element
self.members: list[DtypeNP] = [
dt_factory(member, array_element=self.array_element)
for member in tuple_dt.members
dt_factory(
secop_dt=member, name="f" + str(idx), array_element=self.array_element
)
for idx, member in enumerate(tuple_dt.members)
]

max_depth = [member.max_depth for member in self.members]
self.max_depth = 1 + max(max_depth)

def make_numpy_dtype(self) -> tuple:
dt_list = []
for member in self.members:
Expand Down Expand Up @@ -459,6 +469,8 @@ def __init__(
self.members: DtypeNP = dt_factory(array_dt.members, array_element=True)
self.root_type: DtypeNP

self.max_depth = self.members.max_depth

if isinstance(self.members, ArrayNP):
self.shape.extend(self.members.shape)
self.root_type = self.members.root_type
Expand All @@ -473,11 +485,10 @@ def __init__(
self.root_type = self.members

if self.array_element and self.ragged:
pass
# raise NestedRaggedArray(
# "ragged arrays inside of arrays of copmposite datatypes (struct/tuple)"
# "are not supported"
# )
warnings.warn(
"ragged arrays inside of arrays of copmposite datatypes (struct/tuple)"
"are not supported"
)

def make_numpy_dtype(self) -> tuple:
if self.shape == []:
Expand All @@ -487,20 +498,27 @@ def make_numpy_dtype(self) -> tuple:

def make_concrete_numpy_dtype(self, value) -> tuple:

if self.shape == []:
return self.members.make_concrete_numpy_dtype(value)
elif self.ragged is False:
if self.ragged is False:
return (
self.name,
list(self.members.make_concrete_numpy_dtype(value)).pop(),
self.shape,
)
else:
return (
self.name,
list(self.members.make_concrete_numpy_dtype(value)).pop(),
[len(value)],
)
if value == []:
member_np = self.members.make_concrete_numpy_dtype(None)
val_shape = [0]
elif value is None:
member_np = self.members.make_concrete_numpy_dtype(None)
val_shape = []
else:
member_np = self.members.make_concrete_numpy_dtype(value[0])
val_shape = [len(value)]

if isinstance(self.members, ArrayNP):
val_shape = val_shape + member_np[2]

return (self.name, member_np[1], val_shape)

def make_numpy_compatible_list(self, value: list):
return [self.members.make_numpy_compatible_list(elem) for elem in value]
Expand Down Expand Up @@ -540,10 +558,10 @@ def __init__(self, datatype: DataType) -> None:

self._is_composite: bool = False

self.describe_dict: dict = {}

self.dtype_tree = dt_factory(datatype)

self.max_depth: int = self.dtype_tree.max_depth

if isinstance(self.dtype_tree, ArrayNP):
self.shape = self.dtype_tree.shape
self._is_composite = (
Expand All @@ -567,12 +585,8 @@ def __init__(self, datatype: DataType) -> None:

# all composite Dtypes are transported as numpy arrays
self.dtype = "array"

self.dtype_str = self.numpy_dtype.str
self.describe_dict["dtype_str"] = self.dtype_str

self.dtype_descr = self.numpy_dtype.descr
self.describe_dict["dtype_descr"] = self.dtype_descr

# Scalar atomic Datatypes and arrays of atomic dataypes
else:
Expand All @@ -585,9 +599,18 @@ def __init__(self, datatype: DataType) -> None:
else:
self.dtype = SECOP2DTYPE[datatype.__class__]

self.describe_dict["dtype"] = self.dtype
self.describe_dict["shape"] = self.shape
self.describe_dict["SECOP_datainfo"] = self.secop_dtype_str
def get_datakey(self):
describe_dict: dict = {}
# Composite Datatypes & Arrays of COmposite Datatypes
if self._is_composite:
describe_dict["dtype_str"] = self.dtype_str
describe_dict["dtype_descr"] = self.dtype_descr

describe_dict["dtype"] = self.dtype
describe_dict["shape"] = self.shape
describe_dict["SECOP_datainfo"] = self.secop_dtype_str

return describe_dict

def _secop2numpy_array(self, value) -> np.ndarray:
np_list = self.dtype_tree.make_numpy_compatible_list(value)
Expand All @@ -611,28 +634,29 @@ def val2secop(self, input_val) -> Any:
return self.raw_dtype.validate(input_val)

def update_dtype(self, input_val):
if not self._is_composite:
return
if self._is_composite:
# Composite Datatypes & Arrays of Composite Datatypes

# Composite Datatypes & Arrays of Composite Datatypes
dt = self.dtype_tree.make_concrete_numpy_dtype(input_val)

# Top level elements are not named and shape is
# already covered by the shape var
inner_dt = dt[1]

dt = self.dtype_tree.make_concrete_numpy_dtype(input_val)
if isinstance(self.raw_dtype, ArrayOf):
self.shape = dt[2]

# Top level elements are not named and shape is
# already covered by the shape var
dt = dt[1]
self.numpy_dtype = np.dtype(inner_dt)

self.numpy_dtype = np.dtype(dt)
self.dtype_str = self.numpy_dtype.str
self.dtype_descr = self.numpy_dtype.descr

self.dtype_str = self.numpy_dtype.str
self.describe_dict["dtype_str"] = self.dtype_str
return

self.dtype_descr = self.numpy_dtype.descr
self.describe_dict["dtype_descr"] = self.dtype_descr
if isinstance(self.raw_dtype, ArrayOf):
dt = self.dtype_tree.make_concrete_numpy_dtype(input_val)

self.describe_dict["dtype"] = self.dtype
self.describe_dict["shape"] = self.shape
self.describe_dict["SECOP_datainfo"] = self.secop_dtype_str
self.shape = dt[2]


class SECoPReading:
Expand Down
Loading

0 comments on commit 4779eb6

Please sign in to comment.