diff --git a/src/yadg/dgutils/dsutils.py b/src/yadg/dgutils/dsutils.py index 7dddffc1..252249c0 100644 --- a/src/yadg/dgutils/dsutils.py +++ b/src/yadg/dgutils/dsutils.py @@ -2,6 +2,9 @@ import xarray as xr from xarray import Dataset from typing import Any +import logging + +logger = logging.getLogger(__name__) def append_dicts( @@ -37,19 +40,24 @@ def dicts_to_dataset( fulldate: bool = True, ) -> Dataset: darrs = {} - for k, v in data.items(): + for key, val in data.items(): attrs = {} - u = units.get(k, None) + u = units.get(key, None) if u is not None: attrs["units"] = u - if k == "uts": + if key == "uts": continue - darrs[k] = xr.DataArray(data=v, dims=["uts"], attrs=attrs) - if k in meta and darrs[k].dtype.kind in {"i", "u", "f", "c", "m", "M"}: + if "/" in key: + logger.warning(f"Replacing '/' for '_' in column {key!r}.") + k = key.replace("/", "_") + else: + k = key + darrs[k] = xr.DataArray(data=val, dims=["uts"], attrs=attrs) + if key in meta and darrs[k].dtype.kind in {"i", "u", "f", "c", "m", "M"}: err = f"{k}_std_err" darrs[k].attrs["ancillary_variables"] = err attrs["standard_name"] = f"{k} standard error" - darrs[err] = xr.DataArray(data=meta[k], dims=["uts"], attrs=attrs) + darrs[err] = xr.DataArray(data=meta[key], dims=["uts"], attrs=attrs) if "uts" in data: coords = dict(uts=data.pop("uts")) else: diff --git a/src/yadg/extractors/basic/csv.py b/src/yadg/extractors/basic/csv.py index d98eca1a..c34cd08b 100644 --- a/src/yadg/extractors/basic/csv.py +++ b/src/yadg/extractors/basic/csv.py @@ -138,11 +138,6 @@ def extract( assert len(lines) >= 2 headers = [h.strip().strip(strip) for h in lines[0].split(parameters.sep)] - for hi, header in enumerate(headers): - if "/" in header: - logger.warning("Replacing '/' for '_' in header '%s'.", header) - headers[hi] = header.replace("/", "_") - datecolumns, datefunc, fulldate = dgutils.infer_timestamp_from( headers=headers, spec=parameters.timestamp, timezone=timezone ) diff --git a/tests/test_basiccsv.py b/tests/test_basiccsv.py index 1816b87e..c5a51fb0 100644 --- a/tests/test_basiccsv.py +++ b/tests/test_basiccsv.py @@ -280,3 +280,24 @@ def test_datagram_from_basiccsv(input, ts, datadir): print(f"{ret=}") standard_datagram_test(ret, ts) pars_datagram_test(ret, ts) + + +def test_issue_143(datadir): + input = { + "case": "flow_data.csv", + "version": "4.2", + "parameters": { + "sep": ",", + "timestamp": {"time": {"index": 5}}, + "units": { + "DryCal smL/min": "smL/min", + "DryCal Avg. smL/min": "smL/min", + "Temp. Deg C": "degC", + "Pressure mBar": "mbar", + }, + }, + } + ver = input.pop("version", "4.0") + ret = datagram_from_input(input, "basiccsv", datadir, version=ver) + print(f"{ret=}") + assert ret["0"]["DryCal smL_min"].attrs["units"] == "smL/min" diff --git a/tests/test_basiccsv/flow_data.csv b/tests/test_basiccsv/flow_data.csv new file mode 100644 index 00000000..1304b720 --- /dev/null +++ b/tests/test_basiccsv/flow_data.csv @@ -0,0 +1,30 @@ +Sample,DryCal smL/min ,DryCal Avg. smL/min ,Temp. Deg C,Pressure mBar ,Time, +1,14.5840,14.5950,24.3,971.000,12:45:33, +2,14.5780,14.5930,24.5,971.000,12:46:46, +3,14.5750,14.5910,24.5,971.000,12:47:58, +4,14.5710,14.5890,24.7,971.000,12:49:11, +5,14.5210,14.5830,24.8,971.000,12:50:23, +6,14.5540,14.5800,24.9,971.000,12:51:35, +7,14.5680,14.5800,25.0,971.000,12:52:48, +8,14.5930,14.5800,25.0,971.000,12:54:00, +9,14.6120,14.5830,25.1,971.000,12:55:12, +10,14.6300,14.5860,25.2,971.000,12:56:25, +11,14.6460,14.5890,25.3,971.000,12:57:37, +12,14.6770,14.5940,25.4,971.000,12:58:50, +13,14.7170,14.6000,25.4,971.000,13:00:02, +14,14.8320,14.6120,25.5,971.000,13:01:14, +15,14.8080,14.6210,25.5,971.000,13:02:26, +16,14.7990,14.6290,25.6,971.000,13:03:39, +17,14.6950,14.6320,25.6,971.000,13:04:52, +18,14.6590,14.6330,25.7,970.000,13:06:04, +19,14.7030,14.6360,25.7,971.000,13:07:16, +20,14.7070,14.6390,25.8,971.000,13:08:29, +21,14.8240,14.6460,25.8,971.000,13:09:41, +22,14.8650,14.6540,25.8,971.000,13:10:53, +23,14.8480,14.6600,25.9,971.000,13:12:06, +24,14.7570,14.6640,25.9,971.000,13:13:18, +25,14.6870,14.6640,26.0,970.000,13:14:31, +26,14.7380,14.6670,26.0,971.000,13:15:43, +27,14.8070,14.6710,26.0,971.000,13:16:55, +28,14.8740,14.6770,26.0,970.000,13:18:07, +29,14.8480,14.6820,26.1,971.000,13:19:20,