diff --git a/doc/config.rst b/doc/config.rst index dcc7ddec..0d3187ef 100644 --- a/doc/config.rst +++ b/doc/config.rst @@ -164,10 +164,10 @@ WXT-specific options include: EofE ECHOLOGGER --------------- - All the _min, _max, _bad_ens, etc. options available to the EXO. -- instrument_type: types "ea" and "aa" are supported. -- orientation: orientation of transducers types 'DOWN' or 'UP' are supported. -- average_salinity: average salinity value (PSU) for the water mass for the deployment site and time period. -- average_salinity_note: source of average salinity value. +- ``instrument_type``: types "ea" and "aa" are supported. +- ``orientation``: orientation of transducers types 'DOWN' or 'UP' are supported. +- ``average_salinity``: average salinity value (PSU) for the water mass for the deployment site and time period. +- ``average_salinity_note``: source of average salinity value. Sequoia Scientific LISST ------------------------ @@ -190,4 +190,12 @@ Onset Hobo - ``instrument_type``: can be ``hwl`` (water level), ``hwlb`` (water level as barometer), ``hdo`` (dissolved oxygen) or ``hcnd`` (conductivity) use these based on parameter measured by hobo logger - ``skipfooter``: number of lines to skip in the CSV file at the end of the file - ``ncols``: number of columns of data to read, starting at first -- ``names``: option for user specified column names (only recommended when code will read names using automated/default method) +- ``names``: option for user specified column names (only recommended when code will not read names using automated/default method) + +Lowell TCM Hobo +---------- + +- All the _min, _max, _bad_ens, etc. options available to the EXO. +- ``skipfooter``: number of lines to skip in the CSV file at the end of the file +- ``ncols``: number of columns of data to read, starting at first +- ``names``: option for user specified column names (only recommended when code will not read names using automated/default method) \ No newline at end of file diff --git a/doc/tcm.rst b/doc/tcm.rst new file mode 100644 index 00000000..026d6d47 --- /dev/null +++ b/doc/tcm.rst @@ -0,0 +1,29 @@ +Lowell Tilt Current Meter (TCM) +******************************* + +Data will generally be processed using a series of run scripts. The first script for each instrument type +depends on two :doc:`configuration files `. + +Instrument data to raw .cdf +=========================== + +Convert from exported CSV file to a raw netCDF file with .cdf extension using ``runtcmcsv2cdf.py``. + +runtcmcsv2cdf.py +----------------- + +.. argparse:: + :ref: stglib.core.cmd.tcmcsv2cdf_parser + :prog: runtcmcsv2cdf.py + +Raw .cdf to CF-compliant .nc +============================ + +Convert the raw .cdf data into CF-compliant netCDF file with .nc extension using ``runtcmcdf2nc.py``. + +runtcmcdf2nc.py +---------------- + +.. argparse:: + :ref: stglib.core.cmd.tcmcdf2nc_parser + :prog: runtcmcdf2nc.py diff --git a/setup.py b/setup.py index 4396707d..09d560fa 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,8 @@ "runsigcdf2nc.py=stglib.core.runcmd:runsigcdf2nc", "runlisstcsv2cdf.py=stglib.core.runcmd:runlisstcsv2cdf", "runlisstcdf2nc.py=stglib.core.runcmd:runlisstcdf2nc", + "runtcmcsv2cdf.py=stglib.core.runcmd:runtcmcsv2cdf", + "runtcmcdf2nc.py=stglib.core.runcmd:runtcmcdf2nc", ], }, include_package_data=True, diff --git a/stglib/__init__.py b/stglib/__init__.py index 3580b98f..15fcd11c 100644 --- a/stglib/__init__.py +++ b/stglib/__init__.py @@ -13,6 +13,7 @@ rdi, rsk, sig, + tcm, troll, vec, wxt, diff --git a/stglib/core/cmd.py b/stglib/core/cmd.py index 93da4558..67e13947 100644 --- a/stglib/core/cmd.py +++ b/stglib/core/cmd.py @@ -359,3 +359,20 @@ def sigdlfncdf2nc_parser(): atmarg(parser) return parser + + +def tcmcsv2cdf_parser(): + description = "Convert Lowell Tilt Current Meter .txt file to raw .cdf format. Run this script from the directory containing TCM file." + parser = argparse.ArgumentParser(description=description) + gattsarg(parser) + yamlarg(parser) + + return parser + + +def tcmcdf2nc_parser(): + description = "Convert raw TCM .cdf format to processed .nc files" + parser = argparse.ArgumentParser(description=description) + cdfarg(parser) + + return parser diff --git a/stglib/core/runcmd.py b/stglib/core/runcmd.py index cd53e664..52baaed3 100644 --- a/stglib/core/runcmd.py +++ b/stglib/core/runcmd.py @@ -291,3 +291,17 @@ def runwxtcsv2cdf(): metadata = get_metadata(args) stglib.wxt.csv_to_cdf(metadata) + + +def runtcmcdf2nc(): + args = stglib.cmd.tcmcdf2nc_parser().parse_args() + + run_cdf_to_nc(stglib.tcm.cdf_to_nc, args) + + +def runtcmcsv2cdf(): + args = stglib.cmd.tcmcsv2cdf_parser().parse_args() + + metadata = get_metadata(args) + + stglib.tcm.csv_to_cdf(metadata) diff --git a/stglib/hobo.py b/stglib/hobo.py index e4585754..fdda7239 100644 --- a/stglib/hobo.py +++ b/stglib/hobo.py @@ -328,7 +328,12 @@ def ds_add_attrs(ds): if "BPR_915" in ds: ds["BPR_915"].attrs.update( - {"units": "mbar", "long_name": "Barometric pressure", "epic_code": 915} + { + "units": "mbar", + "long_name": "Barometric pressure", + "epic_code": 915, + "standard_name": "air_pressure", + } ) if "T_21" in ds: diff --git a/stglib/tcm.py b/stglib/tcm.py new file mode 100644 index 00000000..78a4eaea --- /dev/null +++ b/stglib/tcm.py @@ -0,0 +1,384 @@ +import csv +import string +import warnings + +import numpy as np +import pandas as pd +import xarray as xr + +from .aqd import aqdutils +from .core import qaqc, utils + + +def read_tcm( + filnam, + skiprows=1, + skipfooter=0, + names=["DateTime", "Speed", "Bearing", "Velocity-N", "Velocity-E"], +): + """Read data from an Lowell Instruments Tilt Current Meter (TCM) _CR.txt file into an xarray + Dataset. + + Parameters + ---------- + filnam : string + The filename + skiprows : int, optional + How many header rows to skip. Default 1 + skipfooter : int, optional + How many footer rows to skip. Default 0 + + Returns + ------- + xarray.Dataset + An xarray Dataset of the TCM data + """ + tcm = pd.read_csv( + filnam, + usecols=np.arange(len(names)), + names=names, + engine="python", + skiprows=skiprows, + skipfooter=skipfooter, + ) + + tcm["time"] = pd.to_datetime(tcm["DateTime"]) + + tcm.set_index("time", inplace=True) + + return xr.Dataset(tcm) + + +def csv_to_cdf(metadata): + """ + Process TCM .txt file to a raw .cdf file + """ + + basefile = metadata["basefile"] + + kwargs = {"skiprows": metadata["skiprows"], "skipfooter": metadata["skipfooter"]} + if "names" in metadata: + kwargs["names"] = metadata["names"] + else: + kwargs["names"] = get_tcm_col_names(basefile + "_CR.txt", metadata) + + try: + ds = read_tcm(basefile + "_CR.txt", **kwargs) + except IOError: + print(f"Could not read file {basefile}_CR.txt, check file encoding, use utf-8") + + metadata.pop("skiprows") + metadata.pop("skipfooter") + if "ncols" in metadata: + metadata.pop("ncols") + + # write out metadata first, then deal exclusively with xarray attrs + ds = utils.write_metadata(ds, metadata) + + del metadata + + ds = utils.ensure_cf(ds) + + ds = utils.shift_time(ds, 0) + + ds = drop_vars(ds) + + ds.attrs["serial_number"] = get_serial_number(basefile + ".lid") + + # configure file + cdf_filename = ds.attrs["filename"] + "-raw.cdf" + + ds.to_netcdf(cdf_filename, unlimited_dims=["time"]) + + print("Finished writing data to %s" % cdf_filename) + + return ds + + +def drop_vars(ds): + todrop = ["DateTime"] + return ds.drop([x for x in todrop if x in ds]) + + +def ds_rename_vars(ds): + # convert some units + + if "Velocity-E_cms" in ds: + ds["Velocity-E_cms"].values = ( + ds["Velocity-E_cms"].values / 100 + ) # convert from cm/s to m/s + ds = ds.rename({"Velocity-E_cms": "u_1205"}) + + if "Velocity-N_cms" in ds: + ds["Velocity-N_cms"].values = ( + ds["Velocity-N_cms"].values / 100 + ) # convert from cm/s to m/s + ds = ds.rename({"Velocity-N_cms": "v_1206"}) + + if "Speed_cms" in ds: + ds["Speed_cms"].values = ( + ds["Speed_cms"].values / 100 + ) # convert from cm/s to m/s + ds = ds.rename({"Speed_cms": "CS_300"}) + + if "Bearing_degrees" in ds: + ds = ds.rename({"Bearing_degrees": "CD_310"}) + + # drop unneeded vars + todrop = [""] + ds = ds.drop([x for x in todrop if x in ds]) + + return ds + + +def ds_add_attrs(ds): + # Update attributes for EPIC and STG compliance + ds = utils.ds_coord_no_fillvalue(ds) + + ds["time"].attrs.update( + {"standard_name": "time", "axis": "T", "long_name": "time (UTC)"} + ) + + if "u_1205" in ds: + ds["u_1205"].attrs.update( + { + "units": "m s^-1", + "long_name": "Eastward Velocity", + "epic_code": 1205, + } + ) + + if "v_1206" in ds: + ds["v_1206"].attrs.update( + { + "units": "m s^-1", + "long_name": "Northward Velocity", + "epic_code": 1206, + } + ) + + if "T_28" in ds: + ds["T_28"].attrs.update( + { + "units": "degree_C", + "long_name": "Temperature", + "epic_code": 28, + "standard_name": "sea_water_temperature", + } + ) + + if "CS_300" in ds: + ds["CS_300"].attrs.update( + { + "units": "m s^-1", + "long_name": "Current Speed", + "epic_code": 300, + "standard_name": "sea_water_speed", + } + ) + + if "CD_310" in ds: + ds["CD_310"].attrs.update( + { + "units": "degree", + "long_name": "Current Diirection (True)", + "epic_code": 310, + "standard_name": "sea_water_velocity_to_direction", + } + ) + + return ds + + +def get_serial_number(filnam): + """get the serial number of the instrument from the lid file""" + with open(filnam, "rb") as f: + while True: + row = f.readline() + print(row) + if b"SER " in row: + sn = row.split()[1] + return sn.decode("ascii") + + +def strip_non_printable(strin): + """Returns the string without non printable characters""" + printable = set(string.printable) + return "".join(filter(lambda x: x in printable, strin)) + + +def get_tcm_col_names(filnam, metadata): + """get column names and column units from instrument input data file""" + + with open(filnam) as f: + rdr = csv.reader(f) + # check to see if "Time" in first + hdrline = next(rdr) + while "Time" not in hdrline[0]: + hdrline = next(rdr) + + collist = [x.split(")")[0].replace(" ", "") for x in hdrline] + # set first column to DateTime + collist[0] = "DateTime" + # print(collist) + + colnames = [] + colunits = [] + for x in collist: + # spl = x.split(",") + spl = x.replace(" ", "").split("(") + colnames.append(spl[0].strip(".")) + if len(spl) > 1: + colunits.append(spl[1].strip()) + else: + colunits.append("") + + if "ncols" in metadata: + colnames = colnames[: metadata["ncols"]] + colunits = colunits[: metadata["ncols"]] + + # make dict of names and units + dcols = {} + for i in range(len(colnames)): + d = {colnames[i]: colunits[i]} + dcols.update(d) + + # try removing special characters and those not allowed in var or dim names from units + for k in dcols: + # first step try replacing values + if "µ" in dcols[k]: + dcols[k] = dcols[k].replace("µ", "u") + if "°" in dcols[k]: + dcols[k] = dcols[k].replace("°", "") + if "%" in dcols[k]: + dcols[k] = dcols[k].replace("%", "percent") + if "Temp" in k: + if "C" in dcols[k]: + dcols[k] = "C" + elif "F" in dcols[k]: + dcols[k] = "F" + if "/" in dcols[k]: + dcols[k] = dcols[k].replace("/", "") + + # then strip non-ascii characters + dcols[k] = strip_non_printable(dcols[k]) + + names = [] + for k in dcols: + if k == "#" or k == "DateTime": + names.append(k) + else: + names.append(k + "_" + dcols[k]) + + return names + + +def magvar_correct(ds): + """Correct for magnetic declination at site""" + + if "magnetic_variation_at_site" in ds.attrs: + magvardeg = ds.attrs["magnetic_variation_at_site"] + elif "magnetic_variation" in ds.attrs: + magvardeg = ds.attrs["magnetic_variation"] + else: + warnings.warn( + "No magnetic variation information provided; not correcting compass orientation" + ) + magvardeg = 0 + + if magvardeg != 0: + histtext = "Rotating heading and horizontal velocities by {} degrees.".format( + magvardeg + ) + + ds = utils.insert_history(ds, histtext) + + if "U" in ds and "V" in ds: + uvar = "U" + vvar = "V" + elif "u_1205" in ds and "v_1206" in ds: + uvar = "u_1205" + vvar = "v_1206" + + ds[uvar], ds[vvar] = aqdutils.rotate(ds[uvar], ds[vvar], magvardeg) + + if "CD_310" in ds: + dvar = "CD_310" + + ds[dvar] = (ds[dvar] + magvardeg) % 360 + + return ds + + +def cdf_to_nc(cdf_filename): + """ + Load a raw .cdf file and generate a processed .nc file + """ + + # Load raw .cdf data + ds = xr.open_dataset(cdf_filename) + + # remove units in case we change and we can use larger time steps + ds.time.encoding.pop("units") + + # Clip data to in/out water times or via good_ens + ds = utils.clip_ds(ds) + + # rename variables + ds = ds_rename_vars(ds) + + ds = magvar_correct(ds) + + # should function this + for var in ds.data_vars: + ds = qaqc.trim_min(ds, var) + ds = qaqc.trim_max(ds, var) + ds = qaqc.trim_min_diff(ds, var) + ds = qaqc.trim_max_diff(ds, var) + ds = qaqc.trim_med_diff(ds, var) + ds = qaqc.trim_med_diff_pct(ds, var) + ds = qaqc.trim_bad_ens(ds, var) + ds = qaqc.trim_maxabs_diff_2d(ds, var) + ds = qaqc.trim_fliers(ds, var) + + # after check for masking vars by other vars + for var in ds.data_vars: + ds = qaqc.trim_mask(ds, var) + + # check for drop_vars is config yaml + if "drop_vars" in ds.attrs: + ds = qaqc.drop_vars(ds) + + ds = utils.create_z(ds) # added 7/31/2023 + + ds = ds_add_attrs(ds) + + ds = utils.add_standard_names(ds) + + # assign min/max: + ds = utils.add_min_max(ds) + + ds = utils.add_start_stop_time(ds) + + ds = utils.add_delta_t(ds) + + # add lat/lon coordinates + ds = utils.ds_add_lat_lon(ds) + + if "vert_dim" in ds.attrs: + vdim = ds.attrs["vert_dim"] + attrsbak = ds[vdim].attrs + # axis attr set for z in utils.create_z so need to del if other than z + if ds.attrs["vert_dim"] != "z": + ds[vdim].attrs["axis"] = "Z" + del ds["z"].attrs["axis"] + + # Write to .nc file + print("Writing cleaned/trimmed data to .nc file") + nc_filename = ds.attrs["filename"] + "-a.nc" + + ds.to_netcdf( + nc_filename, unlimited_dims=["time"], encoding={"time": {"dtype": "i4"}} + ) + utils.check_compliance(nc_filename, conventions=ds.attrs["Conventions"]) + print("Done writing netCDF file", nc_filename) diff --git a/stglib/tests/data/.gitattributes b/stglib/tests/data/.gitattributes index c935d740..47f53eb6 100644 --- a/stglib/tests/data/.gitattributes +++ b/stglib/tests/data/.gitattributes @@ -1,9 +1,5 @@ TB_1119_17H103145_091918_080000.csv filter=lfs diff=lfs merge=lfs -text bel5_19D105366_012221_120000_utf8.csv filter=lfs diff=lfs merge=lfs -text -1119Aexo_config.yaml filter=lfs diff=lfs merge=lfs -text -config_bel5C.yaml filter=lfs diff=lfs merge=lfs -text -glob_att1119a.txt filter=lfs diff=lfs merge=lfs -text -glob_attbel5C.txt filter=lfs diff=lfs merge=lfs -text TB111803.hdr filter=lfs diff=lfs merge=lfs -text TB111803.v1 filter=lfs diff=lfs merge=lfs -text TB111803.v2 filter=lfs diff=lfs merge=lfs -text @@ -13,30 +9,17 @@ TB111803.ssl filter=lfs diff=lfs merge=lfs -text TB111803.v3 filter=lfs diff=lfs merge=lfs -text TB111803.a3 filter=lfs diff=lfs merge=lfs -text TB111803.sen filter=lfs diff=lfs merge=lfs -text -glob_att1118a_b.txt filter=lfs diff=lfs merge=lfs -text -aqd1118A_config.yaml filter=lfs diff=lfs merge=lfs -text Dunex_2021-09-13.csv filter=lfs diff=lfs merge=lfs -text -glob_att1149.txt filter=lfs diff=lfs merge=lfs -text -wxt1149_config.yaml filter=lfs diff=lfs merge=lfs -text -config_1097C.yaml filter=lfs diff=lfs merge=lfs -text -glob_att1097C.txt filter=lfs diff=lfs merge=lfs -text 1097CIQ_20171214_125948.mat filter=lfs diff=lfs merge=lfs -text -11032Decn_config.yaml filter=lfs diff=lfs merge=lfs -text 1103D_ntu507.log filter=lfs diff=lfs merge=lfs -text -glob_att1103D.txt filter=lfs diff=lfs merge=lfs -text 051001_CSF20SC201.zip filter=lfs diff=lfs merge=lfs -text -1123Aea_example_config.yaml filter=lfs diff=lfs merge=lfs -text ea400_example_data.log filter=lfs diff=lfs merge=lfs -text -glob_att1123A_msl.txt filter=lfs diff=lfs merge=lfs -text -glob_att1137.txt filter=lfs diff=lfs merge=lfs -text -1137aa_config.yaml filter=lfs diff=lfs merge=lfs -text aa400_0074_orange_raw_trim.log filter=lfs diff=lfs merge=lfs -text aa400_0074_orange_raw_trim_utf8.log filter=lfs diff=lfs merge=lfs -text stglib/tests/data/11211Aaqd-a.nc.cfcheck.txt filter=lfs diff=lfs merge=lfs -text stglib/tests/data/11211Aaqd-a.nc filter=lfs diff=lfs merge=lfs -text stglib/tests/data/11211Aaqd-raw.cdf filter=lfs diff=lfs merge=lfs -text stglib/tests/data/glob_att1121a_msl_aqd.txt filter=lfs diff=lfs merge=lfs -text -stglib/tests/data/aqd1121A_config.yaml filter=lfs diff=lfs merge=lfs -text stglib/tests/data/1121AQ04.v3 filter=lfs diff=lfs merge=lfs -text stglib/tests/data/1121AQ04.v2 filter=lfs diff=lfs merge=lfs -text stglib/tests/data/1121AQ04.v1 filter=lfs diff=lfs merge=lfs -text @@ -46,9 +29,6 @@ stglib/tests/data/1121AQ04.a3 filter=lfs diff=lfs merge=lfs -text stglib/tests/data/1121AQ04.a2 filter=lfs diff=lfs merge=lfs -text stglib/tests/data/1121AQ04.a1 filter=lfs diff=lfs merge=lfs -text KorEXO_Measurement_File_Export_081322_170216_rev.csv filter=lfs diff=lfs merge=lfs -text -glob_att1151b.txt filter=lfs diff=lfs merge=lfs -text -1151Bexo_config.yaml filter=lfs diff=lfs merge=lfs -text -glob_att1123A_msl_EPIC.txt filter=lfs diff=lfs merge=lfs -text BEL503.a1 filter=lfs diff=lfs merge=lfs -text BEL503.a2 filter=lfs diff=lfs merge=lfs -text BEL503.hdr filter=lfs diff=lfs merge=lfs -text @@ -80,14 +60,8 @@ TDHAQD02.v1 filter=lfs diff=lfs merge=lfs -text TDHAQD02.v2 filter=lfs diff=lfs merge=lfs -text TDHAQD02.v3 filter=lfs diff=lfs merge=lfs -text TDHAQD02.a2 filter=lfs diff=lfs merge=lfs -text -config_CHC14TDH.yaml filter=lfs diff=lfs merge=lfs -text -gatts_CHC14TDH.txt filter=lfs diff=lfs merge=lfs -text -config_nbm22cce01.yaml filter=lfs diff=lfs merge=lfs -text -nbm22cce01_gatts.txt filter=lfs diff=lfs merge=lfs -text atmpres_NBM22CCE.cdf filter=lfs diff=lfs merge=lfs -text bel3_13H102609_012221_120000_utf8.csv filter=lfs diff=lfs merge=lfs -text -config_bel3C.yaml filter=lfs diff=lfs merge=lfs -text -glob_attbel3C.txt filter=lfs diff=lfs merge=lfs -text atmpres_BEL19B3C03exo.cdf filter=lfs diff=lfs merge=lfs -text *.a1 filter=lfs diff=lfs merge=lfs -text *.a2 filter=lfs diff=lfs merge=lfs -text @@ -98,8 +72,6 @@ atmpres_BEL19B3C03exo.cdf filter=lfs diff=lfs merge=lfs -text *.v1 filter=lfs diff=lfs merge=lfs -text *.v2 filter=lfs diff=lfs merge=lfs -text *.v3 filter=lfs diff=lfs merge=lfs -text -glob_att1113_aqdHR_tst.txt filter=lfs diff=lfs merge=lfs -text -aqdhr1113tst_config.yaml filter=lfs diff=lfs merge=lfs -text *.sen filter=lfs diff=lfs merge=lfs -text *.hdr filter=lfs diff=lfs merge=lfs -text *.hdr2 filter=lfs diff=lfs merge=lfs -text @@ -108,7 +80,9 @@ aqdhr1113tst_config.yaml filter=lfs diff=lfs merge=lfs -text 055109_20220808_1605/055109_20220808_1605_data.txt filter=lfs diff=lfs merge=lfs -text 055109_20220808_1605/055109_20220808_1605_events.txt filter=lfs diff=lfs merge=lfs -text 205598_20220104_1336.zip filter=lfs diff=lfs merge=lfs -text -gatts_205598_20220104_1336_wtw21.txt filter=lfs diff=lfs merge=lfs -text -config_205598_20220104_1336_wtw21.yaml filter=lfs diff=lfs merge=lfs -text atmpres_205598_20220104_1336_wtw21.cdf filter=lfs diff=lfs merge=lfs -text L0221705_v2_rs.csv filter=lfs diff=lfs merge=lfs -text +*.lid filter=lfs diff=lfs merge=lfs -text +Bang_--_MBW.csv filter=lfs diff=lfs merge=lfs -text +Barometer_w_RainGauge.csv filter=lfs diff=lfs merge=lfs -text +1711426_Bang--MBW_0_CR.txt filter=lfs diff=lfs merge=lfs -text diff --git a/stglib/tests/data/11032Decn_config.yaml b/stglib/tests/data/11032Decn_config.yaml index 7910e2d0..4c261ddb 100644 --- a/stglib/tests/data/11032Decn_config.yaml +++ b/stglib/tests/data/11032Decn_config.yaml @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:937663b3e2432f73ef5a1b4c001346d68b0a7e194c195a7e45f9f9935265c13a -size 751 +basefile: '1103D_ntu507.log' +filename: '11032Decn' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +initial_instrument_height: 0.065 +spb: 10 +skiprows: 63 +skipfooter: 1 +serial_number: 507 +INST_TYPE: 'WET Labs ECO-NTUSB' +user_ntucal: 'ntu = 6.203E-07*counts**2 + 8.392E-02*counts - 4.922E+00' +user_ntucal_coeffs: [6.203E-07, 8.392E-02, -4.922E+00] +user_ntucal_date: '2017-07-17' +user_ntucal_note: '8 point Formazin calibration by Steve Suttles on 2017-07-17' +Turb_max: 60 +Turb_min: 0 +Turb_maxabs_diff: 20 +#Turb_max_diff: 20 +#Turb_min_diff: -20 +Turb_bad_ens: ['2018-05-09 17:15', '2018-05-11 00:00'] +good_dates: + - '2018-03-28 19:40' + - '2018-06-13' +Turb_max_std: 10 +Turb_std_ratio: 0.5 +Turb_std_mask: ["Turb"] \ No newline at end of file diff --git a/stglib/tests/data/1119Aexo_config.yaml b/stglib/tests/data/1119Aexo_config.yaml index 03d9d5aa..146d8b82 100644 --- a/stglib/tests/data/1119Aexo_config.yaml +++ b/stglib/tests/data/1119Aexo_config.yaml @@ -1,3 +1,21 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:793fd87064293b783f98f90a7a49e28051a869cfcf896baac8d28eef3d519cf3 -size 1013 +basefile: 'TB_1119_17H103145_091918_080000' +filename: '1119Aexo' # name of output file, -raw.cdf or .nc will be appended to this +# SN: '17H103145' +LatLonDatum: 'NAD83' +ClockError: 0 # sec, negative is slow +initial_instrument_height: 0.13 # meters - estimated!!! from top of grate +initial_instrument_height_note: 'Not recorded, mooring log suggested to use 0.13m' +zeroed_pressure: 'No' # was pressure zeroed before deployment +P_1ac_note: 'No USGS HOBO met station deployed during 1119A. Corrected for variations in atmospheric pressure using Millville, Millville Municipal Airport MesoWest met station (KMIV).' +skiprows: 8 +fDOMRFU_max_diff: 13.5 +fDOMRFU_min_diff: -13.5 +#CHLrfu_max_diff: 7.0 +CHLrfu_med_diff: 5.0 +C_51_min_diff: -0.3 #Conductivity m/s +SpC_48_min_diff: -3.5 #SpConducitivty m/s +SpC_48_max_diff: 3.50 #SpConducitivty m/s +S_41_min_diff: -2.00 #Salinity psu +S_41_max_diff: 2.00 #Salinity psu +Turb_med_diff: 100.0 +good_ens: [5, 100, 250, 600] # test for multiple good_ens ranges diff --git a/stglib/tests/data/1123Aea_example_config.yaml b/stglib/tests/data/1123Aea_example_config.yaml index 36286d2a..34e4847f 100644 --- a/stglib/tests/data/1123Aea_example_config.yaml +++ b/stglib/tests/data/1123Aea_example_config.yaml @@ -1,3 +1,17 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b180561a8cc2a3df2ad9e15268a05da9ba3939533538bc6d070a5ddcda28e75a -size 669 +basefile: 'ea400_example_data' +filename: '11231Aea_example' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +#VerticalDatum: 'NAVD88' +ClockError: 0 #sec +instrument_type: 'ea' +orientation: 'DOWN' # use this to identify orientation of profiler +initial_instrument_height: 0.34 # meters, seabed to transducer +average_salinity: 28.4 +average_salinity_note: 'averaged from exo, PSU' +trim_method: ["altitude", "bin range"] +good_bins: [0,70] +brange_maxabs_diff: 0.02 #trim abrupt changes in altitude +brange_min: 0.3 #trim values less than deadzone distance +brange_max: 0.6 +seabed_elevation_mask: ["brange"] +vert_dim: 'z' diff --git a/stglib/tests/data/1137aa_config.yaml b/stglib/tests/data/1137aa_config.yaml index cc201e1d..6fa4342e 100644 --- a/stglib/tests/data/1137aa_config.yaml +++ b/stglib/tests/data/1137aa_config.yaml @@ -1,3 +1,10 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:057b6fdc922b854640e11ce9d8b1a4d95594a5febc88c25d42b96b6072d1b8e7 -size 501 +basefile: 'aa400_0074_orange_raw_trim_utf8' +filename: '11373aa' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +ClockError: 0 #sec +instrument_type: 'aa' +orientation: 'DOWN' # use this to identify orientation of profiler +initial_instrument_height: 0.34 # meters, seabed to transducer +average_salinity: 32 +average_salinity_note: 'estimated (ocean), PSU' +#skiprows: ?? #optional to speicfy how many rows to skip to start reading data records in log file diff --git a/stglib/tests/data/1151Bexo_config.yaml b/stglib/tests/data/1151Bexo_config.yaml index 54474f28..c96bffa4 100644 --- a/stglib/tests/data/1151Bexo_config.yaml +++ b/stglib/tests/data/1151Bexo_config.yaml @@ -1,3 +1,26 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:debce684b109cc3fc1c5f7ff4387a554fb645c31bf73cfedabf6b6ae7fef4fc9 -size 1217 +basefile: 'KorEXO_Measurement_File_Export_081322_170216_rev' +filename: '1151Bexo' # name of output file, -raw.cdf or .nc will be appended to this +#SN: '17H103144' +LatLonDatum: 'NAD83' +ClockError: 0 # sec, negative is slow +ClockDrift: -90 # sec, add clock drift just to test functionality in stglib +initial_instrument_height: 0.29 # meters - from bed (rtk elv) to center of sensors. +initial_instrument_height_note: 'meters, height above bed' +zeroed_pressure: 'No' # was pressure zeroed before deployment +P_1ac_note: 'Corrected for variations in atmospheric pressure using Cape May, NJ NOAA met station (8536110). Missing atmospheric pressure data at station 8536110 was filled by data from NOAA stations Brandywine Shoal Light (8555889) and Lewes, DE (8557380)' +skiprows: 8 +fDOMRFU_max_diff: 13.5 +fDOMRFU_min_diff: -13.5 +fDOMQSU_max_diff: 35.0 +fDOMQSU_min_diff: -35.0 +TALPErfu_med_diff: 20.0 +TALPE_med_diff: 50.0 +CHLrfu_med_diff: 5.0 +Fch_906_med_diff: 20.0 +C_51_min_diff: -0.75 #Conductivity m/s +C_51_max_diff: 0.75 +SpC_48_min_diff: -9.5 #SpConducitivty m/s +SpC_48_max_diff: 9.50 #SpConducitivty m/s +S_41_min_diff: -7.00 #Salinity psu +S_41_max_diff: 7.00 #Salinity psu +Turb_med_diff: 125.0 \ No newline at end of file diff --git a/stglib/tests/data/1170tcm_config.yaml b/stglib/tests/data/1170tcm_config.yaml new file mode 100644 index 00000000..e4067014 --- /dev/null +++ b/stglib/tests/data/1170tcm_config.yaml @@ -0,0 +1,13 @@ +basefile: '1711426_Bang--MBW_0' +filename: '11701tcm' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +ClockError: -18000 #sec +instrument_type: 'tcm' +initial_instrument_height: 5 # meters, from cave floor +initial_instrument_height_note: 'From cave floor' +skiprows: 1 +skipfooter: 0 +#ncols: 6 +vert_dim: 'depth' +good_dates: [2018-03-26 04:59, 2018-08-01 18:05] +#drop_vars: 'BPR_915' \ No newline at end of file diff --git a/stglib/tests/data/1711426_Bang--MBW_0.lid b/stglib/tests/data/1711426_Bang--MBW_0.lid new file mode 100644 index 00000000..db9ec047 --- /dev/null +++ b/stglib/tests/data/1711426_Bang--MBW_0.lid @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff81ab90f75b7c660d12e92e07781c58a559dca59b69190bcc1bf9834e839601 +size 71459092 diff --git a/stglib/tests/data/1711426_Bang--MBW_0_CR.txt b/stglib/tests/data/1711426_Bang--MBW_0_CR.txt new file mode 100644 index 00000000..667549b0 --- /dev/null +++ b/stglib/tests/data/1711426_Bang--MBW_0_CR.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b33b8898da03aff7c21ea6d2f2a7a6b3f99bac7f55af92cf146d80137459b7a2 +size 1815120 diff --git a/stglib/tests/data/aqd1118A_config.yaml b/stglib/tests/data/aqd1118A_config.yaml old mode 100755 new mode 100644 index 7662181b..bc46f5fb --- a/stglib/tests/data/aqd1118A_config.yaml +++ b/stglib/tests/data/aqd1118A_config.yaml @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0d9441675437967760cd16cdca46b0435ab184ba40f4d35ceefc5894bf89e885 -size 899 +basefile: 'TB111803' +filename: '1118ABaqd' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +#ClockError: 0 # sec, negative is slow +ClockError: 0 #sec +orientation: 'UP' # use this to identify orientation of profiler +initial_instrument_height: 0.13 # meters +initial_instrument_height_note: 'initial instrument height not recorded; value is from 1118C & 1118D' +zeroed_pressure: 'Yes' # was pressure zeroed before deployment +trim_method: 'water level sl' # Water Level SL trims bin if any part of bin or side lobe is out of water - works best when pressure is corrected for atmospheric +P_1ac_note: 'Corrected for variations in atmospheric pressure using NOAA met station Cape May, NJ (8536110). Missing atmospheric pressure data at station 8536110 was filled by data from NOAA stations Brandywine Shoal Light (8555889) and Lewes, DE (8557380)' diff --git a/stglib/tests/data/aqdhr1113tst_config.yaml b/stglib/tests/data/aqdhr1113tst_config.yaml index f0b69878..3eacd683 100644 --- a/stglib/tests/data/aqdhr1113tst_config.yaml +++ b/stglib/tests/data/aqdhr1113tst_config.yaml @@ -1,3 +1,21 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:45c58efd96f59d51d5baa6a87f637ca2c33fa0e3f72d14e07a66fcd9eea2d96d -size 1178 +basefile: 'CAVE0110tst' +filename: '1113aqdHR' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +ClockError: -18000 #sec changge from LST to UTC (-5 hrs) +#ClockDrift: 15 #sec < 60 s no need to apply correction +orientation: 'DOWN' # use this to identify orientation of profiler +initial_instrument_height: 9 # meters +initial_instrument_height_note: 'From cave floor' +zeroed_pressure: 'Yes' # was pressure zeroed before deployment +trim_method: 'none' # Water Level SL trims bin if any part of bin or side lobe is out of water - works best when pressure is corrected for atmospheric +velocity_agc_min: 30 +velocity_cor_min: 85 +agc_cor_min: 85 +P_1ac_note: 'Corrected for variations in atmospheric pressure using Hobo barometeric pressure data located near cave entrance.' +u_1205_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +v_1206_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +w_1204_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +vel1_1277_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +vel2_1278_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +vel3_1279_bad_ens: [2018-03-25 19:45, 2018-08-01 19:31] +vert_dim: 'depth' diff --git a/stglib/tests/data/config_1097C.yaml b/stglib/tests/data/config_1097C.yaml index 8f085294..253ef7cb 100644 --- a/stglib/tests/data/config_1097C.yaml +++ b/stglib/tests/data/config_1097C.yaml @@ -1,3 +1,7 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9a307b3051c1eae8dc84b717e5fa3a5f466da19c9f6d110757628d68020032a9 -size 385 +basefile: '1097CIQ_20171214_125948' +filename: '10971Ciq' # name of output file, -raw.cdf or .nc will be appended to this +LatLonDatum: 'NAD83' +initial_instrument_height: 0.025 +P_1ac_note: 'Corrected for variations in atmospheric pressure using internal SonTek IQ algorithm.' +DATA_CMNT: 'Data potentially affected by presence of ice during periods of this deployment' +orientation: 'up' \ No newline at end of file diff --git a/stglib/tests/data/config_205598_20220104_1336_wtw21.yaml b/stglib/tests/data/config_205598_20220104_1336_wtw21.yaml old mode 100755 new mode 100644 index 36b4bee5..5bb545fa --- a/stglib/tests/data/config_205598_20220104_1336_wtw21.yaml +++ b/stglib/tests/data/config_205598_20220104_1336_wtw21.yaml @@ -1,3 +1,7 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6493867b32e31c5e7393d015afc7d312900f815b4b1276d0d3bf65ca86e667c1 -size 712 +basefile: '205598_20220104_1336/205598_20220104_1336' +Conventions: CF-1.8 +filename: WTW21CTD +featureType: 'profile' +latitude: [nan, 37.584526, 37.584489, 37.584556, 37.584516, 37.584494, 37.584546, 37.584523, 37.584494, 37.584601, 37.58454, 37.584472, 37.584566, 37.584547, 37.584547, 37.584479, 37.584545, 37.584524, 37.584492, 37.584552, 37.584525, nan, 37.584535, 37.584535] +longitude: [nan, -122.142027, -122.142024, -122.142017, -122.142024, -122.141961, -122.141977, -122.142046, -122.142017, -122.142039, -122.141954, -122.141968, -122.141971, -122.141903, -122.141903, -122.141874, -122.141915, -122.141897, -122.141873, -122.141905, -122.141924, nan, -122.141889, -122.141942] +good_ens: [1, 20, 22, 23] diff --git a/stglib/tests/data/config_CHC14TDH.yaml b/stglib/tests/data/config_CHC14TDH.yaml index cb56cb1a..a0c34845 100644 --- a/stglib/tests/data/config_CHC14TDH.yaml +++ b/stglib/tests/data/config_CHC14TDH.yaml @@ -1,3 +1,4 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ae24afa6bb4c6c7840f41b7e6a74823579c35a3a638b527990c7a587ac9d8927 -size 92 +basefile: 'TDHAQD02' +initial_instrument_height: 0.15 +filename: 'CHC14TDH' +orientation: 'UP' diff --git a/stglib/tests/data/config_bel3C.yaml b/stglib/tests/data/config_bel3C.yaml index b52672d9..4ffba855 100644 --- a/stglib/tests/data/config_bel3C.yaml +++ b/stglib/tests/data/config_bel3C.yaml @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c9a1e8662e97c443858d3fedc9cf4f0149ab2e159ab4fbf023632f042f3a68e9 -size 517 +basefile: 'bel3_13H102609_012221_120000_utf8' +# filename: 'bel33Cexo' +filename: 'BEL19B3C03exo' +Conventions: 'CF-1.8' +skiprows: 9 +initial_instrument_height: 0.15 +S_41_min_diff: -10 +Turb_max: 300 +Turb_bad_ens: ['2021-04-10 21:10', '2021-04-10 21:20'] +drop_vars: ['Cable_Pwr_V', 'Wiper_Position_volt', 'Vertical_Position_m', 'ODO_%_local', 'pHmV', 'pH_159', 'nLF_Cond_µS_per_cm', 'OST_62', 'DO'] +title: 'Oceanographic time-series measurements collected in Bellingham Bay, Washington, USA, 2019 to 2021: BEL19B3C03exo' diff --git a/stglib/tests/data/config_bel5C.yaml b/stglib/tests/data/config_bel5C.yaml index 6bd0c63d..330020d9 100644 --- a/stglib/tests/data/config_bel5C.yaml +++ b/stglib/tests/data/config_bel5C.yaml @@ -1,3 +1,8 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7f81ccdf925dbc1395975808484708d24f414d4360b87c2877086a18b3e4221 -size 180 +basefile: 'bel5_19D105366_012221_120000_utf8' +filename: 'bel53Cexo' +skiprows: 9 +initial_instrument_height: 0.15 +S_41_min_diff: -10 +Turb_max_diff: 500 +ClockError: 10 +ClockDrift: 45 diff --git a/stglib/tests/data/config_nbm22cce01.yaml b/stglib/tests/data/config_nbm22cce01.yaml old mode 100755 new mode 100644 index bf46ab50..e2a4be53 --- a/stglib/tests/data/config_nbm22cce01.yaml +++ b/stglib/tests/data/config_nbm22cce01.yaml @@ -1,3 +1,14 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d437f6b561b5acabf5ea02ce4bcab7d90d1c03351e074450e17766170d429fc9 -size 432 +basefile: 'NBMCCE02' +filename: 'NBM22CCEaqd' +initial_instrument_height: 0.095 +orientation: 'UP' +Conventions: 'CF-1.8' +Deployment_date: '2022-12-06 00:24' +Recovery_date: '2023-02-01 22:40' # estimated +trim_method: 'water level sl' +AnalogInput1: 'true' +AnalogInput1_long_name: 'turbidity' +AnalogInput1_units: 'V' +AnalogInput1_min: 0.01 +height_above_geopotential_datum: 0.631 # grate elevation in m +geopotential_datum_name: 'NAVD88' diff --git a/stglib/tests/data/gatts_205598_20220104_1336_wtw21.txt b/stglib/tests/data/gatts_205598_20220104_1336_wtw21.txt old mode 100755 new mode 100644 diff --git a/stglib/tests/data/gatts_CHC14TDH.txt b/stglib/tests/data/gatts_CHC14TDH.txt old mode 100755 new mode 100644 index 8824d797..02016409 --- a/stglib/tests/data/gatts_CHC14TDH.txt +++ b/stglib/tests/data/gatts_CHC14TDH.txt @@ -1,3 +1,29 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ae8e6c87627a5f3367df962884ff8f6f0a4a9a77150d3e2bda66dea90822cb96 -size 712 +MOORING; CHC14TDH +PROGRAM; Sediment Transport in Coastal Environments +PROJECT; Sediment supply to marshes, San Pablo Bay +EXPERIMENT; CHC14 +Region; San Francisco Bay +CruiseID; 2014-678-FA +SciPi; Jessica Lacy +Year; 2014 +Site; TDH +platform_type; Tripod +latitude; 38.00922 +longitude; -122.48609 +WATER_DEPTH; 0 +Deployment_date; 05-Dec-2014 00:12 +Recovery_date; 28-Jan-2015 20:55 +OFAFunding; +CollaboratingAgency; San Francisco Bay NERR +DESCRIPTION; Grate +WATER_DEPTH_SOURCE; Bathymetric map +magnetic_variation; 13.86 +DATA_SUBTYPE; MOORED +DATA_ORIGIN; USGS WCMG Sed Trans Group +COORD_SYSTEM; GEOGRAPHIC +Conventions; CF-1.8 +WATER_MASS; ? +POS_CONST; 0 +DEPTH_CONST; 0 +DRIFTER; 0 +COMPOSITE; 0 diff --git a/stglib/tests/data/glob_att1097C.txt b/stglib/tests/data/glob_att1097C.txt index aa96301b..160175c7 100644 --- a/stglib/tests/data/glob_att1097C.txt +++ b/stglib/tests/data/glob_att1097C.txt @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ec085aa9d6978694336bdb19f00db83716c224cd3fb3a138290060858fe16430 -size 902 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NRPP + DESCRIPTION; JoCo Marsh, Jamaica Bay + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Sed Trans Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1097 + WATER_DEPTH; 1.01 + WATER_DEPTH_NOTE; (meters), nominal depth + latitude; 40.61263 + longitude; -73.79092 + magnetic_variation; -13.46 + Deployment_date; 2017-12-14 13:00 + Recovery_date; 2018-01-24 16:07 + DATA_CMNT; + platform_type; FG Lander + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Jamaica Bay, NY + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov + NAVD88_ref; -1.0221 + NAVD88_ref_note; Elevation (m NAVD88) measured at top of instrument mounting grate \ No newline at end of file diff --git a/stglib/tests/data/glob_att1103D.txt b/stglib/tests/data/glob_att1103D.txt index d9fd9867..3dae7f62 100644 --- a/stglib/tests/data/glob_att1103D.txt +++ b/stglib/tests/data/glob_att1103D.txt @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7128e6dddc50fb6c70f870d7a0a853d60eda57258f3cfca015b7ae8f1b1fb695 -size 958 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NRPP + DESCRIPTION; Ditch Near Site, Great South Bay + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Sed Trans Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.8 + MOORING; 1103 + WATER_DEPTH; 0.40 + WATER_DEPTH_NOTE; (meters), nominal depth + latitude; 40.731563 + longitude; -72.886313 + magnetic_variation; -13.46 + Deployment_date; 2018-03-28 19:40 + Recovery_date; 2018-06-27 17:34 + DATA_CMNT; Instrument was often dry during low tide + platform_type; FG Lander + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Great South Bay, NY + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov + NAVD88_ref; -0.2287 + NAVD88_ref_note; Elevation (m NAVD88) measured at top of instrument mounting grate diff --git a/stglib/tests/data/glob_att1113_aqdHR_tst.txt b/stglib/tests/data/glob_att1113_aqdHR_tst.txt index 286fb7ee..768fbb0a 100644 --- a/stglib/tests/data/glob_att1113_aqdHR_tst.txt +++ b/stglib/tests/data/glob_att1113_aqdHR_tst.txt @@ -1,3 +1,27 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:577303762664f1dbe540bbade5429a021c7924d912d93b8b4620a8e7d872d1a5 -size 1229 + title; Time-series measurements from Aquadopp high-resolution ADCP profiler in Cenote Bang, Yucatan, Mexico from March 25, 2018 to August 1, 2018 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Hazards and Resources Program + EXPERIMENT; CAVE MX + DESCRIPTION; Cenote Bang,Flooded cave, dome S1 + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.8 + MOORING; 1113 + WATER_DEPTH; 12.5 + WATER_DEPTH_NOTE; (meters), from cave floor to water table above (estimated) + latitude;20.21033° + longitude; -87.50127° + magnetic_variation; -1.39 + Deployment_date; 2018-03-25 19:45 + Recovery_date; 2018-08-01 19:31 + DATA_CMNT; All velocity data was filled with null value (NaN) because of suspected interference of the acoustic signals with the cave walls. + platform_type; bouyant FG platform + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Cenote Bang, Tulum, MX + VAR_FILL; NaN + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov + FAN; 2018-XXX-FA, 2018-XXX-FA \ No newline at end of file diff --git a/stglib/tests/data/glob_att1118a_b.txt b/stglib/tests/data/glob_att1118a_b.txt old mode 100755 new mode 100644 index 5fc2e52a..ac8ada19 --- a/stglib/tests/data/glob_att1118a_b.txt +++ b/stglib/tests/data/glob_att1118a_b.txt @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0a48d4eaa73b00c67aa29eab076da282ed9b450e085451c0e7c188746e821d01 -size 983 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NFWF - Hurricane Sandy Coastal Resiliency Program: Ecological + DESCRIPTION; Thompsons Beach, New Jersey + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1118A and 1118B + WATER_DEPTH; 2.4 + WATER_DEPTH_NOTE; (meters), nominal depth + latitude; 39.19514 + longitude; -74.98397 + magnetic_variation; -11.98 + Deployment_date; 2018-09-19 15:00 + Recovery_date; 2019-02-26 18:40 + DATA_CMNT; No magnetic variation recorded, used data from IGRF model via ngdc.noaa.gov. + platform_type; FG Lander + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Delaware Bay + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1119a.txt b/stglib/tests/data/glob_att1119a.txt index f4088c4a..3bbbb876 100644 --- a/stglib/tests/data/glob_att1119a.txt +++ b/stglib/tests/data/glob_att1119a.txt @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ecd2c38e531f641a52dc54eb3e6650cd20bd846e9f8f06ad922b902ff33e62ed -size 970 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NFWF - Hurricane Sandy Coastal Resiliency Program: Ecological + DESCRIPTION; Thompsons Beach New Jersey + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1119A + WATER_DEPTH; 3.0 + WATER_DEPTH_NOTE; (meters), nominal depth + latitude; 39.19533 + longitude; 74.98411 + magnetic_variation; 11.98 + Deployment_date; 2018-09-19 14:24 + Recovery_date; 2018-12-05 14:36 + DATA_CMNT; No magnetic variation recorded, used data from IGRF model via ngdc.noaa.gov. + platform_type; FG Lander + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Delaware Bay + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1123A_msl.txt b/stglib/tests/data/glob_att1123A_msl.txt index c3551f01..36a9b1ba 100644 --- a/stglib/tests/data/glob_att1123A_msl.txt +++ b/stglib/tests/data/glob_att1123A_msl.txt @@ -1,3 +1,29 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1bd7c283287942ddf5bd8cda920b6947ff68e4054b4bd5b161be1989e329914e -size 1199 + title; Oceanographic and Water Quality Measurements in the Herring River, Wellfleet, MA at deep flank site from a EofE EA400 Atlimeter from November 11, 2018 to January 16, 2019 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NRPP + DESCRIPTION; Wellfleet Herring River, Gut Flank Deep + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Sed Trans Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1123A + sea_floor_depth_below_mean_sea_level; 1.25 + WATER_DEPTH; 1.25 + WATER_DEPTH_NOTE; (meters), Mean Sea Level estimated from NAVD88 and vDatum + height_above_geopotential_datum; -1.40 + geopotential_datum_name; NAVD88 + latitude; 41.92636 + longitude; -70.06882 + magnetic_variation; -14.72 + Deployment_date; 2018-11-14 15:00:00 + Recovery_date; 2019-01-16 17:28:00 + DATA_CMNT; + platform_type; Spyder Frame + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Herring River, MA + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1123A_msl_EPIC.txt b/stglib/tests/data/glob_att1123A_msl_EPIC.txt index ce7cfe39..670553b4 100644 --- a/stglib/tests/data/glob_att1123A_msl_EPIC.txt +++ b/stglib/tests/data/glob_att1123A_msl_EPIC.txt @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e5c748f30e231786c41a0cbf934cedaa58cb9507f081700cd17f8c217a447f4 -size 1148 + title; Oceanographic and Water Quality Measurements in the Herring River, Wellfleet, MA at deep flank site from a EofE EA400 Atlimeter from November 11, 2018 to January 16, 2019 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NRPP + DESCRIPTION; Wellfleet Herring River, Gut Flank Deep + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Sed Trans Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; PMEL/EPIC + MOORING; 1123A + WATER_DEPTH; 1.25 + WATER_DEPTH_NOTE; (meters), Mean Sea Level estimated from NAVD88 and vDatum + height_above_geopotential_datum; -1.40 + geopotential_datum_name; NAVD88 + latitude; 41.92636 + longitude; -70.06882 + magnetic_variation; -14.72 + Deployment_date; 2018-11-14 15:00:00 + Recovery_date; 2019-01-16 17:28:00 + DATA_CMNT; + platform_type; Spyder Frame + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Herring River, MA + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1137.txt b/stglib/tests/data/glob_att1137.txt index 4826c222..49e98bf0 100644 --- a/stglib/tests/data/glob_att1137.txt +++ b/stglib/tests/data/glob_att1137.txt @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4300fc213c5a329145ca5995c6401be46b4b35e06cbc97cd5ed6a5dcb057ebe4 -size 992 + SciPi; C. Sherwood + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; DUNEX + DESCRIPTION; Cape Hatteras National Seashore, Pea Island, NC + DATA_SUBTYPE; POLE-MOUNTED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1137 + WATER_DEPTH; 5.4 + WATER_DEPTH_NOTE; (meters), Mean water level from data + latitude; 35.677948 + longitude; -75.476772 + magnetic_variation; -11.05 + Deployment_date; 2021-09-14 19:01 + Recovery_date; 2021-09-28 17:15 + DATA_CMNT; Magnetic declination determined using https://www.ngdc.noaa.gov/geomag/declination.shtml IGRF model + platform_type; POLE + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Atlantic Ocean, Pea Island, NC + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1149.txt b/stglib/tests/data/glob_att1149.txt index cc0ebf12..fd5ea79a 100644 --- a/stglib/tests/data/glob_att1149.txt +++ b/stglib/tests/data/glob_att1149.txt @@ -1,3 +1,25 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6953b8739e2412ddc3d23b986e0cad68a0324bc4247969fc78192eef69f68d32 -size 941 + SciPi; C. Sherwood + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; DUNEX + DESCRIPTION; Pea Island National Wildlife Refuge, NC + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1149 + WATER_DEPTH; 0 + WATER_DEPTH_NOTE; Sensor is located above ground. + latitude; 35.677777 + longitude; -75.48042 + magnetic_variation; -11.04 + Deployment_date; 2021-09-13 14:00 + Recovery_date; 2021-10-24 19:51 + DATA_CMNT; No magnetic variation recorded, used data from IGRF model via ngdc.noaa.gov. + platform_type; pole + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; North Atlantic Ocean + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1151b.txt b/stglib/tests/data/glob_att1151b.txt index 65109679..3b9af43f 100644 --- a/stglib/tests/data/glob_att1151b.txt +++ b/stglib/tests/data/glob_att1151b.txt @@ -1,3 +1,28 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:50d32b3c951e92cd1f8aaafdd73148760feb97e5cc0f11962b11220c8a02f943 -size 1110 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Geology Program + EXPERIMENT; NFWF - Hurricane Sandy Coastal Resiliency Program: Ecological + DESCRIPTION; Stone Harbor, New Jersey + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.6 + MOORING; 1151B + WATER_DEPTH; 1.3 + WATER_DEPTH_NOTE; (meters), mean water depth from data (1.011 avg + .29m from bed to exo sensors). + height_above_geopotential_datum; -1.3481 + geopotential_datum_name; NAVD88 + geopotential_datum_note; Elevation at seabed + latitude; 39.03162 + longitude; -74.78278 + magnetic_variation; -12.05 + Deployment_date; 2022-06-08 11:55 + Recovery_date; 2022-08-12 19:30 + DATA_CMNT; + platform_type; Grating Platform + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Great Channel/Atlantic Ocean + VAR_FILL; 1.e+35 + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov diff --git a/stglib/tests/data/glob_att1170_tcm.txt b/stglib/tests/data/glob_att1170_tcm.txt new file mode 100644 index 00000000..8111d207 --- /dev/null +++ b/stglib/tests/data/glob_att1170_tcm.txt @@ -0,0 +1,28 @@ + title; Time-series measurements from Hobo water level sensor in Cenote Bang, Yucatan, Mexico from March 25, 2018 to August 1, 2018 + SciPi; N. Ganju + PROJECT; USGS Coastal and Marine Hazards and Resources Program + EXPERIMENT; CAVE MX + DESCRIPTION; Cenote Bang,Flooded cave, S-1, set-3 + DATA_SUBTYPE; MOORED + DATA_ORIGIN; USGS WHCMSC Coastal and Estuarine Dynamics Group + COORD_SYSTEM; GEOGRAPHIC + Conventions; CF-1.8 + MOORING; 1168 + sea_floor_depth_below_geoid; 22 + WATER_DEPTH; 22 + WATER_DEPTH_NOTE; (meters), from cave floor to water table above (diver estimated) + latitude; 20.21072 + longitude; -87.50205 + magnetic_variation; -1.39 + Deployment_date; 2018-03-25 18:57 + Recovery_date; 2018-08-01 19:05 + DATA_CMNT; + platform_type; cave ledge + DRIFTER; 0 + POS_CONST; 0 + DEPTH_CONST; 0 + WATER_MASS; Cenote Bang, Tulum, MX + VAR_FILL; NaN + institution; United States Geological Survey, Woods Hole Coastal and Marine Science Center + institution_url; https://woodshole.er.usgs.gov + FAN; 2018-031-FA \ No newline at end of file diff --git a/stglib/tests/data/glob_attbel3C.txt b/stglib/tests/data/glob_attbel3C.txt index 6ed491a4..e892da84 100644 --- a/stglib/tests/data/glob_attbel3C.txt +++ b/stglib/tests/data/glob_attbel3C.txt @@ -1,3 +1,7 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0faafff455d80e8337c3e1db83c0aa7fd2b7b1528e1ec3dae40bd86a71809902 -size 169 +WATER_DEPTH; 6.9 +latitude; 48.75675 +longitude; -122.52279 +Deployment_date; 2021-01-22 18:27 +Recovery_date; 2021-04-13 19:27 +magnetic_variation; 15.95 +MOORING; BEL19B3C diff --git a/stglib/tests/data/glob_attbel5C.txt b/stglib/tests/data/glob_attbel5C.txt index 6fe73179..3ca7e2f8 100644 --- a/stglib/tests/data/glob_attbel5C.txt +++ b/stglib/tests/data/glob_attbel5C.txt @@ -1,3 +1,7 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0c80c769face44e18cb3a7a0ee86382db47dbe115d91eec7b3bdc0e0cd168e75 -size 171 +WATER_DEPTH; 6.3 +latitude; 48.750293 +longitude; -122.575145 +Deployment_date; 2021-01-22 19:17 +Recovery_date; 2021-04-13 21:38 +magnetic_variation; 15.95 +Conventions; CF-1.6 \ No newline at end of file diff --git a/stglib/tests/data/nbm22cce01_gatts.txt b/stglib/tests/data/nbm22cce01_gatts.txt old mode 100755 new mode 100644 index ec92bd8b..bb5fb9d4 --- a/stglib/tests/data/nbm22cce01_gatts.txt +++ b/stglib/tests/data/nbm22cce01_gatts.txt @@ -1,3 +1,4 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7db02084a3f80609c256d124b43e16e17efe9cd791f986eb946f8fafe7cab783 -size 78 +WATER_DEPTH; 0.5 +latitude; 37.930319 +longitude; -122.505512 +MOORING; NBM22CCE diff --git a/stglib/tests/data/wxt1149_config.yaml b/stglib/tests/data/wxt1149_config.yaml index 4dcbb403..93ecfe21 100644 --- a/stglib/tests/data/wxt1149_config.yaml +++ b/stglib/tests/data/wxt1149_config.yaml @@ -1,3 +1,9 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a23d702f25c33afed9b47924bcc932d129645856bd244dcb30bcaa66e4e8d61d -size 271 +basefile: 'Dunex_2021-09-13' +filename: '1149wxt' +LatLonDatum: NAD83 +skiprows: 7 +RTK_elevation_NAVD88: 9.2241 +initial_instrument_height: 2.447 +initial_instrument_height_note: sensor above ground +dir_offset: 0 +dir_offset_note: 'No wind direction correction needed.' \ No newline at end of file diff --git a/stglib/tests/test_scripts.py b/stglib/tests/test_scripts.py index 9a0a68ab..54c09e9d 100644 --- a/stglib/tests/test_scripts.py +++ b/stglib/tests/test_scripts.py @@ -347,6 +347,29 @@ def test_lisst(): lisst_nc("lisst_L0221705-raw.cdf") +def tcm_raw(glob_att, config_yaml): + result = subprocess.run( + [scripts / "runtcmcsv2cdf.py", glob_att, config_yaml], + capture_output=True, + cwd=cwd, + ) + assert "Finished writing data" in result.stdout.decode("utf8") + + +def tcm_nc(nc_file): + result = subprocess.run( + [scripts / "runtcmcdf2nc.py", nc_file], + capture_output=True, + cwd=cwd, + ) + assert "Done writing netCDF file" in result.stdout.decode("utf8") + + +def test_tcm(): + tcm_raw("glob_att1170_tcm.txt", "1170tcm_config.yaml") + tcm_nc("11701tcm-raw.cdf") + + def ensure_cf(script, glob_att, config_yaml): result = subprocess.run( [scripts / script, glob_att, config_yaml],