From 70140fa92598936710012ef2b54f97ab1966f3e4 Mon Sep 17 00:00:00 2001 From: Pietro Stradiotti Date: Wed, 15 Dec 2021 16:51:30 +0100 Subject: [PATCH 1/4] Workaround error when TCA results are not produced --- src/pytesmo/validation_framework/validation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pytesmo/validation_framework/validation.py b/src/pytesmo/validation_framework/validation.py index cc2e01e3..3df09c24 100644 --- a/src/pytesmo/validation_framework/validation.py +++ b/src/pytesmo/validation_framework/validation.py @@ -265,7 +265,14 @@ def calc( for field_name in results[key][0].keys(): entries = [] for result in results[key]: - entries.append(result[field_name][0]) + try: + entries.append(result[field_name][0]) + except KeyError: + # indicating a metric result was not produced + warnings.warn( + f"No TCA results for gpi: {result['gpi']}" + ) + entries.append(np.nan) compact_results[key][field_name] = np.array( entries, dtype=results[key][0][field_name].dtype ) From be3dc11b1dcfc27d69a2501cfb84493676bd343d Mon Sep 17 00:00:00 2001 From: Pietro Stradiotti Date: Thu, 16 Dec 2021 12:04:58 +0100 Subject: [PATCH 2/4] imporved warning message --- src/pytesmo/validation_framework/validation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pytesmo/validation_framework/validation.py b/src/pytesmo/validation_framework/validation.py index 3df09c24..110dd4fd 100644 --- a/src/pytesmo/validation_framework/validation.py +++ b/src/pytesmo/validation_framework/validation.py @@ -267,10 +267,11 @@ def calc( for result in results[key]: try: entries.append(result[field_name][0]) - except KeyError: + except KeyError as e: # indicating a metric result was not produced warnings.warn( - f"No TCA results for gpi: {result['gpi']}" + f"No TCA results for gpi: {result['gpi']}. Error " + f"message: {e}" ) entries.append(np.nan) compact_results[key][field_name] = np.array( From a5edd0bdfc94d41b9b04f77cb0df4ca733888e67 Mon Sep 17 00:00:00 2001 From: Pietro Stradiotti Date: Thu, 16 Dec 2021 13:38:36 +0100 Subject: [PATCH 3/4] Handled TCA error upstream --- .../validation_framework/metric_calculators.py | 11 ++++++----- src/pytesmo/validation_framework/validation.py | 10 +--------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/pytesmo/validation_framework/metric_calculators.py b/src/pytesmo/validation_framework/metric_calculators.py index 34431fd7..2411e057 100644 --- a/src/pytesmo/validation_framework/metric_calculators.py +++ b/src/pytesmo/validation_framework/metric_calculators.py @@ -1717,17 +1717,18 @@ def calc_metrics(self, data, gpi_info): result = super().calc_metrics(data, gpi_info) n_obs = len(data) result["n_obs"][0] = n_obs - if n_obs < self.min_obs: - warnings.warn( - "Not enough observations to calculate metrics.", UserWarning - ) - return result # get the remaining metrics template for this specific combination othernames = list(data.columns) othernames.remove(self.refname) result.update(self._get_metric_template(self.refname, othernames)) + if n_obs < self.min_obs: + warnings.warn( + "Not enough observations to calculate metrics.", UserWarning + ) + return result + # calculate triple collocation metrics ds_names = (self.refname, *othernames) arrays = (data[name].values for name in ds_names) diff --git a/src/pytesmo/validation_framework/validation.py b/src/pytesmo/validation_framework/validation.py index 110dd4fd..cc2e01e3 100644 --- a/src/pytesmo/validation_framework/validation.py +++ b/src/pytesmo/validation_framework/validation.py @@ -265,15 +265,7 @@ def calc( for field_name in results[key][0].keys(): entries = [] for result in results[key]: - try: - entries.append(result[field_name][0]) - except KeyError as e: - # indicating a metric result was not produced - warnings.warn( - f"No TCA results for gpi: {result['gpi']}. Error " - f"message: {e}" - ) - entries.append(np.nan) + entries.append(result[field_name][0]) compact_results[key][field_name] = np.array( entries, dtype=results[key][0][field_name].dtype ) From f481c3cbafc37b299d0227d815a7a24e3cf44792 Mon Sep 17 00:00:00 2001 From: Pietro Stradiotti Date: Thu, 16 Dec 2021 13:46:29 +0100 Subject: [PATCH 4/4] Handled TCA CI bootstrapping error --- .../validation_framework/metric_calculators.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/pytesmo/validation_framework/metric_calculators.py b/src/pytesmo/validation_framework/metric_calculators.py index 2411e057..9ac04d8f 100644 --- a/src/pytesmo/validation_framework/metric_calculators.py +++ b/src/pytesmo/validation_framework/metric_calculators.py @@ -1738,10 +1738,16 @@ def calc_metrics(self, data, gpi_info): for j, metric in enumerate(["snr", "err_std", "beta"]): result[(metric, name)][0] = res[j][i] else: - res = tcol_metrics_with_bootstrapped_ci(*arrays) - for i, name in enumerate(ds_names): - for j, metric in enumerate(["snr", "err_std", "beta"]): - result[(metric, name)][0] = res[j][0][i] - result[(metric + "_ci_lower", name)][0] = res[j][1][i] - result[(metric + "_ci_upper", name)][0] = res[j][2][i] + try: + # handle failing bootstrapping because e.g. + # too small sample size + res = tcol_metrics_with_bootstrapped_ci(*arrays) + for i, name in enumerate(ds_names): + for j, metric in enumerate(["snr", "err_std", "beta"]): + result[(metric, name)][0] = res[j][0][i] + result[(metric + "_ci_lower", name)][0] = res[j][1][i] + result[(metric + "_ci_upper", name)][0] = res[j][2][i] + except ValueError: + # if the calculation fails, the template results (np.nan) are used + pass return result