Skip to content

Commit

Permalink
Changed several errors and imports
Browse files Browse the repository at this point in the history
  • Loading branch information
@dwest77a
dwest77a committed Jan 10, 2025
1 parent 437ec1d commit 14f8ef1
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 216 deletions.
222 changes: 11 additions & 211 deletions padocc/core/errors.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ def __init__(self, proj_code: Union[str,None], groupdir: Union[str,None]) -> Non
msg = getattr(self,'message')
super().__init__(msg)

class PartialDriverError(KerchunkException):
class PartialDriverError(KerchunkException): # Keep
"""All drivers failed (NetCDF3/Hdf5/Tiff) for one or more files within the list"""
def __init__(
self,
Expand All @@ -80,39 +80,8 @@ def __init__(
def get_str(self):
return 'PartialDriverError'

class NaNComparisonError(KerchunkException):
"""When comparing NaN values between objects - different values found"""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = f"NaN values do not match between comparison objects"
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'NaNComparisonError'

class RemoteProtocolError(KerchunkException):
"""All drivers failed (NetCDF3/Hdf5/Tiff) for one or more files within the list"""
def __init__(
self,
filenums: Union[int,None] = None,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:

self.message = f"All drivers failed when performing conversion for files {filenums}"
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'PartialDriverError'

class KerchunkDriverFatalError(KerchunkException):
class KerchunkDriverFatalError(KerchunkException): # Keep
"""All drivers failed (NetCDF3/Hdf5/Tiff) - run without driver bypass to assess the issue with each driver type."""
def __init__(
self,
Expand All @@ -127,55 +96,7 @@ def __init__(
def get_str(self):
return 'KerchunkDriverFatalError'

class IdenticalVariablesError(KerchunkException):
"""All variables found to be suitably identical between files as to not stack or concatenate"""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = "All variables are identical across files"
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'IdenticalVariablesError'

class XKShapeToleranceError(KerchunkException):
"""Attempted validation using a tolerance for shape mismatch on concat-dims, shape difference exceeds tolerance allowance."""
def __init__(
self,
tolerance: int = 0,
diff: int = 0,
dim: str = '',
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = f"Shape difference ({diff}) exceeds allowed tolerance ({tolerance}) for dimension ({dim})"
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'XKShapeToleranceError'

class BlacklistProjectCode(KerchunkException):
"""The project code you are trying to run for is on the list of project codes to ignore."""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = 'Project Code listed in blacklist for bad data - will not be processed.'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'BlacklistProjectCode'

class MissingVariableError(KerchunkException):
class MissingVariableError(KerchunkException): # Keep
"""A variable is missing from the environment or set of arguments."""
def __init__(
self,
Expand Down Expand Up @@ -225,38 +146,7 @@ def __init__(
def get_str(self):
return 'ExpectTimeoutError'

class ProjectCodeError(KerchunkException):
"""Could not find the correct project code from the list of project codes for this run."""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = f'Project Code Extraction Failed'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'ProjectCodeError'

class FilecapExceededError(KerchunkException):
"""During scanning, could not find suitable files within the set of files specified."""
def __init__(
self,
nfiles: int = 0,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = f'Filecap exceeded: {nfiles} files attempted'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'FilecapExceededError'

class ChunkDataError(KerchunkException):
class ChunkDataError(KerchunkException): # Keep
"""Overflow Error from pandas during decoding of chunk information, most likely caused by bad data retrieval."""
def __init__(
self,
Expand Down Expand Up @@ -287,64 +177,6 @@ def __init__(
def get_str(self):
return 'NoValidTimeSlicesError'

class VariableMismatchError(KerchunkException):
"""During testing, variables present in the NetCDF file are not present in Kerchunk"""
def __init__(
self,
missing: Union[dict, None] = None,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
missing = missing or {}

self.message = f'Missing variables {missing} in Kerchunk file'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'VariableMismatchError'

class ShapeMismatchError(KerchunkException):
"""Shapes of ND arrays do not match between Kerchunk and Xarray objects - when using a subset of the Netcdf files."""
def __init__(
self,
var: Union[dict,None] = None,
first: Union[dict,None] = None,
second: Union[dict,None] = None,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:

var = var or {}
first = first or {}
second = second or {}

self.message = f'Kerchunk/NetCDF mismatch for variable {var} with shapes - K {first} vs X {second}'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'ShapeMismatchError'

class TrueShapeValidationError(KerchunkException):
"""Shapes of ND arrays do not match between Kerchunk and Xarray objects - when using the complete set of files."""
def __init__(
self,
message: str = 'kerchunk',
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:

self.message = f'{message} mismatch with shapes using full dataset - check logs'
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'TrueShapeValidationError'

class NoOverwriteError(KerchunkException):
"""Output file already exists and the process does not have forceful overwrite (-f) set."""
def __init__(
Expand All @@ -361,7 +193,7 @@ def __init__(
def get_str(self):
return 'NoOverwriteError'

class MissingKerchunkError(KerchunkException):
class MissingKerchunkError(KerchunkException): # Keep
"""Kerchunk file not found."""
def __init__(
self,
Expand All @@ -384,14 +216,14 @@ def __init__(
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:
self.message = "Fatal Validation Error"
self.message = "Fatal Validation Error - see data report."
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'ValidationError'

class ComputeError(KerchunkException):
class ComputeError(KerchunkException): # Keep
"""Compute stage failed - likely due to invalid config/use of the classes"""
def __init__(
self,
Expand All @@ -405,22 +237,6 @@ def __init__(
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'ComputeError'

class SoftfailBypassError(KerchunkException):
"""Validation could not be completed because some arrays only contained NaN values which cannot be compared."""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:

self.message = "Kerchunk validation failed softly with no bypass - rerun with bypass flag"
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'SoftfailBypassError'

class ConcatenationError(KerchunkException):
"""Variables could not be concatenated over time and are not duplicates - no known solution"""
Expand All @@ -438,7 +254,7 @@ def __init__(
def get_str(self):
return 'ConcatenationError'

class ConcatFatalError(KerchunkException):
class ConcatFatalError(KerchunkException): # Keep
"""Chunk sizes differ between refs - files cannot be concatenated"""
def __init__(
self,
Expand All @@ -457,7 +273,7 @@ def __init__(
def get_str(self):
return 'ConcatFatalError'

class SourceNotFoundError(KerchunkException):
class SourceNotFoundError(KerchunkException): # Keep
"""Source File could not be located."""
def __init__(
self,
Expand Down Expand Up @@ -491,7 +307,7 @@ def __init__(
def get_str(self):
return 'ArchiveConnectError'

class KerchunkDecodeError(KerchunkException):
class KerchunkDecodeError(KerchunkException): # Keep
"""Decoding of Kerchunk file failed - likely a time array issue."""
def __init__(
self,
Expand All @@ -505,20 +321,4 @@ def __init__(
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'KerchunkDecodeError'

class FullsetRequiredError(KerchunkException):
"""This project must be validated using the full set of files."""
def __init__(
self,
verbose: int = 0,
proj_code: Union[str,None] = None,
groupdir: Union[str,None] = None
) -> None:

self.message = f"This project must be validated by opening the full set of files."
super().__init__(proj_code, groupdir)
if verbose < 1:
self.__class__.__module__ = 'builtins'
def get_str(self):
return 'FullsetRequiredError'
return 'KerchunkDecodeError'
1 change: 0 additions & 1 deletion padocc/core/project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
CSVFileHandler,
ListFileHandler,
LogFileHandler,
KerchunkFile
)


Expand Down
3 changes: 1 addition & 2 deletions padocc/groups/mixins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
from typing import Union

from padocc.core import (
FalseLogger,
LoggedOperation
FalseLogger
)
from padocc.core.utils import extract_file, times, apply_substitutions, file_configs

Expand Down
1 change: 0 additions & 1 deletion padocc/phases/scan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

from padocc.core import FalseLogger
from padocc.core.errors import ConcatFatalError
from padocc.core.utils import BypassSwitch
from padocc.core.filehandlers import JSONFileHandler

from .compute import KerchunkDS, cfa_handler
Expand Down
7 changes: 6 additions & 1 deletion padocc/phases/validate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
from padocc.core import ProjectOperation
from padocc.core import LoggedOperation
from padocc.core import BypassSwitch
from padocc.core.utils import open_kerchunk

from padocc.core.errors import ValidationError

from padocc.core.filehandlers import JSONFileHandler
from padocc.core.utils import format_tuple
Expand Down Expand Up @@ -895,6 +896,10 @@ def _run(
vd.save_report()

self.update_status('validate',vd.pass_fail,jobid=self._logid)

if vd.pass_fail == 'Fatal':
raise ValidationError

return vd.pass_fail

def _open_sample(self):
Expand Down

0 comments on commit 14f8ef1

Please sign in to comment.