diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6983eea1e9e..1180a5fce9a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -76,15 +76,14 @@ jobs: timeout-minutes: 2 steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.10' - name: Runs pre-commit run: | pip install pre-commit - pre-commit run -a # Runs unit testing under different python versions. @@ -98,11 +97,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} strategy: + fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ["3.8", "3.10", "3.12"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run tests shell: bash env: diff --git a/CIME/compare_namelists.py b/CIME/compare_namelists.py index 1e86199c045..4e1555984f1 100644 --- a/CIME/compare_namelists.py +++ b/CIME/compare_namelists.py @@ -1,6 +1,4 @@ import os, re, logging - -from collections import OrderedDict from CIME.utils import expect, CIMEError logger = logging.getLogger(__name__) @@ -73,11 +71,11 @@ def _interpret_value(value_str, filename): >>> _interpret_value("3*1.0", "foo") ['1.0', '1.0', '1.0'] >>> _interpret_value("'DMS -> value.nc'", "foo") - OrderedDict([('DMS', 'value.nc')]) + {'DMS': 'value.nc'} >>> _interpret_value("'DMS -> 1.0 * value.nc'", "foo") - OrderedDict([('DMS', '1.0*value.nc')]) + {'DMS': '1.0*value.nc'} >>> _interpret_value("'DMS -> 1.0* value.nc'", "foo") - OrderedDict([('DMS', '1.0*value.nc')]) + {'DMS': '1.0*value.nc'} """ comma_re = re.compile(r"\s*,\s*") dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+|(?:\S+\s*\*\s*\S+))\s*'") @@ -87,7 +85,7 @@ def _interpret_value(value_str, filename): tokens = [item.strip() for item in comma_re.split(value_str) if item.strip() != ""] if "->" in value_str: # dict - rv = OrderedDict() + rv = {} for token in tokens: m = dict_re.match(token) expect( @@ -151,7 +149,7 @@ def _parse_namelists(namelist_lines, filename): ... / ... ''' >>> _parse_namelists(teststr.splitlines(), 'foo') - OrderedDict([('nml', OrderedDict([('val', "'foo'"), ('aval', ["'one'", "'two'", "'three'"]), ('maval', ["'one'", "'two'", "'three'", "'four'"]), ('dval', OrderedDict([('one', 'two'), ('three', 'four')])), ('mdval', OrderedDict([('one', 'two'), ('three', 'four'), ('five', 'six')])), ('nval', '1850')])), ('nml2', OrderedDict([('val2', '.false.')]))]) + {'nml': {'val': "'foo'", 'aval': ["'one'", "'two'", "'three'"], 'maval': ["'one'", "'two'", "'three'", "'four'"], 'dval': {'one': 'two', 'three': 'four'}, 'mdval': {'one': 'two', 'three': 'four', 'five': 'six'}, 'nval': '1850'}, 'nml2': {'val2': '.false.'}} >>> teststr = '''&fire_emis_nl ... fire_emis_factors_file = 'fire_emis_factors_c140116.nc' @@ -159,7 +157,7 @@ def _parse_namelists(namelist_lines, filename): ... / ... ''' >>> _parse_namelists(teststr.splitlines(), 'foo') - OrderedDict([('fire_emis_nl', OrderedDict([('fire_emis_factors_file', "'fire_emis_factors_c140116.nc'"), ('fire_emis_specifier', ["'bc_a1 = BC'", "'pom_a1 = 1.4*OC'", "'pom_a2 = A*B*C'", "'SO2 = SO2'"])]))]) + {'fire_emis_nl': {'fire_emis_factors_file': "'fire_emis_factors_c140116.nc'", 'fire_emis_specifier': ["'bc_a1 = BC'", "'pom_a1 = 1.4*OC'", "'pom_a2 = A*B*C'", "'SO2 = SO2'"]}} >>> _parse_namelists('blah', 'foo') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): @@ -196,19 +194,19 @@ def _parse_namelists(namelist_lines, filename): ... val = 2, 2*13 ... /''' >>> _parse_namelists(teststr.splitlines(), 'foo') - OrderedDict([('nml', OrderedDict([('val', ['2', '13', '13'])]))]) + {'nml': {'val': ['2', '13', '13']}} >>> teststr = '''&nml ... val = 2 2 3 ... /''' >>> _parse_namelists(teststr.splitlines(), 'foo') - OrderedDict([('nml', OrderedDict([('val', ['2', '2', '3'])]))]) + {'nml': {'val': ['2', '2', '3']}} >>> teststr = '''&nml ... val = 'a brown cow' 'a red hen' ... /''' >>> _parse_namelists(teststr.splitlines(), 'foo') - OrderedDict([('nml', OrderedDict([('val', ["'a brown cow'", "'a red hen'"])]))]) + {'nml': {'val': ["'a brown cow'", "'a red hen'"]}} """ comment_re = re.compile(r"^[#!]") @@ -216,7 +214,7 @@ def _parse_namelists(namelist_lines, filename): name_re = re.compile(r"^([^\s=']+)\s*=\s*(.+)$") rcline_re = re.compile(r"^([^&\s':]+)\s*:\s*(.+)$") - rv = OrderedDict() + rv = {} current_namelist = None multiline_variable = None # (name, value) for line in namelist_lines: @@ -238,7 +236,7 @@ def _parse_namelists(namelist_lines, filename): logger.debug(" Parsing variable '{}' with data '{}'".format(name, value)) if "seq_maps.rc" not in rv: - rv["seq_maps.rc"] = OrderedDict() + rv["seq_maps.rc"] = {} expect( name not in rv["seq_maps.rc"], @@ -261,7 +259,7 @@ def _parse_namelists(namelist_lines, filename): # to signify this event if namelist_re.match(line) is None: expect( - rv != OrderedDict(), + rv != {}, "File '{}' does not appear to be a namelist file, skipping".format( filename ), @@ -281,7 +279,7 @@ def _parse_namelists(namelist_lines, filename): ), ) - rv[current_namelist] = OrderedDict() + rv[current_namelist] = {} logger.debug(" Starting namelist '{}'".format(current_namelist)) @@ -342,7 +340,7 @@ def _parse_namelists(namelist_lines, filename): real_value = _interpret_value(line, filename) if type(current_value) is list: expect( - type(real_value) is not OrderedDict, + type(real_value) is not dict, "In file '{}', multiline list variable '{}' had dict entries".format( filename, multiline_variable[0] ), @@ -350,9 +348,9 @@ def _parse_namelists(namelist_lines, filename): real_value = real_value if type(real_value) is list else [real_value] current_value.extend(real_value) - elif type(current_value) is OrderedDict: + elif type(current_value) is dict: expect( - type(real_value) is OrderedDict, + type(real_value) is dict, "In file '{}', multiline dict variable '{}' had non-dict entries".format( filename, multiline_variable[0] ), @@ -459,7 +457,7 @@ def _compare_values(name, gold_value, comp_value, case): name, comp_value_list_item ) - elif type(gold_value) is OrderedDict: + elif type(gold_value) is dict: for key, gold_value_dict_item in gold_value.items(): if key in comp_value: comments += _compare_values( @@ -639,7 +637,7 @@ def _compare_namelists(gold_namelists, comp_namelists, case): COMP: csw_specifier dict item DMS = 1.0*other.nc """ - different_namelists = OrderedDict() + different_namelists = {} for namelist, gold_names in gold_namelists.items(): if namelist not in comp_namelists: different_namelists[namelist] = ["Missing namelist: {}\n".format(namelist)] diff --git a/CIME/namelist.py b/CIME/namelist.py index 562f0ca55ac..d89dee7c2c1 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -101,7 +101,6 @@ # pylint: disable=line-too-long,too-many-lines,invalid-name import re -import collections from contextlib import contextmanager # Disable these because this is our standard setup @@ -664,7 +663,10 @@ def literal_to_python_value(literal, type_=None): >>> literal_to_python_value("") >>> literal_to_python_value("-1.D+10") -10000000000.0 - >>> shouldRaise(ValueError, literal_to_python_value, "nan(1234)") + >>> literal_to_python_value("nan(1234)") + Traceback (most recent call last): + ... + ValueError: could not convert string to float: 'nan(1234)' """ expect( FORTRAN_REPEAT_PREFIX_REGEX.search(literal) is None, @@ -887,21 +889,6 @@ def parse(in_file=None, text=None, groupless=False, convert_tab_to_space=True): return Namelist(namelist_dict) -def shouldRaise(eclass, method, *args, **kw): - """ - A helper function to make doctests py3 compatible - http://python3porting.com/problems.html#running-doctests - """ - try: - method(*args, **kw) - except BaseException: - e = sys.exc_info()[1] - if not isinstance(e, eclass): - raise - return - raise Exception("Expected exception %s not raised" % str(eclass)) - - class Namelist(object): """Class representing a Fortran namelist. @@ -932,7 +919,7 @@ def __init__(self, groups=None): if groups is not None: for group_name in groups: expect(group_name is not None, " Got None in groups {}".format(groups)) - self._groups[group_name] = collections.OrderedDict() + self._groups[group_name] = {} for variable_name in groups[group_name]: self._groups[group_name][variable_name] = groups[group_name][ variable_name @@ -946,7 +933,7 @@ def __call__(self, filename): self.write(filename) def clean_groups(self): - self._groups = collections.OrderedDict() + self._groups = {} def get_group_names(self): """Return a list of all groups in the namelist. @@ -1429,7 +1416,7 @@ def __init__(self, text, groupless=False): # Dictionary with group names as keys, and dictionaries of variable # name-value pairs as values. (Or a single flat dictionary if # `groupless=True`.) - self._settings = collections.OrderedDict() + self._settings = {} # Fortran allows setting a particular index of an array # such as foo(2)='k' # this dict is set to that value if used. @@ -1452,8 +1439,10 @@ def _curr(self): def _next(self): """Return the character at the next position. - >>> shouldRaise(_NamelistEOF, _NamelistParser(' ')._next) - + >>> _NamelistParser(' ')._next() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. """ # If at the end of the file, we should raise _NamelistEOF. The easiest # way to do this is to just advance. @@ -1492,10 +1481,14 @@ def _advance(self, nchars=1, check_eof=False): >>> x._advance(3) >>> (x._pos, x._line, x._col) (7, 3, 1) - >>> shouldRaise(_NamelistEOF, x._advance, 1) - - >>> shouldRaise(_NamelistEOF, _NamelistParser('abc\n')._advance, 4) - + >>> x._advance(1) + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. + >>> _NamelistParser('abc\n')._advance(4) + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('ab') >>> x._advance(check_eof=True) False @@ -1537,8 +1530,10 @@ def _eat_whitespace(self, allow_initial_comment=False): >>> x._eat_whitespace() False >>> x._advance() - >>> shouldRaise(_NamelistEOF, x._eat_whitespace) - + >>> x._eat_whitespace() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser(' \n! blah\n ! blah\n a') >>> x._eat_whitespace() True @@ -1592,11 +1587,15 @@ def _eat_comment(self): >>> x._curr() 'a' >>> x._advance(2) - >>> shouldRaise(_NamelistEOF, x._eat_comment) - + >>> x._eat_comment() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> x = _NamelistParser('! foo\n') - >>> shouldRaise(_NamelistEOF, x._eat_comment) - + >>> x._eat_comment() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. """ if self._curr() != "!": return False @@ -1620,8 +1619,10 @@ def _expect_char(self, chars): >>> x = _NamelistParser('ab') >>> x._expect_char('a') >>> x._advance() - >>> shouldRaise(_NamelistParseError, x._expect_char, 'a') - + >>> x._expect_char('a') + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected 'a' but found 'b' >>> x._expect_char('ab') """ if self._curr() not in chars: @@ -1636,20 +1637,30 @@ def _expect_char(self, chars): def _parse_namelist_group_name(self): r"""Parses and returns a namelist group name at the current position. - >>> shouldRaise(_NamelistParseError, _NamelistParser('abc')._parse_namelist_group_name) - - >>> shouldRaise(_NamelistEOF, _NamelistParser('&abc')._parse_namelist_group_name) - + >>> _NamelistParser('abc')._parse_namelist_group_name() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected '&' but found 'a' + >>> _NamelistParser('&abc')._parse_namelist_group_name() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('&abc ')._parse_namelist_group_name() 'abc' >>> _NamelistParser('&abc\n')._parse_namelist_group_name() 'abc' - >>> shouldRaise(_NamelistParseError, _NamelistParser('&abc/ ')._parse_namelist_group_name) - - >>> shouldRaise(_NamelistParseError, _NamelistParser('&abc= ')._parse_namelist_group_name) - - >>> shouldRaise(_NamelistParseError, _NamelistParser('& ')._parse_namelist_group_name) - + >>> _NamelistParser('&abc/ ')._parse_namelist_group_name() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: 'abc/' is not a valid variable name + >>> _NamelistParser('&abc= ')._parse_namelist_group_name() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: 'abc=' is not a valid variable name + >>> _NamelistParser('& ')._parse_namelist_group_name() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: '' is not a valid variable name """ self._expect_char("&") self._advance() @@ -1662,8 +1673,10 @@ def _parse_variable_name(self, allow_equals=True): variable name; if it is `False`, only white space can be used for this purpose. - >>> shouldRaise(_NamelistEOF, _NamelistParser('abc')._parse_variable_name) - + >>> _NamelistParser('abc')._parse_variable_name() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('foo(2)= ')._parse_variable_name() 'foo(2)' >>> _NamelistParser('abc ')._parse_variable_name() @@ -1734,14 +1747,18 @@ def _parse_character_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> shouldRaise(_NamelistEOF, _NamelistParser('"abc')._parse_character_literal) - + >>> _NamelistParser('"abc')._parse_character_literal() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc" ')._parse_character_literal() '"abc"' >>> _NamelistParser("'abc' ")._parse_character_literal() "'abc'" - >>> shouldRaise(_NamelistParseError, _NamelistParser("*abc* ")._parse_character_literal) - + >>> _NamelistParser("*abc* ")._parse_character_literal() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: *abc* is not a valid character literal >>> _NamelistParser("'abc''def' ")._parse_character_literal() "'abc''def'" >>> _NamelistParser("'abc''' ")._parse_character_literal() @@ -1776,12 +1793,16 @@ def _parse_complex_literal(self): Position on return is the last character of the string; we avoid advancing past that in order to avoid potential EOF errors. - >>> shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.')._parse_complex_literal) - + >>> _NamelistParser('(1.,2.')._parse_complex_literal() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.) ')._parse_complex_literal() '(1.,2.)' - >>> shouldRaise(_NamelistParseError, _NamelistParser("(A,B) ")._parse_complex_literal) - + >>> _NamelistParser("(A,B) ")._parse_complex_literal() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: '(A,B)' is not a valid complex literal """ old_pos = self._pos while self._curr() != ")": @@ -1862,14 +1883,18 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '"abc"' >>> _NamelistParser("'abc' ")._parse_literal() "'abc'" - >>> shouldRaise(_NamelistEOF, _NamelistParser('"abc"')._parse_literal) - + >>> _NamelistParser('"abc"')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('"abc"')._parse_literal(allow_eof_end=True) '"abc"' >>> _NamelistParser('(1.,2.) ')._parse_literal() '(1.,2.)' - >>> shouldRaise(_NamelistEOF, _NamelistParser('(1.,2.)')._parse_literal) - + >>> _NamelistParser('(1.,2.)')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('(1.,2.)')._parse_literal(allow_eof_end=True) '(1.,2.)' >>> _NamelistParser('5 ')._parse_literal() @@ -1882,8 +1907,10 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): 'nan(booga)' >>> _NamelistParser('.FLORIDA$ ')._parse_literal() '.FLORIDA$' - >>> shouldRaise(_NamelistParseError, _NamelistParser('hamburger ')._parse_literal) - + >>> _NamelistParser('hamburger ')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got 'hamburger' >>> _NamelistParser('5,')._parse_literal() '5' >>> _NamelistParser('5\n')._parse_literal() @@ -1898,14 +1925,20 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): '6*(1., 2.)' >>> _NamelistParser('6*"a" ')._parse_literal() '6*"a"' - >>> shouldRaise(_NamelistEOF, _NamelistParser('6*')._parse_literal) - + >>> _NamelistParser('6*')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser('6*')._parse_literal(allow_eof_end=True) '6*' - >>> shouldRaise(_NamelistParseError, _NamelistParser('foo= ')._parse_literal) - - >>> shouldRaise(_NamelistParseError, _NamelistParser('foo+= ')._parse_literal) - + >>> _NamelistParser('foo= ')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo=' + >>> _NamelistParser('foo+= ')._parse_literal() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got 'foo+=' >>> _NamelistParser('5,')._parse_literal(allow_name=True) '5' >>> x = _NamelistParser('foo= ') @@ -1916,10 +1949,14 @@ def _parse_literal(self, allow_name=False, allow_eof_end=False): >>> x._parse_literal(allow_name=True) >>> x._curr() 'f' - >>> shouldRaise(_NamelistParseError, _NamelistParser('6*foo= ')._parse_literal, allow_name=True) - - >>> shouldRaise(_NamelistParseError, _NamelistParser('6*foo+= ')._parse_literal, allow_name=True) - + >>> _NamelistParser('6*foo= ')._parse_literal(allow_name=True) + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo=' + >>> _NamelistParser('6*foo+= ')._parse_literal(allow_name=True) + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got '6*foo+=' >>> x = _NamelistParser('foo = ') >>> x._parse_literal(allow_name=True) >>> x._curr() @@ -2025,8 +2062,10 @@ def _expect_separator(self, allow_eof=False): >>> x._curr() '/' >>> x = _NamelistParser("a") - >>> shouldRaise(_NamelistParseError, x._expect_separator) - + >>> x._expect_separator() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected one of the characters in ' \n,/' but found 'a' >>> x = _NamelistParser(" , a") >>> x._expect_separator() True @@ -2056,8 +2095,10 @@ def _expect_separator(self, allow_eof=False): >>> x._expect_separator(allow_eof=True) True >>> x = _NamelistParser(" / ") - >>> shouldRaise(_NamelistParseError, x._expect_separator, allow_eof=True) - + >>> x._expect_separator(allow_eof=True) + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: found group-terminating '/' in file without group names """ errstring = "found group-terminating '/' in file without group names" # Deal with the possibility that we are already at EOF. @@ -2106,8 +2147,10 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'"], False) >>> _NamelistParser("foo=\n'bar' /")._parse_name_and_values() ('foo', ["'bar'"], False) - >>> shouldRaise(_NamelistParseError, _NamelistParser("foo 'bar' /")._parse_name_and_values) - + >>> _NamelistParser("foo 'bar' /")._parse_name_and_values() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected '=' but found "'" >>> _NamelistParser("foo='bar','bazz' /")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo=,,'bazz',6*/")._parse_name_and_values() @@ -2116,14 +2159,18 @@ def _parse_name_and_values(self, allow_eof_end=False): ('foo', ["'bar'", "'bazz'"], False) >>> _NamelistParser("foo='bar' 'bazz' foo2(2)='ban'")._parse_name_and_values() ('foo', ["'bar'", "'bazz'"], False) - >>> shouldRaise(_NamelistParseError, _NamelistParser("foo= foo2='ban' ")._parse_name_and_values) - + >>> _NamelistParser("foo= foo2='ban' ")._parse_name_and_values() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: expected literal value, but got "foo2='ban'" >>> _NamelistParser("foo=,,'bazz',6* ")._parse_name_and_values(allow_eof_end=True) ('foo', ['', '', "'bazz'", '6*'], False) >>> _NamelistParser("foo(3)='bazz'")._parse_name_and_values(allow_eof_end=True) ('foo(3)', ["'bazz'"], False) - >>> shouldRaise(_NamelistEOF, _NamelistParser("foo=")._parse_name_and_values) - + >>> _NamelistParser("foo=")._parse_name_and_values() + Traceback (most recent call last): + ... + namelist._NamelistEOF: Unexpected end of file encountered in namelist. >>> _NamelistParser("foo=")._parse_name_and_values(allow_eof_end=True) ('foo', [''], False) >>> _NamelistParser("foo= ")._parse_name_and_values(allow_eof_end=True) @@ -2187,52 +2234,54 @@ def _parse_namelist_group(self): >>> x = _NamelistParser("&group /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {})]) + {'group': {}} >>> x._curr() '/' >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']})]) + {'group': {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']}} >>> x = _NamelistParser("&group\n foo='bar','bazz'\n,, foo2=2*5\n /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['5', '5'])]) + {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']} >>> x._curr() '/' >>> x = _NamelistParser("&group /&group /") >>> x._parse_namelist_group() >>> x._advance() - >>> shouldRaise(_NamelistParseError, x._parse_namelist_group) - + >>> x._parse_namelist_group() + Traceback (most recent call last): + ... + namelist._NamelistParseError: Error in parsing namelist: Namelist group 'group' encountered twice. >>> x = _NamelistParser("&group foo='bar', foo='bazz' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {'foo': ["'bazz'"]})]) + {'group': {'foo': ["'bazz'"]}} >>> x = _NamelistParser("&group foo='bar', foo= /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {'foo': ["'bar'"]})]) + {'group': {'foo': ["'bar'"]}} >>> x = _NamelistParser("&group foo='bar', foo= /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('foo', ["'bar'"])]) + {'foo': ["'bar'"]} >>> x = _NamelistParser("&group foo='bar', foo+='baz' /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('foo', ["'bar'", "'baz'"])]) + {'foo': ["'bar'", "'baz'"]} >>> x = _NamelistParser("&group foo+='bar' /", groupless=True) >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('foo', ["'bar'"])]) + {'foo': ["'bar'"]} >>> x = _NamelistParser("&group foo='bar', foo+='baz' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {'foo': ["'bar'", "'baz'"]})]) + {'group': {'foo': ["'bar'", "'baz'"]}} >>> x = _NamelistParser("&group foo+='bar' /") >>> x._parse_namelist_group() >>> x._settings - OrderedDict([('group', {'foo': ["'bar'"]})]) + {'group': {'foo': ["'bar'"]}} """ group_name = self._parse_namelist_group_name() if not self._groupless: @@ -2271,41 +2320,41 @@ def parse_namelist(self): first by namelist group name, then by variable name. >>> _NamelistParser("").parse_namelist() - OrderedDict() + {} >>> _NamelistParser(" \n!Comment").parse_namelist() - OrderedDict() + {} >>> _NamelistParser(" &group /").parse_namelist() - OrderedDict([('group', {})]) + {'group': {}} >>> _NamelistParser("! Comment \n &group /! Comment\n ").parse_namelist() - OrderedDict([('group', {})]) + {'group': {}} >>> _NamelistParser("! Comment \n &group /! Comment ").parse_namelist() - OrderedDict([('group', {})]) + {'group': {}} >>> _NamelistParser("&group1\n foo='bar','bazz'\n,, foo2=2*5\n / &group2 /").parse_namelist() - OrderedDict([('group1', {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']}), ('group2', {})]) + {'group1': {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['5', '5']}, 'group2': {}} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5\n ", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['2*5'])]) + {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['2*5']} >>> _NamelistParser("!blah \n foo='bar','bazz'\n,, foo2=2*5,6\n ", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'", "'bazz'", '']), ('foo2', ['2*5', '6'])]) + {'foo': ["'bar'", "'bazz'", ''], 'foo2': ['2*5', '6']} >>> _NamelistParser("!blah \n foo='bar'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'"])]) + {'foo': ["'bar'"]} >>> _NamelistParser("foo='bar', foo(3)='bazz'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'"]), ('foo(3)', ["'bazz'"])]) + {'foo': ["'bar'"], 'foo(3)': ["'bazz'"]} >>> _NamelistParser("foo(2)='bar'", groupless=True).parse_namelist() - OrderedDict([('foo(2)', ["'bar'"])]) + {'foo(2)': ["'bar'"]} >>> _NamelistParser("foo(2)='bar', foo(3)='bazz'", groupless=True).parse_namelist() - OrderedDict([('foo(2)', ["'bar'"]), ('foo(3)', ["'bazz'"])]) + {'foo(2)': ["'bar'"], 'foo(3)': ["'bazz'"]} >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bazz'"])]) + {'foo': ["'bazz'"]} >>> _NamelistParser("foo='bar'\n foo+='bazz'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'", "'bazz'"])]) + {'foo': ["'bar'", "'bazz'"]} >>> _NamelistParser("foo='bar', foo='bazz'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bazz'"])]) + {'foo': ["'bazz'"]} >>> _NamelistParser("foo='bar', foo=", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'"])]) + {'foo': ["'bar'"]} >>> _NamelistParser("foo='bar', 'bazz'\n foo+='ban'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'", "'bazz'", "'ban'"])]) + {'foo': ["'bar'", "'bazz'", "'ban'"]} >>> _NamelistParser("foo+='bar'", groupless=True).parse_namelist() - OrderedDict([('foo', ["'bar'"])]) + {'foo': ["'bar'"]} """ # Return empty dictionary for empty files. if self._len == 0: diff --git a/CIME/test_status.py b/CIME/test_status.py index 13e52497126..93b2c34a032 100644 --- a/CIME/test_status.py +++ b/CIME/test_status.py @@ -24,9 +24,6 @@ """ from CIME.XML.standard_module_setup import * - -from collections import OrderedDict - import os, itertools from CIME import expected_fails @@ -151,7 +148,7 @@ def __init__(self, test_dir=None, test_name=None, no_io=False): """ test_dir = os.getcwd() if test_dir is None else test_dir self._filename = os.path.join(test_dir, TEST_STATUS_FILENAME) - self._phase_statuses = OrderedDict() # {name -> (status, comments)} + self._phase_statuses = {} # {name -> (status, comments)} self._test_name = test_name self._ok_to_modify = False self._no_io = no_io @@ -202,8 +199,7 @@ def set_status(self, phase, status, comments=""): ... ts.set_status("{}_base_rest".format(COMPARE_PHASE), "FAIL") ... ts.set_status(SHAREDLIB_BUILD_PHASE, "PASS", comments='Time=42') >>> ts._phase_statuses - OrderedDict([('CREATE_NEWCASE', ('PASS', '')), ('XML', ('PASS', '')), ('SETUP', ('PASS', '')), ('SHAREDLIB_BUILD', ('PASS', 'Time=42')), ('COMPARE_base_rest', ('FAIL', '')), ('MODEL_BUILD', ('PEND', ''))]) - + {'CREATE_NEWCASE': ('PASS', ''), 'XML': ('PASS', ''), 'SETUP': ('PASS', ''), 'SHAREDLIB_BUILD': ('PASS', 'Time=42'), 'COMPARE_base_rest': ('FAIL', ''), 'MODEL_BUILD': ('PEND', '')} >>> with TestStatus(test_dir="/", test_name="ERS.foo.A", no_io=True) as ts: ... ts.set_status(CREATE_NEWCASE_PHASE, "PASS") ... ts.set_status(XML_PHASE, "PASS") @@ -214,12 +210,11 @@ def set_status(self, phase, status, comments=""): ... ts.set_status(SHAREDLIB_BUILD_PHASE, "PASS", comments='Time=42') ... ts.set_status(SETUP_PHASE, "PASS") >>> ts._phase_statuses - OrderedDict([('CREATE_NEWCASE', ('PASS', '')), ('XML', ('PASS', '')), ('SETUP', ('PASS', '')), ('SHAREDLIB_BUILD', ('PEND', ''))]) - + {'CREATE_NEWCASE': ('PASS', ''), 'XML': ('PASS', ''), 'SETUP': ('PASS', ''), 'SHAREDLIB_BUILD': ('PEND', '')} >>> with TestStatus(test_dir="/", test_name="ERS.foo.A", no_io=True) as ts: ... ts.set_status(CREATE_NEWCASE_PHASE, "FAIL") >>> ts._phase_statuses - OrderedDict([('CREATE_NEWCASE', ('FAIL', ''))]) + {'CREATE_NEWCASE': ('FAIL', '')} """ expect( self._ok_to_modify, @@ -274,7 +269,7 @@ def get_status(self, phase): return self._phase_statuses[phase][0] if phase in self._phase_statuses else None def get_comment(self, phase): - return self._phase_statuses[phase][1] if phase in self._phase_statuses else None + return self._phase_statuses[phase][1] if phase in self._phase_statuses else "" def current_is(self, phase, status): try: @@ -355,7 +350,7 @@ def _parse_test_status(self, file_contents): ... PASS ERS.foo.A SHAREDLIB_BUILD Time=42 ... ''' >>> _test_helper1(contents) - OrderedDict([('CREATE_NEWCASE', ('PASS', '')), ('XML', ('PASS', '')), ('SETUP', ('FAIL', '')), ('COMPARE_base_rest', ('PASS', '')), ('SHAREDLIB_BUILD', ('PASS', 'Time=42'))]) + {'CREATE_NEWCASE': ('PASS', ''), 'XML': ('PASS', ''), 'SETUP': ('FAIL', ''), 'COMPARE_base_rest': ('PASS', ''), 'SHAREDLIB_BUILD': ('PASS', 'Time=42')} """ for line in file_contents.splitlines(): line = line.strip() diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 8e424de30bc..0bb33f8d605 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -8,11 +8,18 @@ import sysconfig from pathlib import Path from unittest import mock - -from CIME.SystemTests.mvk import MVK -from CIME.SystemTests.mvk import MVKConfig from CIME.tests.utils import chdir +evv4esm = False +try: + from CIME.SystemTests.mvk import MVK +except: + unittest.SkipTest("Skipping mvk tests. E3SM feature") +else: + from CIME.SystemTests.mvk import MVKConfig + + evv4esm = True + def create_complex_case( case_name, @@ -114,6 +121,7 @@ def tearDown(self): @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") @mock.patch("CIME.SystemTests.mvk.evv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_testmod_complex(self, evv, find_test_mods): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -210,6 +218,7 @@ def evv_test_config(case, config): @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_update_testlog(self, machines, append_testlog): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -248,6 +257,7 @@ def test_update_testlog(self, machines, append_testlog): @mock.patch("CIME.SystemTests.mvk.utils.get_urlroot") @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -289,6 +299,7 @@ def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot @mock.patch("CIME.SystemTests.mvk.utils.get_htmlroot") @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -329,6 +340,7 @@ def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") @mock.patch("CIME.SystemTests.mvk.evv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_testmod_simple(self, evv, find_test_mods): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -417,6 +429,7 @@ def test_testmod_simple(self, evv, find_test_mods): @mock.patch("CIME.SystemTests.mvk.case_setup") @mock.patch("CIME.SystemTests.mvk.MVK.build_indv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_build_phase(self, build_indv, case_setup): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -462,6 +475,7 @@ def test_build_phase(self, build_indv, case_setup): @mock.patch("CIME.SystemTests.mvk.SystemTestsCommon._generate_baseline") @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test__generate_baseline(self, evv, append_testlog, _generate_baseline): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -531,6 +545,7 @@ def test__generate_baseline(self, evv, append_testlog, _generate_baseline): @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test__compare_baseline_resubmit(self, evv, append_testlog): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -563,6 +578,7 @@ def test__compare_baseline_resubmit(self, evv, append_testlog): @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test__compare_baseline(self, evv, append_testlog): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -617,6 +633,7 @@ def test__compare_baseline(self, evv, append_testlog): expected_comments, str(temp_dir) ), append_testlog.call_args.args + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_generate_namelist_multiple_components(self): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -647,6 +664,7 @@ def test_generate_namelist_multiple_components(self): "seed_custom = 1\n", ] + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_generate_namelist(self): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -675,6 +693,7 @@ def test_generate_namelist(self): "seed_custom = 1\n", ] + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_compare_baseline(self): case = create_simple_case() @@ -694,6 +713,7 @@ def test_compare_baseline(self): case.set_value.assert_any_call("COMPARE_BASELINE", False) + @unittest.skipUnless(evv4esm, "evv4esm module not found") def test_mvk(self): case = create_simple_case() diff --git a/CIME/wait_for_tests.py b/CIME/wait_for_tests.py index 15d970356fa..fb0b5a1a555 100644 --- a/CIME/wait_for_tests.py +++ b/CIME/wait_for_tests.py @@ -37,7 +37,7 @@ def get_test_time(test_path): ############################################################################### ts = TestStatus(test_dir=test_path) comment = ts.get_comment(RUN_PHASE) - if comment is None or "time=" not in comment: + if "time=" not in comment: logging.warning("No run-phase time data found in {}".format(test_path)) return 0 else: