Skip to content

Commit

Permalink
CI test for input_data_list (#196)
Browse files Browse the repository at this point in the history
* introduce a CI test that checks whether any input files in MOM_input is missing in input_data_list

* minor changes in check_input_data_list

* fix param_templates paths

* minor improvement in check_input_data_list filter function

* more improvements in check_input_data_list filter function

* major check_input_data_list.py refactor and introduction of check_input_data_repo.py

* fix general-ci-tests.yaml

* take 2: fix general-ci-tests.yaml

* add all missing input_data_list entries
  • Loading branch information
alperaltuntas authored Nov 12, 2024
1 parent 112f5ca commit 82d183f
Show file tree
Hide file tree
Showing 7 changed files with 321 additions and 56 deletions.
30 changes: 29 additions & 1 deletion .github/workflows/general-ci-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,35 @@ jobs:
# Run the test
- name: Run the check_default_params script
run: python tests/check_default_params.py


# Job to run check_input_data_list script
check_input_data_list:

runs-on: ubuntu-latest

steps:
# Checkout the repo
- uses: actions/checkout@v4

# Run the test
- name: Run the check_input_data_list script
run: python tests/check_input_data_list.py

# Job to run check_input_data_repo script
check_input_data_repo:

runs-on: ubuntu-latest

steps:
# Checkout the repo
- uses: actions/checkout@v4

# Run the test
- name: Run the check_input_data_repo script
run: |
pip install 'svn>=1,<1.1'
python tests/check_input_data_repo.py
# Job to run the black formatter for cime_config, see black documentation for more info
check_black_format_for_cime_config:

Expand Down
4 changes: 1 addition & 3 deletions param_templates/MOM_input.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2782,9 +2782,7 @@ Global:
"default = 'MOM_IC'
The file into which to write the initial conditions."
datatype: string
value:
$OCN_GRID == "MISOMIP": "MISOMIP_IC"
else: = "${CASE}.mom6.ic.${RUN_STARTDATE}.nc"
value: = "${CASE}.mom6.ic.${RUN_STARTDATE}.nc"
TRIMMING_USES_REMAPPING:
description: |
"[Boolean] default = False
Expand Down
59 changes: 35 additions & 24 deletions param_templates/input_data_list.yaml
Original file line number Diff line number Diff line change
@@ -1,56 +1,67 @@
# The list of input files to be checked out from inputdata
---
mom.input_data_list:
ocean_hgrid:
GRID_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/ocean_hgrid_221123.nc"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/ocean_hgrid.nc"
ocean_vgrid1:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/vgrid_65L_20200626.nc"
ALE_COORDINATE_CONFIG:
$MOM6_VERTICAL_GRID == "zstar_75L": "${INPUTDIR}/zstar_75layer_2.5m_248.4m-2024-03-29.nc"
$MOM6_VERTICAL_GRID == "zstar_65L": "${INPUTDIR}/vgrid_65L_20200626.nc"
$MOM6_VERTICAL_GRID == "hycom1" and $OCN_GRID != "tx0.25v1": "${INPUTDIR}/hybrid_75layer_zstar2.50m-2020-11-23.nc"
$MOM6_VERTICAL_GRID == "hycom1" and $OCN_GRID == "tx0.25v1": "${INPUTDIR}/hycom1_75_800m.nc"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/hycom1_75_800m.nc"
ocean_vgrid2:
DIAG_COORD_DEF_Z:
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/interpolate_zgrid_40L.nc"
ocean_vgrid3:
COORD_FILE:
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/layer_coord.nc"
ocean_topog:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/ocean_topog_230413.nc"
TOPO_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/ocean_topo_tx2_3v2_240501.nc"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/ocean_topog.nc"
ocean_topo_edit:
TOPO_EDITS_FILE:
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/All_edits.nc"
tempsalt:
TEMP_SALT_Z_INIT_FILE:
$OCN_GRID in ["tx2_3v2", "tx0.25v1"]:
$INIT_LAYERS_FROM_Z_FILE == "True":
"${INPUTDIR}/${TEMP_SALT_Z_INIT_FILE}"
saltrestore:
MAX_LAYER_THICKNESS_CONFIG:
$MOM6_VERTICAL_GRID == "hycom1" and $OCN_GRID in ["tx2_3v2"]:
"${DIN_LOC_ROOT}/ocn/mom/grid_indpt/dz_max_90th_quantile.nc"
SURFACE_PRESSURE_FILE:
$OCN_GRID == "MISOMIP": "${INPUTDIR}/MISOMIP_181108.nc"
SALT_RESTORE_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/state_restore_tx2_3_20230416.nc"
tidal:
TIDAL_ENERGY_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/energy_new_tx2_3_conserve_230415_cdf5.nc"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/tidal_amplitude.v20140616.nc"
ocean_channel:
CHANNEL_LIST_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/MOM_channels_global_tx2_3v2_240501"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/MOM_channels_global_025"
ocean_geothermal:
GEOTHERMAL_FILE:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/geothermal_davies2013_tx2_3_20240318_cdf5.nc"
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/geothermal_davies2013_v1.nc"
ocean_seaw:
CHL_FILE:
$OCN_GRID == "tx0.25v1": "${INPUTDIR}/seawifs-clim-1997-2010.1440x1080.v20180328.nc"
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/seawifs-clim-1997-2010-tx2_3v2.230416.nc"
cfcs_forcing: "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/cfc_atm_20230310.nc"
diag_coord_def_rho2:
CFC_BC_FILE: "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/cfc_atm_20230310.nc"
DIAG_COORD_DEF_RHO2:
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/ocean_rho2_190917.nc"
marbl_tracers_ic_file:
$MARBL_CONFIG == "latest": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c231221.nc"
MARBL_TRACERS_IC_FILE:
$MARBL_CONFIG == "latest": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c230331.nc"
$MARBL_CONFIG == "latest+4p2z": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c231221.nc"
marbl_fesedflux_file:
MARBL_FESEDFLUX_FILE:
'"BASE_BIO_ON=TRUE" in $MARBL_TRACER_OPTS':
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/fesedflux_total_reduce_oxic_tx2_3v2.c231205.nc"
marbl_feventflux_file:
MARBL_FEVENTFLUX_FILE:
'"BASE_BIO_ON=TRUE" in $MARBL_TRACER_OPTS':
$OCN_GRID == "tx2_3v2": "${INPUTDIR}/feventflux_5gmol_tx2_3v2.c231205.nc"
riv_flux_file:
RIV_FLUX_FILE:
'"BASE_BIO_ON=TRUE" in $MARBL_TRACER_OPTS':
'$ROF_GRID == "JRA025" and $OCN_GRID == "tx2_3v2"': "${INPUTDIR}/riv_nut.gnews_gnm.rJRA025_to_tx2_3v2_nnsm_e333r100_230415.20240202.nc"
'$ROF_GRID == "r05" and $OCN_GRID == "tx2_3v2"': "${INPUTDIR}/riv_nut.gnews_gnm.r05_to_tx2_3v2_nnsm_e250r250_230914.20240202.nc"
d14c_file_1:
MARBL_D14C_FILE_1:
'"ABIO_DIC_ON=TRUE" in $MARBL_TRACER_OPTS': "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector1_global_1850-2015_yearly_v2.0_c240202.nc"
d14c_file_2:
MARBL_D14C_FILE_2:
'"ABIO_DIC_ON=TRUE" in $MARBL_TRACER_OPTS': "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector2_global_1850-2015_yearly_v2.0_c240202.nc"
d14c_file_3:
MARBL_D14C_FILE_3:
'"ABIO_DIC_ON=TRUE" in $MARBL_TRACER_OPTS': "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector3_global_1850-2015_yearly_v2.0_c240202.nc"
...
5 changes: 1 addition & 4 deletions param_templates/json/MOM_input.json
Original file line number Diff line number Diff line change
Expand Up @@ -2211,10 +2211,7 @@
"IC_OUTPUT_FILE": {
"description": "\"default = 'MOM_IC'\nThe file into which to write the initial conditions.\"\n",
"datatype": "string",
"value": {
"$OCN_GRID == \"MISOMIP\"": "MISOMIP_IC",
"else": "= \"${CASE}.mom6.ic.${RUN_STARTDATE}.nc\""
}
"value": "= \"${CASE}.mom6.ic.${RUN_STARTDATE}.nc\""
},
"TRIMMING_USES_REMAPPING": {
"description": "\"[Boolean] default = False\nWhen trimming the column, also remap T and S.\"\n",
Expand Down
60 changes: 36 additions & 24 deletions param_templates/json/input_data_list.json
Original file line number Diff line number Diff line change
@@ -1,78 +1,90 @@
{
"mom.input_data_list": {
"ocean_hgrid": {
"GRID_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/ocean_hgrid_221123.nc",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/ocean_hgrid.nc"
},
"ocean_vgrid1": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/vgrid_65L_20200626.nc",
"ALE_COORDINATE_CONFIG": {
"$MOM6_VERTICAL_GRID == \"zstar_75L\"": "${INPUTDIR}/zstar_75layer_2.5m_248.4m-2024-03-29.nc",
"$MOM6_VERTICAL_GRID == \"zstar_65L\"": "${INPUTDIR}/vgrid_65L_20200626.nc",
"$MOM6_VERTICAL_GRID == \"hycom1\" and $OCN_GRID != \"tx0.25v1\"": "${INPUTDIR}/hybrid_75layer_zstar2.50m-2020-11-23.nc",
"$MOM6_VERTICAL_GRID == \"hycom1\" and $OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/hycom1_75_800m.nc",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/hycom1_75_800m.nc"
},
"ocean_vgrid2": {
"DIAG_COORD_DEF_Z": {
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/interpolate_zgrid_40L.nc"
},
"ocean_vgrid3": {
"COORD_FILE": {
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/layer_coord.nc"
},
"ocean_topog": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/ocean_topog_230413.nc",
"TOPO_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/ocean_topo_tx2_3v2_240501.nc",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/ocean_topog.nc"
},
"ocean_topo_edit": {
"TOPO_EDITS_FILE": {
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/All_edits.nc"
},
"tempsalt": {
"TEMP_SALT_Z_INIT_FILE": {
"$OCN_GRID in [\"tx2_3v2\", \"tx0.25v1\"]": {
"$INIT_LAYERS_FROM_Z_FILE == \"True\"": "${INPUTDIR}/${TEMP_SALT_Z_INIT_FILE}"
}
},
"saltrestore": {
"MAX_LAYER_THICKNESS_CONFIG": {
"$MOM6_VERTICAL_GRID == \"hycom1\" and $OCN_GRID in [\"tx2_3v2\"]": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/dz_max_90th_quantile.nc"
},
"SURFACE_PRESSURE_FILE": {
"$OCN_GRID == \"MISOMIP\"": "${INPUTDIR}/MISOMIP_181108.nc"
},
"SALT_RESTORE_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/state_restore_tx2_3_20230416.nc"
},
"tidal": {
"TIDAL_ENERGY_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/energy_new_tx2_3_conserve_230415_cdf5.nc",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/tidal_amplitude.v20140616.nc"
},
"ocean_channel": {
"CHANNEL_LIST_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/MOM_channels_global_tx2_3v2_240501",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/MOM_channels_global_025"
},
"ocean_geothermal": {
"GEOTHERMAL_FILE": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/geothermal_davies2013_tx2_3_20240318_cdf5.nc",
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/geothermal_davies2013_v1.nc"
},
"ocean_seaw": {
"CHL_FILE": {
"$OCN_GRID == \"tx0.25v1\"": "${INPUTDIR}/seawifs-clim-1997-2010.1440x1080.v20180328.nc",
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/seawifs-clim-1997-2010-tx2_3v2.230416.nc"
},
"cfcs_forcing": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/cfc_atm_20230310.nc",
"diag_coord_def_rho2": {
"CFC_BC_FILE": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/cfc_atm_20230310.nc",
"DIAG_COORD_DEF_RHO2": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/ocean_rho2_190917.nc"
},
"marbl_tracers_ic_file": {
"$MARBL_CONFIG == \"latest\"": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c231221.nc",
"MARBL_TRACERS_IC_FILE": {
"$MARBL_CONFIG == \"latest\"": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c230331.nc",
"$MARBL_CONFIG == \"latest+4p2z\"": "${INPUTDIR}/ecosys_jan_IC_omip_latlon_1x1_180W_c231221.nc"
},
"marbl_fesedflux_file": {
"MARBL_FESEDFLUX_FILE": {
"\"BASE_BIO_ON=TRUE\" in $MARBL_TRACER_OPTS": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/fesedflux_total_reduce_oxic_tx2_3v2.c231205.nc"
}
},
"marbl_feventflux_file": {
"MARBL_FEVENTFLUX_FILE": {
"\"BASE_BIO_ON=TRUE\" in $MARBL_TRACER_OPTS": {
"$OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/feventflux_5gmol_tx2_3v2.c231205.nc"
}
},
"riv_flux_file": {
"RIV_FLUX_FILE": {
"\"BASE_BIO_ON=TRUE\" in $MARBL_TRACER_OPTS": {
"$ROF_GRID == \"JRA025\" and $OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/riv_nut.gnews_gnm.rJRA025_to_tx2_3v2_nnsm_e333r100_230415.20240202.nc",
"$ROF_GRID == \"r05\" and $OCN_GRID == \"tx2_3v2\"": "${INPUTDIR}/riv_nut.gnews_gnm.r05_to_tx2_3v2_nnsm_e250r250_230914.20240202.nc"
}
},
"d14c_file_1": {
"MARBL_D14C_FILE_1": {
"\"ABIO_DIC_ON=TRUE\" in $MARBL_TRACER_OPTS": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector1_global_1850-2015_yearly_v2.0_c240202.nc"
},
"d14c_file_2": {
"MARBL_D14C_FILE_2": {
"\"ABIO_DIC_ON=TRUE\" in $MARBL_TRACER_OPTS": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector2_global_1850-2015_yearly_v2.0_c240202.nc"
},
"d14c_file_3": {
"MARBL_D14C_FILE_3": {
"\"ABIO_DIC_ON=TRUE\" in $MARBL_TRACER_OPTS": "${DIN_LOC_ROOT}/ocn/mom/grid_indpt/atm_delta_C14_CMIP6_sector3_global_1850-2015_yearly_v2.0_c240202.nc"
}
}
Expand Down
Loading

0 comments on commit 82d183f

Please sign in to comment.