Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Jenkinsfile for s3 archiving #65

Closed
wants to merge 10 commits into from
Closed

Update Jenkinsfile for s3 archiving #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4439b75
Update Jenkinsfile for s3 archiving
zach1221 Feb 29, 2024
273f6f3
Update Jenkinsfile
ankimball Mar 4, 2024
440f36f
Update Jenkinsfile
zach1221 Mar 4, 2024
a1f0b81
Update JLANDDA_RUN_FCST sync with develop
zach1221 Mar 4, 2024
ba61e52
Update JLANDDA_RUN_FCST
zach1221 Mar 4, 2024
10beaa1
Update and rename land_analysis.yaml to land_analysis_era5_hera.yaml
zach1221 Mar 4, 2024
8d55b3d
Create land_analysis_era5_orion.yaml
zach1221 Mar 4, 2024
ec5ffde
Create land_analysis_gswp3_hera.yaml
zach1221 Mar 4, 2024
0b7a42a
Create land_analysis_gswp3_orion.yaml
zach1221 Mar 4, 2024
2d5b2d1
Update land_analysis_test.xml sync with develop
zach1221 Mar 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions jobs/JLANDDA_RUN_FCST
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then
fi

# save analysis restart
mkdir -p ${MEM_MODL_OUTDIR}/restarts/tile
for tile in 1 2 3 4 5 6
do
cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc
Expand Down Expand Up @@ -254,8 +255,8 @@ if [[ ${ATMOS_FORC} == "era5" ]]; then
fi

if [[ ${ATMOS_FORC} == "gswp3" ]]; then
for tile in 1 2 3 4 5 6
do
cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc
done
for tile in 1 2 3 4 5 6
do
cp ${OUTDIR}/${mem_ens}/noahmp/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc
done
fi
6 changes: 3 additions & 3 deletions parm/land_analysis.yaml → parm/land_analysis_era5_hera.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ workflow:
spec: 201912210000 201912210000 24:00:00
entities:
MACHINE: "hera"
ACCOUNT: "naqfc"
ACCOUNT: "nems"
EXP_NAME: "LETKF"
EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/LAND-DA-WORKFLOW"
EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_fixjjob"
JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
FORCING: "era5"
Expand Down Expand Up @@ -128,7 +128,7 @@ workflow:
NPROC_JEDI: "&NPROCS_ANA;"
JEDI_INSTALL: "&JEDI_INSTALL;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_OBS"
command: "&CYCLEDIR;/jobs/JLANDDA_RUN_ANA"
jobname: runana
nodes: "1:ppn=&NPROCS_ANA;"
walltime: 00:15:00
Expand Down
171 changes: 171 additions & 0 deletions parm/land_analysis_era5_orion.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
workflow:
attrs:
realtime: false
scheduler: slurm
cycledef:
- attrs:
group: epic
spec: 201912210000 201912210000 24:00:00
entities:
MACHINE: "orion"
ACCOUNT: "epic"
EXP_NAME: "LETKF"
EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_fixjjob"
JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi"
LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs"
FORCING: "era5"
RES: "96"
FCSTHR: "24"
NPROCS_ANA: "6"
NPROCS_FCST: "6"
OBS_TYPES: "GHCN"
fv3bundle_vn: "psl_develop"
DAtype: "letkfoi_snow"
SNOWDEPTHVAR: "snwdph"
TSTUB: "oro_C96.mx100"
WORKDIR: "&EXP_BASEDIR;/workdir"
CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
EXECdir: "&CYCLEDIR;/exec"
OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_era5_test"
LOG: "&EXP_BASEDIR;/tests"
PATHRT: "&EXP_BASEDIR;"
CTIME: "<cyclestr>@Y@m@d@H</cyclestr>"
PTIME: "<cyclestr offset='-1:00:00'>@Y@m@d@H</cyclestr>"
NTIME: "<cyclestr offset='01:00:00'>@Y@m@d@H</cyclestr>"
log: "&LOG;/workflow.log"
tasks:
task_prepexp:
envars:
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_EXP"
jobname: prepexp
cores: 1
walltime: 00:02:00
queue: batch
task_prepobs:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_OBS"
jobname: prepobs
cores: 1
walltime: 00:02:00
queue: batch
dependency:
taskdep:
attrs:
task: prepexp
task_prepbmat:
envars:
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_BMAT"
jobname: prepbmat
cores: 1
walltime: 00:02:00
queue: batch
dependency:
taskdep:
attrs:
task: prepobs
task_runana:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
NTIME: "&NTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
NPROC_JEDI: "&NPROCS_ANA;"
JEDI_INSTALL: "&JEDI_INSTALL;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_RUN_ANA"
jobname: runana
nodes: "1:ppn=&NPROCS_ANA;"
walltime: 00:15:00
queue: batch
dependency:
taskdep:
attrs:
task: prepbmat
task_runfcst:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
NTIME: "&NTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
JEDI_INSTALL: "&JEDI_INSTALL;"
FCSTHR: "&FCSTHR;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_RUN_FCST"
jobname: runfcst
nodes: "1:ppn=&NPROCS_FCST;"
walltime: 00:30:00
queue: batch
dependency:
taskdep:
attrs:
task: runana
171 changes: 171 additions & 0 deletions parm/land_analysis_gswp3_hera.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
workflow:
attrs:
realtime: false
scheduler: slurm
cycledef:
- attrs:
group: epic
spec: 200001030000 200001030000 24:00:00
entities:
MACHINE: "hera"
ACCOUNT: "epic"
EXP_NAME: "LETKF"
EXP_BASEDIR: "/scratch2/NAGAPE/epic/Chan-hoo.Jeon/landda_gswp3"
JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
FORCING: "gswp3"
RES: "96"
FCSTHR: "24"
NPROCS_ANA: "6"
NPROCS_FCST: "6"
OBS_TYPES: "GHCN"
fv3bundle_vn: "psl_develop"
DAtype: "letkfoi_snow"
SNOWDEPTHVAR: "snwdph"
TSTUB: "oro_C96.mx100"
WORKDIR: "&EXP_BASEDIR;/workdir"
CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
EXECdir: "&CYCLEDIR;/exec"
OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_gswp3_test"
LOG: "&EXP_BASEDIR;/tests"
PATHRT: "&EXP_BASEDIR;"
CTIME: "<cyclestr>@Y@m@d@H</cyclestr>"
PTIME: "<cyclestr offset='-24:00:00'>@Y@m@d@H</cyclestr>"
NTIME: "<cyclestr offset='24:00:00'>@Y@m@d@H</cyclestr>"
log: "&LOG;/workflow.log"
tasks:
task_prepexp:
envars:
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_EXP"
jobname: prepexp
cores: 1
walltime: 00:02:00
queue: batch
task_prepobs:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_OBS"
jobname: prepobs
cores: 1
walltime: 00:02:00
queue: batch
dependency:
taskdep:
attrs:
task: prepexp
task_prepbmat:
envars:
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_PREP_BMAT"
jobname: prepbmat
cores: 1
walltime: 00:02:00
queue: batch
dependency:
taskdep:
attrs:
task: prepobs
task_runana:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
NTIME: "&NTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
NPROC_JEDI: "&NPROCS_ANA;"
JEDI_INSTALL: "&JEDI_INSTALL;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_RUN_ANA"
jobname: runana
nodes: "1:ppn=&NPROCS_ANA;"
walltime: 00:15:00
queue: batch
dependency:
taskdep:
attrs:
task: prepbmat
task_runfcst:
envars:
OBS_TYPES: "&OBS_TYPES;"
MACHINE: "&MACHINE;"
ACCOUNT: "&ACCOUNT;"
EXP_NAME: "&EXP_NAME;"
LANDDA_INPUTS: "&LANDDA_INPUTS;"
ATMOS_FORC: "&FORCING;"
RES: "&RES;"
TSTUB: "&TSTUB;"
WORKDIR: "&WORKDIR;"
CYCLEDIR: "&CYCLEDIR;"
EXECdir: "&EXECdir;"
OUTDIR: "&OUTDIR;"
CTIME: "&CTIME;"
PTIME: "&PTIME;"
NTIME: "&NTIME;"
fv3bundle_vn: "&fv3bundle_vn;"
DAtype: "&DAtype;"
SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
JEDI_INSTALL: "&JEDI_INSTALL;"
FCSTHR: "&FCSTHR;"
account: "&ACCOUNT;"
command: "&CYCLEDIR;/jobs/JLANDDA_RUN_FCST"
jobname: runfcst
nodes: "1:ppn=&NPROCS_FCST;"
walltime: 00:30:00
queue: batch
dependency:
taskdep:
attrs:
task: runana
Loading
Loading