From 8559ff763aacfea2a4b7623d24df4178d6bd4fcf Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 10:36:09 -0500 Subject: [PATCH 01/34] Ouranos ESPO-G6-R2 script + new capablities This script introduces new features to the tool, including the capability to process the climate datasets, including those consisting of multiple models, submodels (those with specific configuration sets), ensemble members, and multiple scenarios (SSPs). The parent calling script is in charge of parallelization scheme, if needed. With this script, a few issues related to the current deficiencies of datatool could be resolved simultaneously. Signed-off-by: Kasra Keshavarz --- scripts/espo-g6-r2/espo-g6-r2.sh | 435 +++++++++++++++++++++++++++++++ 1 file changed, 435 insertions(+) create mode 100755 scripts/espo-g6-r2/espo-g6-r2.sh diff --git a/scripts/espo-g6-r2/espo-g6-r2.sh b/scripts/espo-g6-r2/espo-g6-r2.sh new file mode 100755 index 0000000..4c2bf06 --- /dev/null +++ b/scripts/espo-g6-r2/espo-g6-r2.sh @@ -0,0 +1,435 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variable) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - only r1i1p1f1 + -S | --scenario) scenario="$2" ; shift 2 ;; # required + -M | --model) model="$2" ; shift 2 ;; # required + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check the prefix is not set +if [[ -z $prefix ]]; then + prefix="data" +fi + +# check if $model is provided +if [[ -z $model ]]; then + echo "ERROR $(basename $0): --model value(s) missing" + exit 1; +fi + +# useful log date format function +logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +# check if the dates are within datasets date range +# define $startYear and $endYear +startYear=$(date --date "$startDate" +"%Y") +endYear=$(date --date "$endDate" +"%Y") + +# if $startYear is before 1950 raise a "WARNING" and set startDate +if [[ $startYear -lt 1950 ]]; then + echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" + echo "$(logDate)$(basename $0): WARNING! \`start-date\` is set to 1950-01-01 00:00:00" + startDate="1950-01-01" + startYear="1950" +fi +# if $endYear is beyond 2100 raise a "WARNING" and set endDate +if [[ $endYear -gt 2100 ]]; then + echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" + echo "$(logDate)$(basename $0): WARNING! \`end-date\` is set to 2100-12-31 00:00:00" + endDate="2100-12-31" + endYear="2100" +fi + + +# ===================== +# Necessary Assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + +# paths +datatoolPath="$(dirname $0)/../../" # datatool's path +# daymet index scripts works on RDRSv2.1 grids as well +# and ESPO-G6-R2 has similar grid system as RDRSv2.1 +coordIdxScript="$datatoolPath/assets/ncl_scripts/coord_daymet_idx.ncl" +coordClosestIdxScript="$datatoolPath/assets/ncl_scripts/coord_closest_daymet_idx.ncl" + + +# ========================== +# Necessary Global Variables +# ========================== +latDim="rlat" +lonDim="rlon" +timeDim="time" + + +# =================== +# Necessary Functions +# =================== +# Modules below available on Digital Research Alliance of Canada's Graham HPC +## core modules +load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +unload_core_modules () { + # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT + module -q unload cdo/2.0.4 + module -q unload nco/5.0.6 +} +## ncl modules +load_ncl_module () { + module -q load ncl/6.6.2 +} +unload_ncl_module () { + module -q unload ncl/6.6.2 +} + +# loading core modules for the script +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# ================ +# Useful functions +# ================ +####################################### +# compare float values using basic +# calculator, i.e., `bc` +# +# Arguments: +# 1: -> firstNum: first int/float +# 2: -> SecondNum: second int/float +# 3: -> operator: comparison operator +####################################### +bc_compare () { + # local variables + local firstNum=$1 + local secondNum=$2 + local operator=$3 + + # implement the comparison + echo "$(bc <<< "$firstNum $operator $secondNum")" +} + + +# =============== +# Data Processing +# =============== +# display info +echo "$(logDate)$(basename $0): processing Ouranos ESPO-G6-R2..." + +# create $modelArr array from input comma-delimited values +IFS=',' read -ra modelArr <<< $(echo $model) +# create $scenarioArr array from input comma-delimited values +IFS=',' read -ra scenarioArr <<< $(echo $scenario) +# create $ensembleArr array from input comma-delimited values +IFS=',' read -ra ensembleArr <<< $(echo $ensemble) +# create $variableArr array from input comma-delimited values +IFS=',' read -ra variableArr <<< $(echo $variables) + + +# ====================== +# Extract domain extents +# ====================== + +# parse the upper and lower bounds of a given spatial limit +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) + +# unload and load necessary modules +unload_core_modules +load_ncl_module +# choose a sample file as all files share the same grid +domainFile=$(find ${datasetDir} -type f -name "*.nc" | head -n 1) +# parse the upper and lower bounds of a given spatial limit +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) + +# extract the associated indices corresponding to $latLims and $lonLims +coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" + +# if spatial index out-of-bound, i.e., 'ERROR' is return +if [[ "${coordIdx}" == "ERROR" ]]; then + # extract the closest index values + coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordClosestIdxScript")" +fi + +# parse the output index for latitude and longitude +lonLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 1)" +latLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 2)" + +# reload necessary modules +unload_ncl_module +load_core_modules + +# ============================================ +# Build date arrays for time-series extraction +# ============================================ +# file date intervals in years - dataset's default +interval=4 + +startFormat="%Y0101" +endFormat="%Y1231" # will be redefined later depending on the $modelName + +actualFormat='%Y%m%d' + +# define needed variables +let "difference = $endYear - $startYear" +let "steps = $difference / $interval" + +# build $startDateFileArr, $endDateFileArr +startDateFileArr=() +endDateFileArr=() + +# range of jumps +range=$(seq 0 $steps) + +# filling the arrays +for iter in $range; do + # jumps every $interval years + let "jumps = $iter * $interval" + + # current date after necessary jumps + let "toDate = $jumps + $startYear" + + # extract start and end values + startValue="$(date --date "${toDate}0101" +"${startFormat}")" + endValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${endFormat}")" + + # check if endValue is beyond 2100 + endValueYear="$(date --date "${endValue}" +"%Y")" + # double-check end-date + if [[ "$endValueYear" -gt 2100 ]]; then + endValue="21001231" # irregular last date for dataset files + fi + + # fill up relevant arrays + startDateFileArr+=("${startValue}") + endDateFileArr+=("${endValue}") + +done + +# build actualStartArr array for temporal subsetting +actualStartDateArr=("${startDateFileArr[@]}") +actualStartDateArr[0]="$(date --date "${startDate}" +"${actualFormat}")" + +# and similarly, the actualEndArr array +actualEndDateArr=("${endDateFileArr[@]}") +lastIndex=$(( "${#actualEndDateArr[@]}" - 1 )) +actualEndDateArr[${lastIndex}]="$(date --date "${endDate}" +"${actualFormat}")" + + +# ===================== +# Extract dataset files +# ===================== +# Typical directory structure of the dataset is: +# ${datasetDir}/${model}/%submodel/${scenario}/${ensemble}/day/${var}/ +# and each ${var} directory contains files in the following nomenclature: +# ${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${model}_%submodel_${scenario}_${ensemble}_%yyyymmdd-%yyyymmdd.nc +# with the former date value indicating the starting year of data inside the +# file, and the latter demonstrating the ending date of data +# +# NOTE: %submodel must be determined in the upstream caller +# + +# create dataset directories in $cache and $outputDir +echo "$(logDate)$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" +echo "$(logDate)$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# iterate over models/submodels +for model in "${modelArr[@]}"; do + # extract model and submodel names + modelName=$(echo $model | cut -d '/' -f 1) + submodelName=$(echo $model | cut -d '/' -f 2) + + # iterate over scenarios, e.g., ssp245, ssp370, ssp585 + for scenario in "${scenarioArr[@]}"; do + + # iterate over ensemble members, e.g., r1p1, r1p2, etc. + for ensemble in "${ensembleArr[@]}"; do + + pathTemplate="${modelName}/${submodelName}/${scenario}/${ensemble}/day/" + if [[ -e "${datasetDir}/${pathTemplate}" ]]; then + echo "$(logDate)$(basename $0): processing ${model}.${scenario}.${ensemble} files" + mkdir -p "${cache}/${pathTemplate}" + mkdir -p "${outputDir}/${pathTemplate}" + else + echo "$(logDate)$(basename $0): ERROR! ${model}.${scenario}.${ensemble} does not exist." + break 1; + fi + + # iterate over date range of interest using index + for idx in "${!startDateFileArr[@]}"; do + + # dates for files + fileStartDate="${startDateFileArr[$idx]}" + fileEndDate="${endDateFileArr[$idx]}" + # dates for subsetting + actualStartDate="${actualStartDateArr[$idx]}" + actualEndDate="${actualEndDateArr[$idx]}" + # dates for ncks slabs + actualStartDateFormatted="$(date --date $actualStartDate +'%Y-%m-%d')" + actualEndDateFormatted="$(date --date $actualEndDate +'%Y-%m-%d')" + + # destination NetCDF file + dst="day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_${modelName}_${submodelName}_${scenario}_${ensemble}_${actualStartDate}-${actualEndDate}.nc" + + # address inconsistencies with NetCDF file end-date values in the + # dataset + if [[ "$modelName" == "NIMS-KMA" ]] || \ + [[ "$modelName" == "MOHC" ]]; then + fileEndDate=$(date --date "${fileEndDate}" +"%Y1230") + fi + + # iterate over dataset variables of interest + for var in "${variableArr[@]}"; do + + # define file for further operation + # address inconsistencies with NetCDF file name in the dataset + if [[ "$modelName" == "DKRZ" ]]; then + src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" + else + src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${modelName}_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" + fi + + # spatial subsetting + until ncks -A -v ${var} \ + -d "$latDim","${latLimsIdx}" \ + -d "$lonDim","${lonLimsIdx}" \ + -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ + ${datasetDir}/${pathTemplate}/${var}/${src} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process" >$2 + sleep 10; + done # until ncks + done # for $variableArr + + # change lon values so the extents are from ~-180 to 0 + # this is solely for easymore compatibility + until ncap2 -O -s 'where(lon>0) lon=lon-360' \ + "${cache}/${pathTemplate}/${dst}" \ + "${outputDir}/${pathTemplate}/${prefix}${dst}"; do + echo "$(logDate)$(basename $0): Process killed: restarting process" >$2 + sleep 10; + done # until ncap2 + + done # for $startDateArr + done # for $ensembleArr + done # for $scenarioArr +done # for $modelArr + +# wait for everything to finish - just in case +sleep 10 + +mkdir "$HOME/empty_dir" +echo "$(logDate)$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(logDate)$(basename $0): temporary files from $cache are removed" +echo "$(logDate)$(basename $0): results are produced under $outputDir" + From d59dd2cc9f8fe8db2932b42f0049a37b8282cb95 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 10:49:17 -0500 Subject: [PATCH 02/34] Fixing short usage and comments --- scripts/espo-g6-r2/espo-g6-r2.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/espo-g6-r2/espo-g6-r2.sh b/scripts/espo-g6-r2/espo-g6-r2.sh index 4c2bf06..e84fe5c 100755 --- a/scripts/espo-g6-r2/espo-g6-r2.sh +++ b/scripts/espo-g6-r2/espo-g6-r2.sh @@ -36,7 +36,7 @@ # Usage Functions # =============== short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" } @@ -69,7 +69,7 @@ do -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required -p | --prefix) prefix="$2" ; shift 2 ;; # optional -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - only r1i1p1f1 + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant -S | --scenario) scenario="$2" ; shift 2 ;; # required -M | --model) model="$2" ; shift 2 ;; # required From 5a07e842af0b3cfd3a55f42f48a9b2db5c968031 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 14:06:46 -0500 Subject: [PATCH 03/34] Adding new parallelization schemes Multi parallelization schemes are added, so the package not only submit array jobs based on the given date range and the chunk schemes, but also considers submitting jobs based on various models, ensemble members, and scenarios. These new parallelization schemes mostly applies to climate datasets, but not necessarily. This commit aims to save time for the user and fasten the processing time for datasets. This commit resolves issue #25 on remote GitHub hosting repository. Furthermore, it adds the ESPO dataset to the list of datasets as well. Moreover, a new option is implement to show the list of currently available datasets to the users. Signed-off-by: Kasra Keshavarz --- extract-dataset.sh | 247 +++++++++++++++++++++++---------------------- 1 file changed, 128 insertions(+), 119 deletions(-) diff --git a/extract-dataset.sh b/extract-dataset.sh index ce2540a..0e1c9ec 100755 --- a/extract-dataset.sh +++ b/extract-dataset.sh @@ -1,6 +1,6 @@ #!/bin/bash # Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan +# Copyright (C) 2022-2023, University of Saskatchewan # Copyright (C) 2023-2024, University of Calgary # # This file is part of Meteorological Data Processing Workflow @@ -36,7 +36,9 @@ # Help functions # ============== short_usage () { - echo "usage: $(basename $0) [-jh] [-i DIR] [-d DATASET] [-co DIR] [-se DATE] [-ln REAL,REAL] [-p STR]" >&1; + echo "usage: $(basename $0) [-jh] [-i DIR] [-d DATASET] [-co DIR] [-se DATE] [-ln REAL,REAL] [-p STR] + +Try \`$(basename $0) --help\` for more options." >&1; } version () { @@ -44,7 +46,6 @@ version () { exit 0; } - usage () { echo "Meteorological Data Processing Script - version $(cat $(dirname $0)/VERSION) @@ -61,8 +62,12 @@ Script options: -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds -a, --shape-file=PATH Path to the ESRI shapefile; optional - -m, --ensemble=ens1,[ens2[...]] Ensemble members to process; optional + -m, --ensemble=ens1,[ens2,[...]] Ensemble members to process; optional Leave empty to extract all ensemble members + -M, --model=model1,[model2,[...]] Models that are part of a dataset, + only applicable to climate datasets, optional + -S, --scenario=scn1,[scn2,[...]] Climate scenarios to process, only applicable + to climate datasets, optional -j, --submit-job Submit the data extraction process as a job on the SLURM system; optional -k, --no-chunk No parallelization, recommended for small domains @@ -74,35 +79,30 @@ Script options: fails; optional -u, --account Digital Research Alliance of Canada's sponsor's account name; optional, defaults to 'rpp-kshook' + -L, --list-datasets List all the available datasets and the + corresponding keywords for '--dataset' option -V, --version Show version -h, --help Show this screen and exit +For bug reports, questions, discussions open an issue +at https://github.com/kasra-keshavarz/datatool/issues" >&1; + + exit 0; +} + +list_datasets () { +echo "Meteorological Data Processing Script - version $(cat $(dirname $0)/VERSION) Currently, the following meteorological datasets are available for processing: - - 1. NCAR-GWF WRF CONUS I (DOI: 10.1007/s00382-016-3327-9) - 2. NCAR-GWF WRF CONUS II (DOI: 10.5065/49SN-8E08) - 3. ECMWF ERA5 (DOI: 10.24381/cds.adbb2d47) - 4. ECCC RDRSv2.1 (DOI: 10.5194/hess-25-4917-2021) - 5. CCRN CanRCM4-WFDEI-GEM-CaPA (DOI: 10.5194/essd-12-629-2020) - 6. WFDEI-GEM-CaPA (DOI: 10.20383/101.0111) - 7. ORNL Daymet (DOI: 10.3334/ORNLDAAC/2129) - 8. BCC-CSM2-MR (DOI: TBD) - 9. CNRM-CM6-1 (DOI: TBD) - 10. EC-Earth3-Veg (DOI: TBD) - 11. GFDL-CM4 (DOI: TBD) - 12. GFDL-ESM4 (DOI: TBD) - 13. IPSL-CM6A-LR (DOI: TBD) - 14. MRI-ESM2-0 (DOI: TBD) - 15. Hybrid-observation (DOI: 10.5194/hess-23-5151-2019) - -For bug reports, questions, discussions open an issue -at https://github.com/kasra-keshavarz/datatool/issues" >&1; +$(cat $(dirname $0)/DATASETS | sed 's/^\(.*\)$/\o033[34m\1\o033[0m/')" >&1; exit 0; } +# useful log date format function +logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + # ===================== # Necessary Assumptions @@ -113,12 +113,17 @@ alias date='TZ=UTC date' # expand aliases for the one stated above shopt -s expand_aliases +# necessary local paths for the program +scriptPath="$(dirname $0)/scripts" # scripts' path +datatoolPath="$(dirname $0)" # datatool's path +extract_submodel="${datatoolPath}/assets/bash_scripts/extract_subdir_level.sh" # script path + # ======================= # Parsing input arguments # ======================= # argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o jhVbE:d:i:v:o:s:e:t:l:n:p:c:m:ka:u: --long submit-job,help,version,parsable,email:,dataset:,dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,no-chunk,shape-file:,account: -- "$@") +parsedArguments=$(getopt -a -n extract-dataset -o jhVbLE:d:i:v:o:s:e:t:l:n:p:c:m:M:S:ka:u: --long submit-job,help,version,parsable,list-datasets,email:,dataset:,dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,model:,scenario:,no-chunk,shape-file:,account: -- "$@") validArguments=$? # check if there is no valid options if [ "$validArguments" != "0" ]; then @@ -139,6 +144,7 @@ do case "$1" in -h | --help) usage ; shift ;; # optional -V | --version) version ; shift ;; # optional + -L | --list-datasets) list_datasets ; shift ;; # optional -j | --submit-job) jobSubmission=true ; shift ;; # optional -E | --email) email="$2" ; shift 2 ;; # optional -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required @@ -151,6 +157,8 @@ do -l | --lat-lims) latLims="$2" ; shift 2 ;; # required -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required -m | --ensemble) ensemble="$2" ; shift 2 ;; # optional + -M | --model) model="$2" ; shift 2 ;; # optional + -S | --scenario) scenario="$2" ; shift 2 ;; # optional -k | --no-chunk) parallel=false ; shift ;; # optional -p | --prefix) prefixStr="$2" ; shift 2 ;; # required -b | --parsable) parsable=true ; shift ;; # optional @@ -205,7 +213,7 @@ else fi # if account is not provided, use `rpp-kshook` as default -if [[ -z $account ]]; then +if [[ -z $account ]] && [[ $jobSubmission == "true" ]]; then account="rpp-kshook" if [[ -z $parsable ]]; then echo "$(basename $0): WARNING! --account not provided, using \`rpp-kshook\` by default." @@ -325,38 +333,6 @@ chunk_dates () { fi } -####################################### -# Chunking ensemble members in array -# elements -# -# Arguments: -# 1: -> esnemble: comma-separated -# values of ensemble members -# -# Outputs: -# Global ensembleArr array containing -# individual members names or an -# empty array if '--ensemble' -# argument was not applicable -####################################### -chunk_ensemble () { - # local variables - local value="$1" - - # make global 'ensembleArr' array - IFS=',' read -ra ensembleArr <<< "$(echo "$value")" - - # check to see if the '--ensemble' - # argument was applicable - if [[ "${#ensembleArr[@]}" -gt 0 ]]; then - : - else - # make an empty array for datasets that - # do not have any ensemble members - ensembleArr=("") - fi -} - # ====================== # Necessary preparations @@ -374,6 +350,8 @@ declare -A funcArgs=([jobSubmission]="$jobSubmission" \ [prefixStr]="$prefixStr" \ [cache]="$cache" \ [ensemble]="$ensemble" \ + [model]="$model" \ + [scenario]="$scenario" ); @@ -382,8 +360,9 @@ declare -A funcArgs=([jobSubmission]="$jobSubmission" \ # ======================== call_processing_func () { # input arguments as local variables - scriptFile="$1" # script local path + local scriptFile="$1" # script local path local chunkTStep="$2" # chunking time-frame periods + local submodelFlag="$3" # flag for submodels' existence # local variables local scriptName=$(basename $scriptFile | cut -d '.' -f 1) # script/dataset name @@ -393,6 +372,12 @@ call_processing_func () { # make the $logDir if haven't been created yet mkdir -p $logDir + # if dataset contains sub-models, extract them + if [[ $submodelFlag == 1 ]]; then + model=$($extract_submodel "$datasetDir" "$model") + funcArgs[model]=$model + fi + # typical script to run for all sub-modules local script=$(cat <<- EOF bash ${scriptFile} \ @@ -406,23 +391,47 @@ call_processing_func () { --lon-lims="${funcArgs[lonLims]}" \ --prefix="${funcArgs[prefixStr]}" \ --cache="${funcArgs[cache]}" \ - --ensemble="${funcArgs[ensemble]}" + --ensemble="${funcArgs[ensemble]}" \ + --scenario="${funcArgs[scenario]}" \ + --model="${funcArgs[model]}" EOF ) # evaluate the script file using the arguments provided if [[ "${funcArgs[jobSubmission]}" == true ]]; then - # chunk time-frame and ensembles + # ========================================== + # Chunk time-frame and other relevant arrays + # ========================================== + # chunk dates chunk_dates "$chunkTStep" - chunk_ensemble "$ensemble" # 'ensemble' is a global variable + + # chunking ensemble members + IFS=',' read -ra ensembleArr <<< $ensemble + # chunking models + IFS=',' read -ra modelArr <<< $model + # chunking scenarios + IFS=',' read -ra scenarioArr <<< $scenario + + # =========================== + # Building job array iterator + # =========================== + let "dateIter = ${#ensembleArr[@]} * ${#modelArr[@]} * ${#scenarioArr[@]}" + let "ensembleIter = ${#modelArr[@]} * ${#scenarioArr[@]}" + let "modelIter = ${#scenarioArr[@]}" + + # ============================== + # Length of processing job array + # ============================== # length of total number of tasks and indices - taskLen=$(( ${#startDateArr[@]} * ${#ensembleArr[@]} )) - jobArrLen=$(( $taskLen - 1 )) + let "taskLen = ${#startDateArr[@]} * ${#ensembleArr[@]} * ${#modelArr[@]} * ${#scenarioArr[@]}" + let "jobArrLen = $taskLen - 1" - # parallel run + # ============ + # Parallel run + # ============ # FIXME: This needs to be moved into a template scheduler - # document + # document, and various schedulers need to be supported sbatch <<- EOF #!/bin/bash #SBATCH --array=0-$jobArrLen @@ -441,22 +450,35 @@ call_processing_func () { $(declare -p startDateArr) $(declare -p endDateArr) $(declare -p ensembleArr) + $(declare -p modelArr) + $(declare -p scenarioArr) - idxDate="\$(( \${SLURM_ARRAY_TASK_ID} % \${#startDateArr[@]} ))" - idxMember="\$(( \${SLURM_ARRAY_TASK_ID} / \${#startDateArr[@]} ))" + idxDate="\$(( (\${SLURM_ARRAY_TASK_ID} / ${dateIter}) % \${#startDateArr[@]} ))" + idxMember="\$(( (\${SLURM_ARRAY_TASK_ID} / ${ensembleIter}) % \${#ensembleArr[@]} ))" + idxModel="\$(( (\${SLURM_ARRAY_TASK_ID} / ${modelIter}) % \${#modelArr[@]} ))" + idxScenario="\$(( \${SLURM_ARRAY_TASK_ID} % \${#scenarioArr[@]} ))" - tBegin="\${startDateArr[\${idxDate}]}" - tEnd="\${endDateArr[\${idxDate}]}" - member="\${ensembleArr[\${idxMember}]}" + tBegin="\${startDateArr[\$idxDate]}" + tEnd="\${endDateArr[\$idxDate]}" + memberChosen="\${ensembleArr[\$idxMember]}" + modelChosen="\${modelArr[\$idxModel]}" + scenarioChosen="\${scenarioArr[\$idxScenario]}" - echo "${scriptName}.sh: #\${SLURM_ARRAY_TASK_ID} chunk submitted." - echo "${scriptName}.sh: Chunk start date is \$tBegin" - echo "${scriptName}.sh: Chunk end date is \$tEnd" - if [[ -n \${member} ]]; then - echo "${scriptName}.sh: Ensemble member is \$member" + echo "$(logDate)$(basename $0): Calling ${scriptName}.sh..." + echo "$(logDate)$(basename $0): #\${SLURM_ARRAY_TASK_ID} chunk submitted." + echo "$(logDate)$(basename $0): Chunk start date is \$tBegin" + echo "$(logDate)$(basename $0): Chunk end date is \$tEnd" + if [[ -n \${modelChosen} ]]; then + echo "$(logDate)$(basename $0): Model is \${modelChosen}" + fi + if [[ -n \${scenarioChosen} ]]; then + echo "$(logDate)$(basename $0): Scenario is \${scenarioChosen}" + fi + if [[ -n \${memberChosen} ]]; then + echo "$(logDate)$(basename $0): Ensemble member is \${memberChosen}" fi - srun ${script} --start-date="\$tBegin" --end-date="\$tEnd" --cache="${cache}/cache-\${SLURM_ARRAY_JOB_ID}-\${SLURM_ARRAY_TASK_ID}" --ensemble="\${member}" + srun ${script} --start-date="\$tBegin" --end-date="\$tEnd" --cache="${cache}/cache-\${SLURM_ARRAY_JOB_ID}-\${SLURM_ARRAY_TASK_ID}" --ensemble="\${memberChosen}" --model="\${modelChosen}" --scenario="\${scenarioChosen}" EOF if [[ -z $parsable ]]; then @@ -476,10 +498,14 @@ call_processing_func () { # FIXME: This list needs to become part of a configuration # file in future releases - -scriptPath="$(dirname $0)/scripts" +# $scriptPath is defined at the top case "${dataset,,}" in + + # ============ + # WRF products + # ============ + # NCAR-GWF CONUSI "conus1" | "conusi" | "conus_1" | "conus_i" | "conus 1" | "conus i" | "conus-1" | "conus-i") call_processing_func "$scriptPath/conus_i/conus_i.sh" "3months" @@ -490,70 +516,53 @@ case "${dataset,,}" in call_processing_func "$scriptPath/conus_ii/conus_ii.sh" "1month" ;; + # ========== + # Reanalysis + # ========== + # ECMWF ERA5 "era_5" | "era5" | "era-5" | "era 5") call_processing_func "$scriptPath/era5/era5_simplified.sh" "2years" ;; - # ECCC RDRS + # ECCC RDRS "rdrs" | "rdrsv2.1") call_processing_func "$scriptPath/rdrs/rdrs.sh" "6months" ;; - # CanRCM4-WFDEI-GEM-CaPA - "canrcm4-wfdei-gem-capa" | "canrcm4_wfdei_gem_capa") - call_processing_func "$scriptPath/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh" "5years" - ;; - - # WFDEI-GEM-CaPA - "wfdei-gem-capa" | "wfdei_gem_capa" | "wfdei-gem_capa" | "wfdei_gem-capa") - call_processing_func "$scriptPath/wfdei_gem_capa/wfdei_gem_capa.sh" "5years" - ;; + # ==================== + # Observation datasets + # ==================== # Daymet dataset "daymet" | "Daymet" ) call_processing_func "$scriptPath/daymet/daymet.sh" "5years" ;; - # BCC-CSM2-MR - "bcc" | "bcc_csm2_mr" | "bcc-csm2-mr" ) - call_processing_func "$scriptPath/bcc_csm2_mr/bcc_csm2_mr.sh" "50years" - ;; - - # CNRM_CM6_1 - "cnrm" | "cnrm_cm6_1" | "cnrm-cm6-1" ) - call_processing_func "$scriptPath/cnrm_cm6_1/cnrm_cm6_1.sh" "50years" - ;; + # ================ + # Climate datasets + # ================ - # EC_EARTH3_VEG - "ec" | "ec_earth3_veg" | "ec-earth3-veg" ) - call_processing_func "$scriptPath/ec_earth3_veg/ec_earth3_veg.sh" "50years" + # ESPO-G6-R2 dataset + "espo" | "espo-g6-r2" | "espo_g6_r2" | "espo_g6-r2" | "espo-g6_r2" ) + call_processing_func "$scriptPath/espo-g6-r2/espo-g6-r2.sh" "151years" "1" ;; - # GFDL_CM4 - "gfdl_cm4" | "gfdl-cm4" ) - call_processing_func "$scriptPath/gfdl_cm4/gfdl_cm4.sh" "50years" + # Ouranos-MRCC5-CMIP6 dataset + "crcm5-cmip6" | "mrcc5-cmip6" | "crcm5" | "mrcc5" ) + call_processing_func "$scriptPath/crcm5-cmip6/crcm5-cmip6.sh" "1year" ;; - # GDFL_ESM4 - "gfdl_esm4" | "gfdl-esm4" ) - call_processing_func "$scriptPath/gfdl_esm4/gfdl_esm4.sh" "50years" - ;; - - # IPSL_CM6A_LR - "ipsl" | "ipsl_cm6a_lr" | "ipsl-cm6a-lr" ) - call_processing_func "$scriptPath/ipsl_cm6a_lr/ipsl_cm6a_lr.sh" "50years" + # CanRCM4-WFDEI-GEM-CaPA + "canrcm4_g" | "canrcm4-wfdei-gem-capa" | "canrcm4_wfdei_gem_capa") + call_processing_func "$scriptPath/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh" "5years" ;; - - # MRI_ESM2_0 - "mri" | "mri-esm2-0" | "mri_esm2_0" ) - call_processing_func "$scriptPath/mri_esm2_0/mri_esm2_0.sh" "50years" + + # WFDEI-GEM-CaPA + "wfdei_g" | "wfdei-gem-capa" | "wfdei_gem_capa" | "wfdei-gem_capa" | "wfdei_gem-capa") + call_processing_func "$scriptPath/wfdei_gem_capa/wfdei_gem_capa.sh" "5years" ;; - # Hybrid Observation Dataset - "hybrid" | "hybrid-obs" | "hybrid_obs" | "hybrid_observation" | "hybrid-observation" ) - call_processing_func "$scriptPath/hybrid_obs/hybrid_obs.sh" "50years" - ;; # dataset not included above *) From 22f952935d2e85c553968906a27d3ad830a83992 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 14:14:06 -0500 Subject: [PATCH 04/34] Separating dataset information from the main extract-dataset script This is meant to clearly organize the information provided inside the package. The new file lists all the available datasets and the keyword that users can provide the `--dataset` option. Previously, this information was part of the main Usage message `--help` of the main script. Signed-off-by: Kasra Keshavarz --- DATASETS | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 DATASETS diff --git a/DATASETS b/DATASETS new file mode 100644 index 0000000..3e71742 --- /dev/null +++ b/DATASETS @@ -0,0 +1,44 @@ +|--------------------------------|-----------|---------------------------| +| DATASET NAME | keyword | DOI | +|--------------------------------|-----------|---------------------------| +|1. NCAR-GWF WRF CONUS I | conus_i | 10.1007/s00382-016-3327-9 | +|2. NCAR-GWF WRF CONUS II | conus_ii | 10.5065/49SN-8E08 | +|3. ECMWF ERA5 | era5 | 10.24381/cds.adbb2d47 | +|4. ECCC RDRSv2.1 | rdrs | 10.5194/hess-25-4917-2021 | +|5. CCRN CanRCM4-WFDEI-GEM-CaPA | canrcm4_g | 10.5194/essd-12-629-2020 | +|6. WFDEI-GEM-CaPA | wfdei_g | 10.20383/101.0111 | +|7. ORNL Daymet | daymet | 10.3334/ORNLDAAC/2129 | +|8. Alberta Government | ab-gov | TBD | +| 8.1. BCC-CSM2-MR | | TBD | +| 8.2. CNRM-CM6-1 | | TBD | +| 8.3. EC-Earth3-Veg | | TBD | +| 8.4. GFDL-CM4 | | TBD | +| 8.5. GFDL-ESM4 | | TBD | +| 8.6. IPSL-CM6A-LR | | TBD | +| 8.7. MRI-ESM2-0 | | TBD | +| 8.8. Hybrid-observation | | 10.5194/hess-23-5151-2019 | +|9. Ouranos ESPO-G6-R2 |espo-r6-r2 |10.1038/s41597-023-02855-z | +| 9.1. AS-RCEC | | ditto | +| 9.2. BCC | | ditto | +| 9.3. CAS | | ditto | +| 9.4. CCCma | | ditto | +| 9.5. CMCC | | ditto | +| 9.6. CNRM-CERFACS | | ditto | +| 9.7. CSIRO | | ditto | +| 9.8. CSIRO-ARCCSS | | ditto | +| 9.9. DKRZ | | ditto | +| 9.10. EC-Earth-Con | | ditto | +| 9.11. INM | | ditto | +| 9.12. IPS | | ditto | +| 9.13. MIROC | | ditto | +| 9.14. MOHC | | ditto | +| 9.15. MPI-M | | ditto | +| 9.16. MRI | | ditto | +| 9.17. NCC | | ditto | +| 9.18. NIMS-KMA | | ditto | +| 9.19. NOAA-GFDL | | ditto | +| 9.20. NUIST | | ditto | +|10. Ouranos MRCC5-CMIP6 |crcm5-cmip6| TBD | +| 10.1. CanESM5 | | TBD | +| 10.2. MPI-ESM1-2-LR | | TBD | +|--------------------------------|-----------|---------------------------| From 4f90faaebb26ae822da3710cae33656d75ae7b68 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 15:05:59 -0500 Subject: [PATCH 05/34] Adding GDDP-NEX-CMIP6 info --- DATASETS | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/DATASETS b/DATASETS index 3e71742..afef400 100644 --- a/DATASETS +++ b/DATASETS @@ -41,4 +41,40 @@ |10. Ouranos MRCC5-CMIP6 |crcm5-cmip6| TBD | | 10.1. CanESM5 | | TBD | | 10.2. MPI-ESM1-2-LR | | TBD | +|11. NASA GDDP-NEX-CMIP6 | gddp-nex |10.1038/s41597-022-01393-4 | +| 11.0. ACCESS-CM2 | | ditto | +| 11.1. ACCESS-ESM1-5 | | ditto | +| 11.2. BCC-CSM2-MR | | ditto | +| 11.3. CanESM5 | | ditto | +| 11.4. CESM2 | | ditto | +| 11.5. CESM2-WACCM | | ditto | +| 11.6. CMCC-CM2-SR5 | | ditto | +| 11.7. CMCC-ESM2 | | ditto | +| 11.8. CNRM-CM6-1 | | ditto | +| 11.9. CNRM-ESM2-1 | | ditto | +| 11.10. EC-Earth3 | | ditto | +| 11.11. EC-Earth3-Veg-LR | | ditto | +| 11.12. FGOALS-g3 | | ditto | +| 11.13. GFDL-CM4 | | ditto | +| 11.14. GFDL-CM4_gr2 | | ditto | +| 11.15. GFDL-ESM4 | | ditto | +| 11.16. GISS-E2-1-G | | ditto | +| 11.17. HadGEM3-GC31-LL | | ditto | +| 11.18. HadGEM3-GC31-MM | | ditto | +| 11.19. IITM-ESM | | ditto | +| 11.20. INM-CM4-8 | | ditto | +| 11.21. INM-CM5-0 | | ditto | +| 11.22. IPSL-CM6A-LR | | ditto | +| 11.23. KACE-1-0-G | | ditto | +| 11.24. KIOST-ESM | | ditto | +| 11.25. MIROC6 | | ditto | +| 11.26. MIROC-ES2L | | ditto | +| 11.27. MPI-ESM1-2-HR | | ditto | +| 11.28. MPI-ESM1-2-LR | | ditto | +| 11.29. MRI-ESM2-0 | | ditto | +| 11.30. NESM3 | | ditto | +| 11.31. NorESM2-LM | | ditto | +| 11.32. NorESM2-MM | | ditto | +| 11.33. TaiESM1 | | ditto | +| 11.34. UKESM1-0-LL | | ditto | |--------------------------------|-----------|---------------------------| From 1c558105f3ca8520cd990b81e3c567990282078b Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 15:08:39 -0500 Subject: [PATCH 06/34] Fixing DOI value for ab-gov dataset --- DATASETS | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/DATASETS b/DATASETS index afef400..f0522a2 100644 --- a/DATASETS +++ b/DATASETS @@ -8,15 +8,15 @@ |5. CCRN CanRCM4-WFDEI-GEM-CaPA | canrcm4_g | 10.5194/essd-12-629-2020 | |6. WFDEI-GEM-CaPA | wfdei_g | 10.20383/101.0111 | |7. ORNL Daymet | daymet | 10.3334/ORNLDAAC/2129 | -|8. Alberta Government | ab-gov | TBD | -| 8.1. BCC-CSM2-MR | | TBD | -| 8.2. CNRM-CM6-1 | | TBD | -| 8.3. EC-Earth3-Veg | | TBD | -| 8.4. GFDL-CM4 | | TBD | -| 8.5. GFDL-ESM4 | | TBD | -| 8.6. IPSL-CM6A-LR | | TBD | -| 8.7. MRI-ESM2-0 | | TBD | -| 8.8. Hybrid-observation | | 10.5194/hess-23-5151-2019 | +|8. Alberta Government | ab-gov | 10.5194/hess-23-5151-201 | +| 8.1. BCC-CSM2-MR | | ditto | +| 8.2. CNRM-CM6-1 | | ditto | +| 8.3. EC-Earth3-Veg | | ditto | +| 8.4. GFDL-CM4 | | ditto | +| 8.5. GFDL-ESM4 | | ditto | +| 8.6. IPSL-CM6A-LR | | ditto | +| 8.7. MRI-ESM2-0 | | ditto | +| 8.8. Hybrid-observation | | ditto | |9. Ouranos ESPO-G6-R2 |espo-r6-r2 |10.1038/s41597-023-02855-z | | 9.1. AS-RCEC | | ditto | | 9.2. BCC | | ditto | From ed03d80317afee5363eda4caf74bed8665380922 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 15:12:28 -0500 Subject: [PATCH 07/34] Adding NASA GDDP-NEX-CMIP6 script address --- extract-dataset.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extract-dataset.sh b/extract-dataset.sh index 0e1c9ec..b4b9840 100755 --- a/extract-dataset.sh +++ b/extract-dataset.sh @@ -550,9 +550,14 @@ case "${dataset,,}" in # Ouranos-MRCC5-CMIP6 dataset "crcm5-cmip6" | "mrcc5-cmip6" | "crcm5" | "mrcc5" ) - call_processing_func "$scriptPath/crcm5-cmip6/crcm5-cmip6.sh" "1year" + call_processing_func "$scriptPath/crcm5-cmip6/crcm5-cmip6.sh" "1years" ;; + # NASA GDDP-NEX-CMIP6 + "gddp" | "gddp-nex" | "gddp-nex" | "gddp-nex-cmip6") + call_processing_func "$scriptPath/gddp-nex-cmip6/gddp-nex-cmip6.sh" "100years" "0" + ;; + # CanRCM4-WFDEI-GEM-CaPA "canrcm4_g" | "canrcm4-wfdei-gem-capa" | "canrcm4_wfdei_gem_capa") call_processing_func "$scriptPath/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh" "5years" From 58e262ad7cf064d2d1ec75905f198724e4dcf9f4 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Fri, 23 Feb 2024 15:25:30 -0500 Subject: [PATCH 08/34] ESPO-G6-R2 data processing example --- examples/espo_example_ssrb_1950_2100.sh | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 examples/espo_example_ssrb_1950_2100.sh diff --git a/examples/espo_example_ssrb_1950_2100.sh b/examples/espo_example_ssrb_1950_2100.sh new file mode 100755 index 0000000..d05e070 --- /dev/null +++ b/examples/espo_example_ssrb_1950_2100.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Meteorological Data Processing Workflow +# Copyright (C) 2022, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This is a simple example to extract Daymet data for the +# South Saskatchewan River Basin (SSRB) approximate extents +# from Jan 1950 to Dec 2100. + +# As is mentioned on the main webpage of the repository, it is +# recommended to submit annual jobs for this dataset. + +# Always call the script in the root directory of the repository +cd .. +echo "The current directory is: $(pwd)" + +./extract-dataset.sh \ + --dataset="espo-g6-r2" \ + --dataset-dir="/project/rrg-mclark/data/meteorological-data/espo-g6-r2/ESPO-G6-R2v1.0.0" + --variable="pr,tasmax,tasmin" \ + --output-dir="/project/rrg-mclark/AON/share/ESPO-G6-R2-SMM" \ + --start-date="1950-01-01" \ + --end-date="2100-12-31" \ + --model="AS-RCEC,BCC,CAS,CCCma,CMCC,CNRM-CERFACS,CSIRO,CSIRO-ARCCSS,DKRZ,EC-Earth-Consortium,INM,IPSL,MIROC,MOHC,MPI-M,MRI,NCC,NIMS-KMA,NOAA-GFDL,NUIST" + --lat-lims=49,54 \ + --lon-lims=-120,-98 \ + --ensemble="r1i1p1f1,r1i1p1f2" \ + --scenario="ssp245,ssp370,ssp585" \ + --prefix="SRB_" \ + --cache='$SLURM_TMPDIR' \ + --email="example@company.ca" \ + --submit-job; + From a9f3c70950f5167b7d9ef908b023cfc8d7c46250 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Thu, 29 Feb 2024 12:20:41 -0500 Subject: [PATCH 09/34] Multiple minor modifications 1. the "function" keywords added to make the style compatible with that of Google's recommendations, 2. required arguments and options are revised alongside the relevant comments, 3. typos are fixed Signed-off-by: Kasra Keshavarz --- scripts/espo-g6-r2/README.md | 74 ++++++++++++++++++++++++++++++++ scripts/espo-g6-r2/espo-g6-r2.sh | 65 +++++++++++----------------- 2 files changed, 99 insertions(+), 40 deletions(-) create mode 100644 scripts/espo-g6-r2/README.md diff --git a/scripts/espo-g6-r2/README.md b/scripts/espo-g6-r2/README.md new file mode 100644 index 0000000..6adf7e0 --- /dev/null +++ b/scripts/espo-g6-r2/README.md @@ -0,0 +1,74 @@ +# ECCC `RDRS` v2.1 +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `RDRS` v2.1 dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rpp-kshook/Model_Output/RDRSv2.1 # rpp-kshook allocation +/project/rrg-mclark/data/meteorological-data/rdrsv2.1 # rrg-mclark allocation +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/RDRSv2.1 +├── 1980 +│ ├── 1980010112.nc +│ ├── 1980010212.nc +│ ├── 1980010312.nc +│ ├── . +│ ├── . +│ ├── . +│ └── 1980123112.nc +. +. +. +├── %Y +│ ├── %Y010112.nc +│ ├── . +│ ├── . +│ ├── %Y%m%d12.nc +│ ├── . +│ ├── . +│ └── %Y123112.nc +. +. +. +└── 2018 + ├── 2018010112.nc + ├── . + ├── . + ├── . + └── 2018123112.nc +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `ERA5` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The NetCDF files of the dataset contain 28 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ module load nco/5.0.6 +foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/RDRSv2.1/1980/1980010112.nc +``` + +## Spatial Extent +The spatial extent of the `RDRS` v2.1 is on latitutes from `+5.75` to `+64.75` and longitudes from `-179.9925` to `179.9728` covering North America. The resolution is 0.09 degrees (~10km). + +## Temporal Extent +The time-steps are hourly covering from `January 1980` to `December 2018`. + +## Short Description on `RDRS` v2.1 Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 1.5 (or 2) meters, 2) surface pressure, 3) air temperature at 1.5 (or 2) meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `RDRS` v2.1 dataset and their details are described in the table below: +|Variable Name |RDRSv2.1 Variable |Unit |IPCC abbreviation|Comments | +|----------------------|-------------------|-----|-----------------|----------------------| +|surface pressure |RDRS_v2.1_P_P0_SFC |mb |ps | | +|specific humidity@1.5m|RDRS_v2.1_P_HU_1.5m|1 |huss | | +|air tempreature @1.5m |RDRS_v2.1_P_TT_1.5m|C |tas | | +|wind speed @10m |RDRS_v2.1_P_UVC_10m|kts |wspd |WIND=SQRT(U102+V102)| +|precipitation |RDRS_v2.1_A_PR0_SFC|m/hr | |CaPA outputs | +|short wave radiation |RDRS_v2.1_P_FB_SFC |W m-2|rsds |Downward solar flux | +|long wave radiation |RDRS_v2.1_P_FI_SFC |W m-2|rlds |Downward infrared flux| diff --git a/scripts/espo-g6-r2/espo-g6-r2.sh b/scripts/espo-g6-r2/espo-g6-r2.sh index e84fe5c..b327c77 100755 --- a/scripts/espo-g6-r2/espo-g6-r2.sh +++ b/scripts/espo-g6-r2/espo-g6-r2.sh @@ -33,7 +33,7 @@ # =============== -# Usage Functions +# Usage functions # =============== short_usage() { echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" @@ -69,7 +69,7 @@ do -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required -p | --prefix) prefix="$2" ; shift 2 ;; # optional -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant + -m | --ensemble) ensemble="$2" ; shift 2 ;; # required -S | --scenario) scenario="$2" ; shift 2 ;; # required -M | --model) model="$2" ; shift 2 ;; # required @@ -85,18 +85,17 @@ done # check the prefix is not set if [[ -z $prefix ]]; then - prefix="data" -fi - -# check if $model is provided -if [[ -z $model ]]; then - echo "ERROR $(basename $0): --model value(s) missing" - exit 1; + prefix="data_" fi # useful log date format function logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +# ================ +# Necessary checks +# ================ + # check if the dates are within datasets date range # define $startYear and $endYear startYear=$(date --date "$startDate" +"%Y") @@ -109,6 +108,7 @@ if [[ $startYear -lt 1950 ]]; then startDate="1950-01-01" startYear="1950" fi + # if $endYear is beyond 2100 raise a "WARNING" and set endDate if [[ $endYear -gt 2100 ]]; then echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" @@ -117,9 +117,17 @@ if [[ $endYear -gt 2100 ]]; then endYear="2100" fi +# check if $model, $ensemble, and $scenario is given +if [[ -z $model ]] || \ + [[ -z $ensemble ]] || \ + [[ -z $scenario ]]; then + echo "$(logDate)$(basename $0): ERROR! \`--model\`, \`--ensemble\`, and \`--scenario\` values are required" + exit 1; +fi + # ===================== -# Necessary Assumptions +# Necessary assumptions # ===================== # TZ to be set to UTC to avoid invalid dates due to Daylight Saving alias date='TZ=UTC date' @@ -136,7 +144,7 @@ coordClosestIdxScript="$datatoolPath/assets/ncl_scripts/coord_closest_daymet_idx # ========================== -# Necessary Global Variables +# Necessary global variables # ========================== latDim="rlat" lonDim="rlon" @@ -144,24 +152,24 @@ timeDim="time" # =================== -# Necessary Functions +# Necessary functions # =================== # Modules below available on Digital Research Alliance of Canada's Graham HPC ## core modules -load_core_modules () { +function load_core_modules () { module -q load cdo/2.0.4 module -q load nco/5.0.6 } -unload_core_modules () { +function unload_core_modules () { # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT module -q unload cdo/2.0.4 module -q unload nco/5.0.6 } ## ncl modules -load_ncl_module () { +function load_ncl_module () { module -q load ncl/6.6.2 } -unload_ncl_module () { +function unload_ncl_module () { module -q unload ncl/6.6.2 } @@ -188,31 +196,8 @@ join_by () { local IFS="$1"; shift; echo "$*"; } lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } -# ================ -# Useful functions -# ================ -####################################### -# compare float values using basic -# calculator, i.e., `bc` -# -# Arguments: -# 1: -> firstNum: first int/float -# 2: -> SecondNum: second int/float -# 3: -> operator: comparison operator -####################################### -bc_compare () { - # local variables - local firstNum=$1 - local secondNum=$2 - local operator=$3 - - # implement the comparison - echo "$(bc <<< "$firstNum $operator $secondNum")" -} - - # =============== -# Data Processing +# Data processing # =============== # display info echo "$(logDate)$(basename $0): processing Ouranos ESPO-G6-R2..." From 0d35a6929e29ecc82052f0cd2ffbaf3ddb46f4c5 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Thu, 29 Feb 2024 12:24:52 -0500 Subject: [PATCH 10/34] AB Government Climate Dataset Script The script deals with the Climate Dataset produced by the Alberta Government. The dataset is not public yet, and is planned to be available soon. Signed-off-by: Kasra Keshavarz --- scripts/ab-gov/ab-gov.sh | 509 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 509 insertions(+) create mode 100755 scripts/ab-gov/ab-gov.sh diff --git a/scripts/ab-gov/ab-gov.sh b/scripts/ab-gov/ab-gov.sh new file mode 100755 index 0000000..61a28b9 --- /dev/null +++ b/scripts/ab-gov/ab-gov.sh @@ -0,0 +1,509 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n ab-gov -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # required + -M | --model) model="$2" ; shift 2 ;; # required + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# useful log date format function +logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +# check if $model is given +if [[ -z $model ]]; then + echo "$(logDate)$(basename $0): ERROR! \`--model\` value(s) required" + exit 1; +fi + +# check if $scenario is not given +if [[ ! "${model,,}" == *"hybrid"* ]] && \ + [[ -z $scenario ]]; then + echo "$(logDate)$(basename $0): ERROR! \`--scenario\` value(s) required" + echo "$(logDate)$(basename $0): WARNING! \`--scenario\` not required for \`Hybrid-observation\` model" + exit 1; +fi + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary Assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary global variables +# ========================== +latDim="lat" +lonDim="lon" +timeDim="time" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Digital Research Alliance of Canada's Graham HPC +## core modules +load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +unload_core_modules () { + # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT + module -q unload cdo/2.0.4 + module -q unload nco/5.0.6 +} +## ncl modules +load_ncl_module () { + module -q load ncl/6.6.2 +} +unload_ncl_module () { + module -q unload ncl/6.6.2 +} + +# loading core modules for the script +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + +# log date format +log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +#offset lims +offset () { float="$1"; offset="$2"; printf "%.1f," $(echo "$float + $offset" | bc) | sed 's/,$//'; } + + +# =============== +# Data processing +# =============== +# display info +echo "$(log_date)$(basename $0): processing Alberta Government Climate dataset..." + +# array of scenarios +IFS=',' read -ra scenarioArr <<< "$scenario" +# array of models +IFS=',' read -ra modelArr <<< "$model" +# array of variables +IFS=',' read -ra variableArr <<< "$variables" +# there is no "esemble" members defined for this dataset + +# since, the dataset's grid cell system is gaussian, assure to to_float() +# the $latLims and $lonLims values +latLims="$(lims_to_float "$latLims")" +lonLims="$(lims_to_float "$lonLims")" + +# since longitudes are within the [-180, +180] range, no change is +# necessary + +# since Hybrid-observation has no scenario, add a hidden scenario for +# later usage down the file +if [[ "${model,,}" == *"hybrid"* ]]; then + scenarioArr+=('HiddenScenario') +fi + + +# ================ +# Necessary checks +# ================ + +# check if the dates are within datasets date range +# define $startYear and $endYear +startYear=$(date --date "$startDate" +"%Y") +endYear=$(date --date "$endDate" +"%Y") + +# taking care of various possible scenarios for $startDate and $endDate +# $scenario and $model +## #1 if startYear is before 2015, and historical is NOT selected as a +## scenario, issue a WARNING and add historical to $scenarioArr +if [[ "$startYear" -lt 2015 ]] && \ + [[ "${scenarioArr[*]}" == "historical" ]]; then + # issue a warning and add historical to the scenarios + echo "$(logDate)$(basename $0): WARNING! Dates preceeding 2015 belongs to \`hisotrical\` scenario" + echo "$(logDate)$(basename $0): WARNING! \`historical\` is added to \`--scenario\` list" + scenarioArr+=("historical") +fi + +## #2 if endYear is beyond 2014, and SSP scenarios are NOT +## selected, issue an ERROR and terminate with exitcode 1 +if [[ "$endYear" -gt 2014 ]] && \ + [[ "${scenarioArr[*]}" == "ssp" ]]; then # `ssp` is treated as *ssp* + echo "$(logDate)$(basename $0): ERROR! Dates past 2015 belong to \`ssp\` scenarios" + echo "$(logDate)$(basename $0): ERROR! Choose the appropriate date range and try again" + exit 1; +fi + +## #3 if "Hybrid-observations" is needed, SSP scenarios are not applicable +## and $startYear and $endYear must be between 1950-2019 +if [[ "${model,,}" == *"hybrid"* ]] && \ + [[ "${scenario,,}" == *"ssp"* ]]; then + echo "$(logDate)$(basename $0): WARNING! \`Hybrid-observations\` does not have SSP scenarios" +fi +if [[ "${model,,}" == *"hybrid"* ]]; then + if [[ "$startYear" -lt "1950" ]] || \ + [[ "$endYear" -gt "2019" ]]; then + echo "$(logDate)$(basename $0): WARNING! \`Hybrid-observations\` date range is only from 1950 until 2019" + fi +fi + +## #4 if "historical" scenario's date range is from 1950 until 2014 +if [[ "${scenarios,,}" == *"historical"* ]]; then + if [[ "$startYear" -lt "1950" ]] || \ + [[ "$endYear" -gt "2014" ]]; then + echo "$(logDate)$(basename $0): WARNING! \`historical\` scenario's date range is only from 1950 until 2014" + fi +fi + +## #5 if "ssp*" scenario's date range is before 2014 or beyond 2100 +if [[ "${scenarios,,}" == *"ssp"* ]]; then + if [[ "$startYear" -lt "2015" ]]; then + echo "$(logDate)$(basename $0): WARNING! \`ssp*\` scenario's start date is 2014-01-01" + elif [[ "$endYear" -gt "2100" ]]; then + echo "$(logDate)$(basename $0): WARNING! \`ssp*\` scenario's end date is 2100-12-31" + echo "$(logDate)$(basename $0): WARNING! \`--end-date\` is set to 2100-12-31" + fi +fi + + +# ============================================ +# Build date arrays for time-series extraction +# ============================================ +# file date intervals in years - dataset's default +interval=1 + +fileDateFormat="%Y" +actualDateFormat="%Y-%m-%d" + +# define needed variables +let "difference = $endYear - $startYear" +let "steps = $difference / $interval" + +# build $startDateFileArr, $endDateFileArr +startDateFileArr=() +endDateFileArr=() +actualStartDateArr=() +actualEndDateArr=() + +# range of jumps +range=$(seq 0 $steps) + +# filling the arrays +for iter in $range; do + # jumps every $interval years + let "jumps = $iter * $interval" + + # current date after necessary jumps + let "toDate = $jumps + $startYear" + + # extract start and end values for files + startValue="$(date --date "${toDate}0101" +"${fileDateFormat}")" + endValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${fileDateFormat}")" + + # double-check end-date + if [[ "$endValue" -gt 2100 ]]; then + endValue="2100" # irregular last date for dataset files + fi + + # extract start and end values for actual dates + actualStartValue="$(date --date "${toDate}0102" +"${actualDateFormat}")" + actualEndValue="$(date --date "${toDate}0101 +${interval}years" +"${actualDateFormat}")" + + # fill up relevant arrays + startDateFileArr+=("${startValue}") + endDateFileArr+=("${endValue}") + + actualStartDateArr+=("${actualStartValue}") + actualEndDateArr+=("${actualEndValue}") +done + +# build actualStartArr array for temporal subsetting +actualStartDateArr[0]="$(date --date "${startDate}" +"${actualDateFormat}")" + +# and similarly, the actualEndArr array +lastIndex=$(( "${#actualEndDateArr[@]}" - 1 )) +actualEndDateArr[${lastIndex}]="$(date --date "${endDate}" +"${actualDateFormat}")" + + +# ===================== +# Extract dataset files +# ===================== +# Typical directory structure of the dataset is: +# ${datasetDir}/${model}/ +# and each ${model} directory contains files in the following nomenclature: +# Downscaled_${model}_MBCDS_${scenario}_pr_tmn_tmx_%Y.nc +# with the %Y year value indicating the starting year of data inside the +# file +# +# The nomenclature for the "hybrid-observations" $model is different: +# Hybrid_Daily_BCABSK_US_${var}_%Y.nc +# +# The date range of each $model and scenario is as follows: +# * all models except "Hybrid-observations": +# * historical: 1950-2014 +# * ssp126: 2015-2100 +# * ssp285: 2015-2100 +# * ssp370: 2015-2100 +# * ssp585: 2015-2100 +# * "Hybrid-observations" model: 1950-2019 (no scenarios) + +# create dataset directories in $cache and $outputDir +echo "$(logDate)$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" +echo "$(logDate)$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# iterate over models/submodels +for model in "${modelArr[@]}"; do + # extract model and submodel names + modelName=$(echo $model | cut -d '/' -f 1) + + # $modelVerboseFlag is set to 1 + modelVerboseFlag=1 + + # iterate over scenarios, e.g., ssp126, ssp245, ssp370, ssp585 + for scenario in "${scenarioArr[@]}"; do + + # $scenarioVerboseFlag set to 1 + scenarioVerboseFlag=1 + + # FIXME: the check needs to consider various names of the + # "hybrid-observations", as it is a long name and users will make typo + # mistakes. + pathTemplate="${modelName}/" + if [[ -e "${datasetDir}/${pathTemplate}" ]]; then + mkdir -p "${cache}/${pathTemplate}" + mkdir -p "${outputDir}/${pathTemplate}" + else + echo "$(logDate)$(basename $0): ERROR! '${model}' model does not exist." + break; + fi + + # iterate over date range of interest using index + for idx in "${!startDateFileArr[@]}"; do + + # dates for files + fileStartDate="${startDateFileArr[$idx]}" + fileEndDate="${endDateFileArr[$idx]}" + # dates for subsetting + actualStartDate="${actualStartDateArr[$idx]}" + actualEndDate="${actualEndDateArr[$idx]}" + # dates for ncks slabs + actualStartDateFormatted="$(date --date "${actualStartDate}" +'%Y-%m-%d')" + actualEndDateFormatted="$(date --date "${actualEndDate}" +'%Y-%m-%d')" + + # variable list for output file print + variableNames=$(echo $variables | tr ',' '_') + + # if 'Hybrid_observation' is selected, it will be treated + # differently + case "${modelName,,}" in + "hybrid_observation" | "hybrid-observation" | "hybrid" | "hybrid-obs" | "hybrid_obs" ) + # if 'HiddenScenario' is not selected, then break + if [[ ${scenario} == "HiddenScenario" ]]; then + + # if $fileStartDate is beyond 2019, break the loop + if [[ $fileStartDate -gt "2019" ]]; then + break # break $scenario for loop + fi + pathTemplate="Hybrid-observation/" + for var in ${variableArr[@]}; do + # source and destination file names + src="Hybrid_Daily_BCABSK_US_${var}_${fileStartDate}.nc" + dst="Hybrid_Daily_BCABSK_US_${variableNames}_${fileStartDate}.nc" + + # verbose message + if [[ -n $modelVerboseFlag ]]; then + echo "$(logDate)$(basename $0): processing '${model}' files" + unset modelVerboseFlag + fi + + # spatial subsetting + until ncks -A -v ${var} \ + -d "$latDim","${latLims}" \ + -d "$lonDim","${lonLims}" \ + -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ + ${datasetDir}/${pathTemplate}/${src} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process" >&2 + sleep 10; + done # until ncks + + # copy the results + cp -r ${cache}/${pathTemplate}/${dst} \ + ${outputDir}/${pathTemplate}/${prefix}${dst}; + + done # for $variableArr + else + # see if SSP scenario exists for the $model + sspFile=$(find ${datasetDir}/${pathTemplate} -type f -name "*${scenario}*.nc" | head -n 1) + if [[ -z $sspFile ]] && + [[ -n $scenarioVerboseFlag ]]; then + echo "$(logDate)$(basename $0): ERROR! '${model}.${scenario}' does not exist" + unset scenarioVerboseFlag + fi + fi + ;; + + # all other models + *) + if [[ "${scenario}" == *"HiddenScenario"* ]]; then + break 2; + fi + # define file for further operation + src="Downscaled_${modelName}_MBCDS_${scenario}_pr_tmn_tmx_${fileStartDate}.nc" + dst="Downscaled_${modelName}_MBCDS_${scenario}_${variableNames}_${fileStartDate}.nc" + + # if historical is set as a scenario, and $fileStartDate is beyond 2014, break + # the loop + if [[ ${model,,} == *"historical"* ]] && \ + [[ ${fileStartDate} -gt "2014" ]]; then + break + fi + + # see if SSP scenario exists for the $model + sspFile=$(find ${datasetDir}/${pathTemplate} -type f -name "*${scenario}*.nc" | head -n 1) + if [[ -z $sspFile ]]; then + echo "$(logDate)$(basename $0): ERROR! '${model}.${scenario}' does not exist" + break + fi + + # verbose message + if [[ -n $scenarioVerboseFlag ]]; then + echo "$(logDate)$(basename $0): processing '${model}.${scenario}' files" + unset scenarioVerboseFlag + fi + + # spatial subsetting + until ncks -A -v "${variables}" \ + -d "$latDim","${latLims}" \ + -d "$lonDim","${lonLims}" \ + -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ + ${datasetDir}/${pathTemplate}/${src} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process" >&2 + sleep 10; + done # until ncks + + # copy the results + cp -r ${cache}/${pathTemplate}/${dst} \ + ${outputDir}/${pathTemplate}/${prefix}${dst}; + + ;; + esac + + done # for $startDateArr + done # for $scenarioArr +done # for $modelArr + +# wait for everything to finish - just in case +sleep 10 + +mkdir "$HOME/empty_dir" +echo "$(logDate)$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(logDate)$(basename $0): temporary files from $cache are removed" +echo "$(logDate)$(basename $0): results are produced under $outputDir" + + From defdde26334d90a1ec7a816c427249f97794684a Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 12:37:56 -0500 Subject: [PATCH 11/34] Adding variable list for various elevation levels Since some hydrological models can use near-surface level or 40m level data, the necessary list of variables for both levels are added. Furthermore, a link to the official website for the dataset is added for further clarity. Signed-off-by: Kasra Keshavarz --- scripts/eccc-rdrs/README.md | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 scripts/eccc-rdrs/README.md diff --git a/scripts/eccc-rdrs/README.md b/scripts/eccc-rdrs/README.md new file mode 100644 index 0000000..f7e2e35 --- /dev/null +++ b/scripts/eccc-rdrs/README.md @@ -0,0 +1,79 @@ +# ECCC `RDRS` v2.1 +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `RDRS` v2.1 dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rpp-kshook/Model_Output/RDRSv2.1 # rpp-kshook allocation +/project/rrg-mclark/data/meteorological-data/rdrsv2.1 # rrg-mclark allocation +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/RDRSv2.1 +├── 1980 +│ ├── 1980010112.nc +│ ├── 1980010212.nc +│ ├── 1980010312.nc +│ ├── . +│ ├── . +│ ├── . +│ └── 1980123112.nc +. +. +. +├── %Y +│ ├── %Y010112.nc +│ ├── . +│ ├── . +│ ├── %Y%m%d12.nc +│ ├── . +│ ├── . +│ └── %Y123112.nc +. +. +. +└── 2018 + ├── 2018010112.nc + ├── . + ├── . + ├── . + └── 2018123112.nc +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `ERA5` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The NetCDF files of the dataset contain 28 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ module load nco/5.0.6 +foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/RDRSv2.1/1980/1980010112.nc +``` + +## Spatial Extent +The spatial extent of the `RDRS` v2.1 is on latitutes from `+5.75` to `+64.75` and longitudes from `-179.9925` to `179.9728` covering North America. The resolution is 0.09 degrees (~10km). + +## Temporal Extent +The time-steps are hourly covering from `January 1980` to `December 2018`. + +## Short Description on `RDRS` v2.1 Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity (@1.5-2.0m or @40m), 2) surface pressure, 3) air temperature (@1.5m-2.0m or 40m), 4) wind speed (@10m or @40m), 5) precipitation (surface level), 6) downward short wave radiation (surface level), and 7) downward long wave radiation (surface level). These variables are available through `RDRS` v2.1 dataset and their details are described in the table below: +|Variable Name |RDRSv2.1 Variable |Unit |IPCC abbreviation|Comments | +|----------------------|---------------------|-----|-----------------|----------------------| +|surface pressure |RDRS_v2.1_P_P0_SFC |mb |ps | | +|specific humidity@1.5m|RDRS_v2.1_P_HU_1.5m |1 |huss | | +|air tempreature @1.5m |RDRS_v2.1_P_TT_1.5m |°C |tas | | +|wind speed @10m |RDRS_v2.1_P_UVC_10m |kts |wspd |WIND=SQRT(U102+V102)| +|precipitation |RDRS_v2.1_A_PR0_SFC |m/hr | |CaPA outputs | +|short wave radiation |RDRS_v2.1_P_FB_SFC |W m-2|rsds |Downward solar flux | +|long wave radiation |RDRS_v2.1_P_FI_SFC |W m-2|rlds |Downward infrared flux| +|specific humidity @40m|RDRS_v2.1_P_HU_09944 |1 |huss | | +|air temperature @40m |RDRS_v2.1_P_TT_09944 |°C |tas | | +|wind speed @40m |RDRS_v2.1_P_UVC_09944|kts |wspd | | + +Please visit the [official website](https://github.com/julemai/CaSPAr/wiki/Available-products) for the dataset for the most up-to-date information. From b3f972823fd5e983366e4b73598fb7b527a590e8 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 12:41:38 -0500 Subject: [PATCH 12/34] Path to the dataset for rpp-kshook allocation is updated Since multiple HPCs are now used for the workflows, it is important to have consistent datasets synchronized regularly. Therefore, this commit attempts to reflect these efforts by creating consistent paths for various HPCs/allocations. Signed-off-by: Kasra Keshavarz --- scripts/eccc-rdrs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/eccc-rdrs/README.md b/scripts/eccc-rdrs/README.md index f7e2e35..cf65495 100644 --- a/scripts/eccc-rdrs/README.md +++ b/scripts/eccc-rdrs/README.md @@ -4,7 +4,7 @@ In this file, the details of the dataset is explained. ## Location of Dataset Files The `RDRS` v2.1 dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: ```console -/project/rpp-kshook/Model_Output/RDRSv2.1 # rpp-kshook allocation +/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/rdrsv2.1 # rpp-kshook allocation /project/rrg-mclark/data/meteorological-data/rdrsv2.1 # rrg-mclark allocation ``` and the structure of the dataset hourly files is as following: From 74ce455a27882ce99f0a98b70b241d49f0d568a8 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 12:43:37 -0500 Subject: [PATCH 13/34] Bumping version to v0.5.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7532512..8f0916f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.2-dev +0.5.0 From a1fafaf0efcf684cd13d5917650ed77e67e8346a Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 12:49:32 -0500 Subject: [PATCH 14/34] Addressing issues #39, #37, #36, #35, #34, and #25 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this commit, the following are addressed: * Correcting paths for the local scripts, * Renaming scripts to reflect the owner of the script for further clarification, * Adding parallelization schemes based on model, ensemble, and scenario, * Adding gcc/9.3.0 as the reference clib for the modules loaded to prevent mismatch between various environments defined on the HPCs, * Assuring ESPG:4326 is considered for the input shape file if there is no CRS defined, * Getting rid of \t characters in the help messages, * Correcting short help message to be more informative, * Adding function declarations to follow Google’s shell scripting guidelines, * Assuring --account=STR is described in the help message. Signed-off-by: Kasra Keshavarz --- extract-dataset.sh | 182 ++++++++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 83 deletions(-) diff --git a/extract-dataset.sh b/extract-dataset.sh index b4b9840..946b73e 100755 --- a/extract-dataset.sh +++ b/extract-dataset.sh @@ -35,62 +35,62 @@ # ============== # Help functions # ============== -short_usage () { +function short_usage () { echo "usage: $(basename $0) [-jh] [-i DIR] [-d DATASET] [-co DIR] [-se DATE] [-ln REAL,REAL] [-p STR] - + Try \`$(basename $0) --help\` for more options." >&1; } -version () { +function version () { echo "$(basename $0): version $(cat $(dirname $0)/VERSION)"; exit 0; } -usage () { +function usage () { echo "Meteorological Data Processing Script - version $(cat $(dirname $0)/VERSION) Usage: $(basename $0) [options...] Script options: - -d, --dataset Meteorological forcing dataset of interest - -i, --dataset-dir=DIR The source path of the dataset file(s) - -v, --variable=var1[,var2[...]] Variables to process - -o, --output-dir=DIR Writes processed files to DIR - -s, --start-date=DATE The start date of the data - -e, --end-date=DATE The end date of the data - -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds - -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds - -a, --shape-file=PATH Path to the ESRI shapefile; optional - -m, --ensemble=ens1,[ens2,[...]] Ensemble members to process; optional - Leave empty to extract all ensemble members - -M, --model=model1,[model2,[...]] Models that are part of a dataset, - only applicable to climate datasets, optional - -S, --scenario=scn1,[scn2,[...]] Climate scenarios to process, only applicable - to climate datasets, optional - -j, --submit-job Submit the data extraction process as a job - on the SLURM system; optional - -k, --no-chunk No parallelization, recommended for small domains - -p, --prefix=STR Prefix prepended to the output files - -b, --parsable Parsable SLURM message mainly used - for chained job submissions - -c, --cache=DIR Path of the cache directory; optional - -E, --email=user@example.com E-mail user when job starts, ends, or - fails; optional - -u, --account Digital Research Alliance of Canada's sponsor's - account name; optional, defaults to 'rpp-kshook' - -L, --list-datasets List all the available datasets and the - corresponding keywords for '--dataset' option - -V, --version Show version - -h, --help Show this screen and exit + -d, --dataset Meteorological forcing dataset of interest + -i, --dataset-dir=DIR The source path of the dataset file(s) + -v, --variable=var1[,var2[...]] Variables to process + -o, --output-dir=DIR Writes processed files to DIR + -s, --start-date=DATE The start date of the data + -e, --end-date=DATE The end date of the data + -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds + -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds + -a, --shape-file=PATH Path to the ESRI shapefile; optional + -m, --ensemble=ens1,[ens2,[...]] Ensemble members to process; optional + Leave empty to extract all ensemble members + -M, --model=model1,[model2,[...]] Models that are part of a dataset, + only applicable to climate datasets, optional + -S, --scenario=scn1,[scn2,[...]] Climate scenarios to process, only applicable + to climate datasets, optional + -j, --submit-job Submit the data extraction process as a job + on the SLURM system; optional + -k, --no-chunk No parallelization, recommended for small domains + -p, --prefix=STR Prefix prepended to the output files + -b, --parsable Parsable SLURM message mainly used + for chained job submissions + -c, --cache=DIR Path of the cache directory; optional + -E, --email=user@example.com E-mail user when job starts, ends, or + fails; optional + -u, --account=ACCOUNT Digital Research Alliance of Canada's sponsor's + account name; optional, defaults to 'rpp-kshook' + -L, --list-datasets List all the available datasets and the + corresponding keywords for '--dataset' option + -V, --version Show version + -h, --help Show this screen and exit For bug reports, questions, discussions open an issue at https://github.com/kasra-keshavarz/datatool/issues" >&1; - + exit 0; } -list_datasets () { +function list_datasets () { echo "Meteorological Data Processing Script - version $(cat $(dirname $0)/VERSION) Currently, the following meteorological datasets are @@ -103,6 +103,8 @@ $(cat $(dirname $0)/DATASETS | sed 's/^\(.*\)$/\o033[34m\1\o033[0m/')" >&1; # useful log date format function logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } +# useful maximum function +max () { printf "%s\n" "${@:2}" | sort "$1" | tail -n1; } # ===================== # Necessary Assumptions @@ -143,10 +145,10 @@ while : do case "$1" in -h | --help) usage ; shift ;; # optional - -V | --version) version ; shift ;; # optional + -V | --version) version ; shift ;; # optional -L | --list-datasets) list_datasets ; shift ;; # optional -j | --submit-job) jobSubmission=true ; shift ;; # optional - -E | --email) email="$2" ; shift 2 ;; # optional + -E | --email) email="$2" ; shift 2 ;; # optional -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required -d | --dataset) dataset="$2" ; shift 2 ;; # required -v | --variable) variables="$2" ; shift 2 ;; # required @@ -160,9 +162,9 @@ do -M | --model) model="$2" ; shift 2 ;; # optional -S | --scenario) scenario="$2" ; shift 2 ;; # optional -k | --no-chunk) parallel=false ; shift ;; # optional - -p | --prefix) prefixStr="$2" ; shift 2 ;; # required + -p | --prefix) prefixStr="$2" ; shift 2 ;; # required -b | --parsable) parsable=true ; shift ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # optional -u | --account) account="$2" ; shift 2 ;; # optional -a | --shape-file) shapefile="$2" ; shift 2 ;; # optional @@ -223,18 +225,23 @@ fi # if shapefile is provided extract the extents from it if [[ -n $shapefile ]]; then # load GDAL module - module -q load gdal; + module -q load gcc/9.3.0 gdal/3.4.3; # extract the shapefile extent IFS=' ' read -ra shapefileExtents <<< "$(ogrinfo -so -al "$shapefile" | sed 's/[),(]//g' | grep Extent)" # transform the extents in case they are not in EPSG:4326 - IFS=':' read -ra sourceProj4 <<< "$(gdalsrsinfo $shapefile | grep -e "PROJ.4")" # source Proj4 value - # transform limits and assing to variables + IFS=':' read -ra sourceProj4 <<< "$(gdalsrsinfo $shapefile | grep -e "PROJ.4")" >&2 + # Assuming EPSG:4326 if no definition of the CRS is provided + if [[ ${#sourceProj4[@]} -eq 0 ]]; then + echo "$(basename $0): WARNING! Assuming EPSG:4326 for --shape-file as none provided" + sourceProj4=('PROJ4.J' '+proj=longlat +datum=WGS84 +no_defs') + fi + # transform limits and assign to variables IFS=' ' read -ra leftBottomLims <<< $(echo "${shapefileExtents[@]:1:2}" | gdaltransform -s_srs "${sourceProj4[1]}" -t_srs EPSG:4326 -output_xy) IFS=' ' read -ra rightTopLims <<< $(echo "${shapefileExtents[@]:4:5}" | gdaltransform -s_srs "${sourceProj4[1]}" -t_srs EPSG:4326 -output_xy) # define $latLims and $lonLims from $shapefileExtents lonLims="${leftBottomLims[0]},${rightTopLims[0]}" latLims="${leftBottomLims[1]},${rightTopLims[1]}" - module -q unload gdal; + module -q unload gdal/3.4.3; fi # check mandatory arguments whether provided @@ -276,7 +283,7 @@ dateFormat="%Y-%m-%d %H:%M:%S" # Globals: # startDate: start date of the # subsetting process -# parallel: true by default, false if +# parallel: true by default, false if # --no-chunk is activated # startDateArr: array of chunked # start dates @@ -287,7 +294,6 @@ dateFormat="%Y-%m-%d %H:%M:%S" # endDate: end date of the process # dateFormat: default date format # for manipulations -# # # Arguments: # 1: -> tStep: string of time-step @@ -298,7 +304,7 @@ dateFormat="%Y-%m-%d %H:%M:%S" # will be filled for each chunk of # date for further processing ####################################### -chunk_dates () { +function chunk_dates () { # local variables local toDate="$startDate" local tStep="$1" @@ -339,26 +345,26 @@ chunk_dates () { # ====================== # put necessary arguments in an array - just for legibility declare -A funcArgs=([jobSubmission]="$jobSubmission" \ - [datasetDir]="$datasetDir" \ - [variables]="$variables" \ - [outputDir]="$outputDir" \ - [timeScale]="$timeScale" \ - [startDate]="$startDate" \ - [endDate]="$endDate" \ - [latLims]="$latLims" \ - [lonLims]="$lonLims" \ - [prefixStr]="$prefixStr" \ - [cache]="$cache" \ - [ensemble]="$ensemble" \ - [model]="$model" \ - [scenario]="$scenario" - ); + [datasetDir]="$datasetDir" \ + [variables]="$variables" \ + [outputDir]="$outputDir" \ + [timeScale]="$timeScale" \ + [startDate]="$startDate" \ + [endDate]="$endDate" \ + [latLims]="$latLims" \ + [lonLims]="$lonLims" \ + [prefixStr]="$prefixStr" \ + [cache]="$cache" \ + [ensemble]="$ensemble" \ + [model]="$model" \ + [scenario]="$scenario" + ); # ======================== # Data processing function # ======================== -call_processing_func () { +function call_processing_func () { # input arguments as local variables local scriptFile="$1" # script local path local chunkTStep="$2" # chunking time-frame periods @@ -404,7 +410,7 @@ call_processing_func () { # ========================================== # chunk dates chunk_dates "$chunkTStep" - + # chunking ensemble members IFS=',' read -ra ensembleArr <<< $ensemble # chunking models @@ -415,16 +421,21 @@ call_processing_func () { # =========================== # Building job array iterator # =========================== - let "dateIter = ${#ensembleArr[@]} * ${#modelArr[@]} * ${#scenarioArr[@]}" - let "ensembleIter = ${#modelArr[@]} * ${#scenarioArr[@]}" - let "modelIter = ${#scenarioArr[@]}" + let "ensembleLen = $(max -g ${#ensembleArr[@]} 1)" + let "modelLen = $(max -g ${#modelArr[@]} 1)" + let "scenarioLen = $(max -g ${#scenarioArr[@]} 1)" + let "dateLen = $(max -g ${#startDateArr[@]} 1)" + + let "dateIter = $ensembleLen * $modelLen * $scenarioLen" + let "ensembleIter = $modelLen * $scenarioLen" + let "modelIter = $scenarioLen" # ============================== # Length of processing job array # ============================== - # length of total number of tasks and indices - let "taskLen = ${#startDateArr[@]} * ${#ensembleArr[@]} * ${#modelArr[@]} * ${#scenarioArr[@]}" + # length of total number of tasks and indices + let "taskLen = $dateLen * $ensembleLen * $modelLen * $scenarioLen" let "jobArrLen = $taskLen - 1" # ============ @@ -453,10 +464,10 @@ call_processing_func () { $(declare -p modelArr) $(declare -p scenarioArr) - idxDate="\$(( (\${SLURM_ARRAY_TASK_ID} / ${dateIter}) % \${#startDateArr[@]} ))" - idxMember="\$(( (\${SLURM_ARRAY_TASK_ID} / ${ensembleIter}) % \${#ensembleArr[@]} ))" - idxModel="\$(( (\${SLURM_ARRAY_TASK_ID} / ${modelIter}) % \${#modelArr[@]} ))" - idxScenario="\$(( \${SLURM_ARRAY_TASK_ID} % \${#scenarioArr[@]} ))" + idxDate="\$(( (\${SLURM_ARRAY_TASK_ID} / ${dateIter}) % ${dateLen} ))" + idxMember="\$(( (\${SLURM_ARRAY_TASK_ID} / ${ensembleIter}) % ${ensembleLen} ))" + idxModel="\$(( (\${SLURM_ARRAY_TASK_ID} / ${modelIter}) % ${modelLen} ))" + idxScenario="\$(( \${SLURM_ARRAY_TASK_ID} % ${scenarioLen} ))" tBegin="\${startDateArr[\$idxDate]}" tEnd="\${endDateArr[\$idxDate]}" @@ -508,12 +519,12 @@ case "${dataset,,}" in # NCAR-GWF CONUSI "conus1" | "conusi" | "conus_1" | "conus_i" | "conus 1" | "conus i" | "conus-1" | "conus-i") - call_processing_func "$scriptPath/conus_i/conus_i.sh" "3months" + call_processing_func "$scriptPath/gwf-ncar-conus_i/conus_i.sh" "3months" ;; # NCAR-GWF CONUSII "conus2" | "conusii" | "conus_2" | "conus_ii" | "conus 2" | "conus ii" | "conus-2" | "conus-ii") - call_processing_func "$scriptPath/conus_ii/conus_ii.sh" "1month" + call_processing_func "$scriptPath/gwf-ncar-conus_ii/conus_ii.sh" "1month" ;; # ========== @@ -522,12 +533,12 @@ case "${dataset,,}" in # ECMWF ERA5 "era_5" | "era5" | "era-5" | "era 5") - call_processing_func "$scriptPath/era5/era5_simplified.sh" "2years" + call_processing_func "$scriptPath/ecmwf-era5/era5_simplified.sh" "2years" ;; # ECCC RDRS "rdrs" | "rdrsv2.1") - call_processing_func "$scriptPath/rdrs/rdrs.sh" "6months" + call_processing_func "$scriptPath/eccc-rdrs/rdrs.sh" "6months" ;; # ==================== @@ -536,7 +547,7 @@ case "${dataset,,}" in # Daymet dataset "daymet" | "Daymet" ) - call_processing_func "$scriptPath/daymet/daymet.sh" "5years" + call_processing_func "$scriptPath/ornl-daymet/daymet.sh" "5years" ;; # ================ @@ -545,27 +556,32 @@ case "${dataset,,}" in # ESPO-G6-R2 dataset "espo" | "espo-g6-r2" | "espo_g6_r2" | "espo_g6-r2" | "espo-g6_r2" ) - call_processing_func "$scriptPath/espo-g6-r2/espo-g6-r2.sh" "151years" "1" + call_processing_func "$scriptPath/ouranos-espo-g6-r2/espo-g6-r2.sh" "151years" "1" ;; # Ouranos-MRCC5-CMIP6 dataset "crcm5-cmip6" | "mrcc5-cmip6" | "crcm5" | "mrcc5" ) - call_processing_func "$scriptPath/crcm5-cmip6/crcm5-cmip6.sh" "1years" + call_processing_func "$scriptPath/ouranos-crcm5-cmip6/crcm5-cmip6.sh" "1years" ;; + # Alberta Government Downscaled Climate Dataset - CMIP6 + "alberta" | "ab-gov" | "ab" | "ab_gov" | "abgov" ) + call_processing_func "$scriptPath/ab-gov/ab-gov.sh" "151years" "0" + ;; + # NASA GDDP-NEX-CMIP6 - "gddp" | "gddp-nex" | "gddp-nex" | "gddp-nex-cmip6") - call_processing_func "$scriptPath/gddp-nex-cmip6/gddp-nex-cmip6.sh" "100years" "0" + "gddp" | "nex" | "gddp-nex" | "nex-gddp" | "gddp-nex-cmip6" | "nex-gddp-cmip6") + call_processing_func "$scriptPath/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh" "100years" "0" ;; # CanRCM4-WFDEI-GEM-CaPA "canrcm4_g" | "canrcm4-wfdei-gem-capa" | "canrcm4_wfdei_gem_capa") - call_processing_func "$scriptPath/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh" "5years" + call_processing_func "$scriptPath/ccrn-canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh" "5years" ;; # WFDEI-GEM-CaPA "wfdei_g" | "wfdei-gem-capa" | "wfdei_gem_capa" | "wfdei-gem_capa" | "wfdei_gem-capa") - call_processing_func "$scriptPath/wfdei_gem_capa/wfdei_gem_capa.sh" "5years" + call_processing_func "$scriptPath/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh" "5years" ;; From 18f9652e1e437030f117b580669f88465df56e66 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 12:53:36 -0500 Subject: [PATCH 15/34] Assuring compatibility of the style with Google's shell scripting guidelines --- scripts/ab-gov/ab-gov.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/ab-gov/ab-gov.sh b/scripts/ab-gov/ab-gov.sh index 61a28b9..1158921 100755 --- a/scripts/ab-gov/ab-gov.sh +++ b/scripts/ab-gov/ab-gov.sh @@ -1,5 +1,6 @@ #!/bin/bash # Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan # Copyright (C) 2023-2024, University of Calgary # # This file is part of Meteorological Data Processing Workflow @@ -20,7 +21,8 @@ # ========================= # Credits and contributions # ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html # ================ @@ -107,7 +109,7 @@ fi # ===================== -# Necessary Assumptions +# Necessary assumptions # ===================== # TZ to be set to UTC to avoid invalid dates due to Daylight Saving alias date='TZ=UTC date' @@ -129,20 +131,20 @@ timeDim="time" # =================== # Modules below available on Digital Research Alliance of Canada's Graham HPC ## core modules -load_core_modules () { +function load_core_modules () { module -q load cdo/2.0.4 module -q load nco/5.0.6 } -unload_core_modules () { +function unload_core_modules () { # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT module -q unload cdo/2.0.4 module -q unload nco/5.0.6 } ## ncl modules -load_ncl_module () { +function load_ncl_module () { module -q load ncl/6.6.2 } -unload_ncl_module () { +function unload_ncl_module () { module -q unload ncl/6.6.2 } From c806428d3d0a33a1d040e1d68ec67a98cbfcc517 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 13:43:44 -0500 Subject: [PATCH 16/34] Organizing the assets directory Various files within this directory is categorized to be more informative for the users/devs. Signed-off-by: Kasra Keshavarz --- assets/bash_scripts/extract_subdir_level.sh | 28 ++++++++++++++++++ .../coord_XLAT_XLONG_conus_i.nc | Bin .../coord_XLAT_XLONG_conus_ii.nc | Bin .../coord_closest_daymet_idx.ncl | 0 assets/{ => ncl_scripts}/coord_daymet_idx.ncl | 0 assets/{ => ncl_scripts}/coord_wrf_idx.ncl | 0 6 files changed, 28 insertions(+) create mode 100755 assets/bash_scripts/extract_subdir_level.sh rename assets/{ => nc_coords}/coord_XLAT_XLONG_conus_i.nc (100%) rename assets/{ => nc_coords}/coord_XLAT_XLONG_conus_ii.nc (100%) rename assets/{ => ncl_scripts}/coord_closest_daymet_idx.ncl (100%) rename assets/{ => ncl_scripts}/coord_daymet_idx.ncl (100%) rename assets/{ => ncl_scripts}/coord_wrf_idx.ncl (100%) diff --git a/assets/bash_scripts/extract_subdir_level.sh b/assets/bash_scripts/extract_subdir_level.sh new file mode 100755 index 0000000..83a4cc9 --- /dev/null +++ b/assets/bash_scripts/extract_subdir_level.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Input comma-separated string +root_path="$1" +input_string="$2" + +# Split the input string by comma +IFS=',' read -ra directories <<< "$input_string" + +# Initialize an empty string to store results +result_string="" + +# Iterate over each directory +for dir in "${directories[@]}"; do + # Find subdirectories + IFS=' ' read -ra subdirs <<< $(find "$root_path/$dir" -mindepth 1 -maxdepth 1 -type d -printf "%f ") + + # Prepend each subdirectory with its original value from input_string + for subdir in ${subdirs[@]}; do + result_string+="$dir/${subdir##*/}," + done +done + +# Remove the trailing comma, if any +result_string=${result_string%,} + +echo "$result_string" + diff --git a/assets/coord_XLAT_XLONG_conus_i.nc b/assets/nc_coords/coord_XLAT_XLONG_conus_i.nc similarity index 100% rename from assets/coord_XLAT_XLONG_conus_i.nc rename to assets/nc_coords/coord_XLAT_XLONG_conus_i.nc diff --git a/assets/coord_XLAT_XLONG_conus_ii.nc b/assets/nc_coords/coord_XLAT_XLONG_conus_ii.nc similarity index 100% rename from assets/coord_XLAT_XLONG_conus_ii.nc rename to assets/nc_coords/coord_XLAT_XLONG_conus_ii.nc diff --git a/assets/coord_closest_daymet_idx.ncl b/assets/ncl_scripts/coord_closest_daymet_idx.ncl similarity index 100% rename from assets/coord_closest_daymet_idx.ncl rename to assets/ncl_scripts/coord_closest_daymet_idx.ncl diff --git a/assets/coord_daymet_idx.ncl b/assets/ncl_scripts/coord_daymet_idx.ncl similarity index 100% rename from assets/coord_daymet_idx.ncl rename to assets/ncl_scripts/coord_daymet_idx.ncl diff --git a/assets/coord_wrf_idx.ncl b/assets/ncl_scripts/coord_wrf_idx.ncl similarity index 100% rename from assets/coord_wrf_idx.ncl rename to assets/ncl_scripts/coord_wrf_idx.ncl From e03937f6e4f2e1fe3178479dec6667a42fa6f947 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 16:43:49 -0500 Subject: [PATCH 17/34] README file for ab-gov dataset The README file for this dataset is added, offering necessary information for the users. Signed-off-by: Kasra Keshavarz --- scripts/ab-gov/README.md | 129 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 scripts/ab-gov/README.md diff --git a/scripts/ab-gov/README.md b/scripts/ab-gov/README.md new file mode 100644 index 0000000..84f8225 --- /dev/null +++ b/scripts/ab-gov/README.md @@ -0,0 +1,129 @@ +# Alberta Government Climate Dataset (`ab-gov`) +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `ab-gov` dataset is located under the following directories accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/ab-gov # rpp-kshook allocation +/project/rrg-mclark/data/meteorological-data/ab-gov # rrg-mclark allocation +``` + +and the structure of the dataset yearly files (containing daily time-steps) is as following: +```console +/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/ab-gov +├── BCC-CSM2-MR +│ ├── Downscaled_BCC-CSM2-MR_MBCDS_historical_pr_tmn_tmx_1950.nc +│ ├── . +│ ├── . +│ ├── . +│ ├── Downscaled_BCC-CSM2-MR_MBCDS_historical_pr_tmn_tmx_2014.nc +│ ├── Downscaled_BCC-CSM2-MR_MBCDS_ssp126_pr_tmn_tmx_2015.nc +│ ├── . +│ ├── . +│ ├── . +│ ├── Downscaled_BCC-CSM2-MR_MBCDS_ssp126_pr_tmn_tmx_2100.nc +│ ├── Downscaled_BCC-CSM2-MR_MBCDS_ssp370_pr_tmn_tmx_2015.nc +│ ├── . +│ ├── . +│ ├── . +│ ├── . +│ └── Downscaled_BCC-CSM2-MR_MBCDS_ssp126_pr_tmn_tmx_2100.nc +. +. +. +├── %model +│ ├── Downscaled_%{model}_MBCDS_historical_pr_tmn_tmx_1950.nc +│ ├── . +│ ├── . +│ ├── Downscaled_%{model}_MBCDS_historical_pr_tmn_tmx_2014.nc +│ ├── Downscaled_%{model}_MBCDS_ssp%%%_pr_tmn_tmx_2015.nc +│ ├── . +│ ├── . +│ ├── Downscaled_%{model}_MBCDS_%{scenario}_pr_tmn_tmx_%{year}.nc +│ ├── . +│ ├── . +│ └── Downscaled_%{model}_MBCDS_%{scenario}_pr_tmn_tmx_2100.nc +. +. +. +└── Hybrid-observation + ├── Hybrid_Daily_BCABSK_US_pr_1950.nc + ├── . + ├── . + ├── . + ├── Hybrid_Daily_BCABSK_US_%{var}_%{year}.nc + ├── . + ├── . + ├── . + └── Hybrid_Daily_BCABSK_US_tmin_2019.nc +``` + +# `ab-gov` Climate Models +This dataset offers outputs of various climate models. Table below +summarizes the models and relevant keywords that could be used with the +main `datatool` script: + +|# |Model (keyword for `--model`) |Scenarios (keyword for `--scenario`) | +|---|------------------------------|-------------------------------------| +|1 |`BCC-CSM2-MR` |`historical`, `ssp126`, `ssp370` | +|2 |`CNRM-CM6-1` |`historical`, `ssp126`, `ssp585` | +|3 |`EC-Earth3-Veg` |`historical`, `ssp126`, `ssp370` | +|4 |`GFDL-CM4` |`historical`, `ssp245` | +|5 |`GFDL-ESM4` |`historical`, `ssp585` | +|6 |`IPSL-CM6A-LR` |`historical`, `ssp126`, `ssp370` | +|7 |`MRI-ESM2-0` |`historical`, `ssp370`, `ssp585` | +|8 |`Hybrid-observation` |no keyword necessary | + + +## Coordinate Variables, Spatial and Temporal extents, and Time-stamps + +### Coordinate Variables +The coordinate variables of the `ab-gov` climate dataset files are `lon` and `lat` representing the longitude and latitude points, respectively. + +### Temporal Extents and Time-stamps +The time-stamps are already included in the original files. The dataset offers +**daily** time-series of climate variables. The following table +describes the temporal extent for senarios included in this dataset: +|# |Scenarios (keyword for `--scenario`) |Temporal extent | +|---|-------------------------------------|----------------------------| +|1 |`historical` |`1950-01-02` to `2015-01-01`| +|2 |`ssp126` |`2015-01-02` to `2101-01-01`| +|3 |`ssp245` |`2015-01-02` to `2101-01-01`| +|4 |`ssp370` |`2015-01-02` to `2101-01-01`| +|5 |`ssp585` |`2015-01-02` to `2101-01-01`| + +> [!Note] +> Values of the `Temporal extent` column are the limits for `--start-date` +> and `--end-date` options with the main `datatool` script. + +> [!Note] +> The `Hybrid-observation` model does not accept any `--scenario` values, +> however, it covers climate date from `1950-01-01` to `2020-01-01`. + + +## Dataset Variables +The NetCDF files of the dataset contain 3 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load gcc/9.3.0 +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ ncdump -h /project/rpp-kshook/Climate_Forcing_Data/meteorological-data/ab-gov/BCC-CSM2-MR/Downscaled_BCC-CSM2-MR_MBCDS_ssp126_pr_tmn_tmx_2015.nc +``` + +## Spatial Extent +The `ab-gov` dataset covers the entire Canadian province of Alberta (AB), in addition to northern parts of British Columbia (BC), western parts of Saskatchewan (SK), and northern parts of the American State of Montana (MT). + +## Short Description on `ab-gov` Climate Dataset Variables +This dataset only offers three climate variables: 1) daily precipitation +time-series (surface level), 2) daily minimum temperature time-series +(@1.5m, near-surface level), and 3) daily maximum temperature time-series +(@1.5m, near-surface level). Since the frequency of this dataset is daily, +and only offers precipitation and temperature values, therefore, it could +be potentially used for forcing conceptual hydrological models that only +need daily time-series of these variables. + +The table below, summarizes the variables offered by this dataset: +|Variable Name |Variable (keyword for `--variable`)|Unit |IPCC Abbreviation|Comments | +|----------------------|-----------------------------------|------|-----------------|----------------------| +|maximum temperature |`tmax` |°C |tasmax | | +|minimum temperature |`tmin` |°C |tasmin | | +|preciptiation |`pr` |mm/day|pr | | From 8a899b05ba702ba3b7655b037cef0e48e64d7a51 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 16:53:23 -0500 Subject: [PATCH 18/34] Minor structural changes This commit assures all dataset scripts follows the convention of - under the `scripts` path. Furthermore, necessary adjusments on the styles of the scripts has been implemented, including: * adding `--model`, `--scenario`, and `--ensemble` options, if missing, for compatibility with the main caller script, as these options are given to the script by `extract-dataset.sh` script, * assuring scripting style follows that of Google's shell scripting guidelines, * the paths to the externally called scripts are properlly adjusted, after modifications to the structure of datatool's `assets` directory, and * minor changes to the source code to assure compatibility with the v0.5.0 of datatool. Signed-off-by: Kasra Keshavarz --- scripts/ccrn-canrcm4_wfdei_gem_capa/LICENSE | 675 ++++++++++++++++++ scripts/ccrn-canrcm4_wfdei_gem_capa/README.md | 91 +++ .../canrcm4_wfdei_gem_capa.sh | 199 ++++++ scripts/ccrn-wfdei_gem_capa/README.md | 51 ++ scripts/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh | 191 +++++ scripts/ecmwf-era5/ERA5_downloads.zip | Bin 0 -> 21373 bytes scripts/ecmwf-era5/LICENSE | 675 ++++++++++++++++++ scripts/ecmwf-era5/README.md | 60 ++ scripts/ecmwf-era5/era5.sh | 512 +++++++++++++ scripts/ecmwf-era5/era5_simplified.sh | 201 ++++++ scripts/gwf-ncar-conus_ii/LICENSE | 675 ++++++++++++++++++ scripts/gwf-ncar-conus_ii/README.md | 121 ++++ scripts/gwf-ncar-conus_ii/conus_ii.sh | 552 ++++++++++++++ scripts/ornl-daymet/LICENSE | 675 ++++++++++++++++++ scripts/ornl-daymet/README.md | 93 +++ scripts/ornl-daymet/daymet.sh | 365 ++++++++++ 16 files changed, 5136 insertions(+) create mode 100644 scripts/ccrn-canrcm4_wfdei_gem_capa/LICENSE create mode 100644 scripts/ccrn-canrcm4_wfdei_gem_capa/README.md create mode 100755 scripts/ccrn-canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh create mode 100644 scripts/ccrn-wfdei_gem_capa/README.md create mode 100755 scripts/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh create mode 100644 scripts/ecmwf-era5/ERA5_downloads.zip create mode 100644 scripts/ecmwf-era5/LICENSE create mode 100644 scripts/ecmwf-era5/README.md create mode 100755 scripts/ecmwf-era5/era5.sh create mode 100755 scripts/ecmwf-era5/era5_simplified.sh create mode 100644 scripts/gwf-ncar-conus_ii/LICENSE create mode 100644 scripts/gwf-ncar-conus_ii/README.md create mode 100755 scripts/gwf-ncar-conus_ii/conus_ii.sh create mode 100644 scripts/ornl-daymet/LICENSE create mode 100644 scripts/ornl-daymet/README.md create mode 100755 scripts/ornl-daymet/daymet.sh diff --git a/scripts/ccrn-canrcm4_wfdei_gem_capa/LICENSE b/scripts/ccrn-canrcm4_wfdei_gem_capa/LICENSE new file mode 100644 index 0000000..53d1f3d --- /dev/null +++ b/scripts/ccrn-canrcm4_wfdei_gem_capa/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/scripts/ccrn-canrcm4_wfdei_gem_capa/README.md b/scripts/ccrn-canrcm4_wfdei_gem_capa/README.md new file mode 100644 index 0000000..de8863b --- /dev/null +++ b/scripts/ccrn-canrcm4_wfdei_gem_capa/README.md @@ -0,0 +1,91 @@ +# CCRN `CanRCM4-WFDEI-GEM-CaPA` +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `CanRCM4-WFDEI-GEM-CaPA` dataset is located under the following directory accessible from Digital Alliance of Canada (formerly Compute Canada) Graham cluster: +``` +/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA +├── r8i2p1r1 +│ ├── hus_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ ├── pr_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ ├── ps_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ ├── rlds_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ ├── rsds_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ ├── ta_r8i2p1r1_z1_1951-2100.Feb29.nc4 +│ └── wind_r8i2p1r1_z1_1951-2100.Feb29.nc4 +. +. +. +├── %ensembleMember +│ ├── hus_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ ├── pr_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ ├── ps_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ ├── rlds_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ ├── rsds_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ ├── ta_%ensembleMember_z1_1951-2100.Feb29.nc4 +│ └── wind_%ensembleMembe_z1_1951-2100.Feb29.nc4 +. +. +. +└── r10i2p1r5 + ├── hus_%ensembleMember_z1_1951-2100.Feb29.nc4 + ├── pr_%ensembleMember_z1_1951-2100.Feb29.nc4 + ├── ps_%ensembleMember_z1_1951-2100.Feb29.nc4 + ├── rlds_%ensembleMember_z1_1951-2100.Feb29.nc4 + ├── rsds_%ensembleMember_z1_1951-2100.Feb29.nc4 + ├── ta_%ensembleMember_z1_1951-2100.Feb29.nc4 + └── wind_%ensembleMembe_z1_1951-2100.Feb29.nc4 +``` + +Below is a list of ensemble members for this datase: +``` +/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA +├── r10i2p1r1 +├── r10i2p1r2 +├── r10i2p1r3 +├── r10i2p1r4 +├── r10i2p1r5 +├── r8i2p1r1 +├── r8i2p1r2 +├── r8i2p1r3 +├── r8i2p1r4 +├── r8i2p1r5 +├── r9i2p1r1 +├── r9i2p1r2 +├── r9i2p1r3 +├── r9i2p1r4 +└── r9i2p1r5 +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `CanRCM4-WFDEI-GEM-CaPA` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +Each NetCDF file belongs to a single variable. The list of variables included in the dataset is descriped in [Short Description on Dataset Variables](##short-description-on-dataset-variables) +## Spatial Extent +The spatial extent of the dataset is on latitutes from `31.0625` to `71.9375` and longitudes from `-149.9375` to `-50.0625` covering North America. The resolution is 0.125 degrees. + +## Temporal Extent +The time-steps are 3-hourly covering from `January 1951` to `December 2100`. + +## Short Description on Dataset Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at the Lowest Model Level (sigma=0.995), 2) surface pressure, 3) air temperature at the Lowest Model Level, 4) wind speed at the Lowest Model Level (sigma=0.995), 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `CanRCM4-WFDEI-GEM-CaPA` dataset and their details are described in the table below: + +|Variable Name |Dataset Variable |Unit |IPCC abbreviation|Comments | +|----------------------|-------------------|-----|-----------------|----------------------| +|surface pressure |ps |Pa |ps |surface pressure | +|specific humidity@1.5m|hus |1 |huss |Specific Humidity at Lowest Model Level (sigma=0.995)| +|air tempreature @1.5m |ta |K |tas |Air Temperature at Lowest Model Level (sigma=0.995)| +|wind speed @10m |wind |m s-1|wspd |Wind Modulus at Lowest Model Level (sigma=0.995)| +|precipitation |pr |kg m-2 s-1| |precipitation flux | +|short wave radiation |rsds |W m-2|rsds |Surface Downwelling Shortwave Flux| +|long wave radiation |lsds |W m-2|rlds |Surface Downwelling Longwave Flux| + diff --git a/scripts/ccrn-canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh b/scripts/ccrn-canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh new file mode 100755 index 0000000..ace5534 --- /dev/null +++ b/scripts/ccrn-canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh @@ -0,0 +1,199 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n canrcm4-wfdei-gem-capa -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # optional + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# make array of ensemble members +if [[ -n "$ensemble" ]]; then + IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input +else + IFS=' ' read -ra ensembleArr <<< $(echo $(cd $datasetDir && ls -d */ | cut -d '/' -f 1)) +fi + +# raise error when $model or $scenario is provided +if [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + +# make array of variable names +IFS=',' read -ra variablesArr <<< "$(echo "$variables")" + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary global variables +# ========================== +format="%Y-%m-%dT%H:%M:%S" # date format +fileStruct="z1_1951-2100.Feb29.nc4" # source dataset files' suffix constant + +latVar="lat" +lonVar="lon" +timeVar="time" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing CCRN CanRCM4-WFDEI-GEM_CaPA..." + +# make the output directory +echo "$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" + +# reformat $startDate and $endDate +startDateFormated="$(date --date="$startDate" +"$format")" # startDate +endDateFormated="$(date --date="$endDate" +"$format")" # endDate + +# extract $startYear and $endYear +startYear="$(date --date="$startDate" +"%Y")" +endYear="$(date --date="$endDate" +"%Y")" + +for member in "${ensembleArr[@]}"; do + # creating yearly directory + echo "$(basename $0): processing member $member" + mkdir -p "$outputDir/$member" # making the output directory + + # loop over variables + for var in "${variablesArr[@]}"; do + ncks -O -d "$latVar",$(lims_to_float "$latLims") \ + -d "$lonVar",$(lims_to_float "$lonLims") \ + -d "$timeVar","$startDateFormated","$endDateFormated" \ + "$datasetDir/$member/${var}_${member}_${fileStruct}" "$outputDir/$member/${prefix}${var}_${member}_z1_${startYear}-${endYear}.Feb29.nc4" + done + + # wait to make sure the for loop is finished + wait + +done + + +echo "$(basename $0): results are produced under $outputDir." + diff --git a/scripts/ccrn-wfdei_gem_capa/README.md b/scripts/ccrn-wfdei_gem_capa/README.md new file mode 100644 index 0000000..1546a06 --- /dev/null +++ b/scripts/ccrn-wfdei_gem_capa/README.md @@ -0,0 +1,51 @@ +# CCRN `WFDEI-GEM-CaPA` +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `WFDEI-GEM-CaPA` dataset is located under the following directory accessible from Digital Alliance of Canada (formerly Compute Canada) Graham cluster: + +``` +/project/rpp-kshook/Model_Output/181_WFDEI-GEM-CaPA_1979-2016 +``` +and the structure of the dataset hourly files is as following: + +```console +/project/rpp-kshook/Model_Output/181_WFDEI-GEM-CaPA_1979-2016 +├── hus_WFDEI_GEM_1979_2016.Feb29.nc +├── pr_WFDEI_GEM_1979_2016.Feb29.nc +├── ps_WFDEI_GEM_1979_2016.Feb29.nc +├── rlds_WFDEI_GEM_1979_2016.Feb29.nc +├── rsds_WFDEI_GEM_1979_2016.Feb29.nc +├── ta_WFDEI_GEM_1979_2016.Feb29.nc +└── wind_WFDEI_GEM_1979_2016.Feb29.nc +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `WFDEI-GEM-CaPA` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. + +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The list of variables included in the dataset is descriped in [Short Description on Dataset Variables](##short-description-on-dataset-variables) + +## Spatial Extent +The spatial extent of the dataset is on latitutes from `31.0625` to `71.9375` and longitudes from `-149.9375` to `-50.0625` covering North America. The resolution is 0.125 degrees. + +## Temporal Extent +The time-steps are 3-hourly covering from `January 1951` to `December 2100`. + +## Short Description on Dataset Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 40 meters, 2) surface pressure, 3) air temperature at 40 meters, 4) wind speed at 40 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WFDEI-GEM-CaPA` dataset and their details are described in the table below: +|Variable Name |Dataset Variable |Unit |IPCC abbreviation|Comments | +|----------------------|-------------------|-----|-----------------|----------------------| +|surface pressure |ps |Pa |ps |surface pressure at time stamp| +|specific humidity@40m |hus |kg/kg|huss |specific humidity elevated to 40m at time stamp| +|air tempreature @40m |ta |K |tas |air temperature elevated to 40m at time stamp| +|wind speed @40m |wind |m/s |wspd |wind speed elevated to 40m at time stamp| +|precipitation |pr |kg m-2 s-1| |Mean rainfall rate over the previous 3 hours| +|short wave radiation |rsds |W m-2|rsds |Mean surface incident shortwave radiation over the previous 3 hours| +|long wave radiation |lsds |W m-2|rlds |Mean surface incident shortwave radiation over the previous 3 hour| + diff --git a/scripts/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh b/scripts/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh new file mode 100755 index 0000000..d9a8406 --- /dev/null +++ b/scripts/ccrn-wfdei_gem_capa/wfdei_gem_capa.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n wfdei_gem_capa -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "$(basename $0): ERROR! arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "$(basename $0): ERROR! invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]] || \ + [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + +# make array of variable names +IFS=',' read -ra variablesArr <<< "$(echo "$variables")" + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary global variables +# ========================== +# the structure of file names is as follows: "%var__WFDEI_GEM_1979_2016.Feb29.nc" +format="%Y-%m-%dT%H:%M:%S" # date format +fileStruct="_WFDEI_GEM_1979_2016.Feb29.nc" # source dataset files' suffix constant + +latVar="lat" +lonVar="lon" +timeVar="time" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing CCRN WFDEI-GEM_CaPA..." + +# make the output directory +echo "$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" + +# reformat $startDate and $endDate +startDateFormated="$(date --date="$startDate" +"$format")" # startDate +endDateFormated="$(date --date="$endDate" +"$format")" # endDate + +# extract $startYear and $endYear +startYear="$(date --date="$startDate" +"%Y")" +endYear="$(date --date="$endDate" +"%Y")" + +# making the output directory +mkdir -p "$outputDir" + +# loop over variables +for var in "${variablesArr[@]}"; do + ncks -O -d "$latVar",$(lims_to_float "$latLims") \ + -d "$lonVar",$(lims_to_float "$lonLims") \ + -d "$timeVar","$startDateFormated","$endDateFormated" \ + "$datasetDir/${var}${fileStruct}" "$outputDir/${prefix}${var}_WFDEI_GEM_${startYear}_${endYear}.Feb29.nc" + +done + +# wait to assure the loop is over +wait + +echo "$(basename $0): results are produced under $outputDir." + diff --git a/scripts/ecmwf-era5/ERA5_downloads.zip b/scripts/ecmwf-era5/ERA5_downloads.zip new file mode 100644 index 0000000000000000000000000000000000000000..8db3e96afb4fc77e5891810eb75ef9051efed4b9 GIT binary patch literal 21373 zcmbT8WmFwV)~;~~?iO5vy9Wyp+%3V~Ex5b8ySux)OK^90g1gIwp6TgyPo}?_Tj596 zI%`qX*}Lj}YD-Cp0)rp}K)n7Fc4(6U_~*C+0sz1Uzyr|vW@l#dO;bmmUxtTCU0?T` z`8R7_GiyUl6KZn@c{vyW@FuAm4cjnxO?w-6O&fbC06>r%KmdT(UkL!9D>ELrNDJ?L z^EE$v4Fs|5NQHX#6Rrrv5}zo()IGY6gK5w}Bt;30_k`QYK&uKFQ)I-K&4e=pw2fdG z$7)EpOhU{~!1U><5cymcxW3(CVaI*EU>BM;_LnTWV@HhM0Rzw&3zSQGc*nT`(aq+G zTA$3usbiy%sH+@)7FJ$SB~1IG)R8pX7cAW_Dq(AGR!QDz9$2N*D)<4Iy^=55YdJOf zLhMOI!dWnZVFW(k09^n!qZtWl)`J0azG6R;SQySzQ7FcI=--ea`5v&T9+A_RmUWH( znR)S4L`-QR@McuKE%{4;*76-CJLd4xp!^mTF!}P5cXiFFklH3xpi-omob4)&Y}kZ| z71T$QvB70u1{`R_T;nr}=Wj>eHds6)ra_(`u~&%c0vO@l7V^L&1gd^GmSQ?*EV@@< zB|2Gzy9&gHNySP!wE9K%Aw}bABz1M6Uc*V9=GqTApmK<&DwAGsr+9@(7VD<&CN-~h zz-Dnue84;;KFb+Zn0WFYw(6yY_|DVdvk%sjQhL>xil^A%b(_y!B9ZRE>z> z0RNq{ra=p@!kJx3d|c@`+ze?+2)hp}8ADr;_AtgMW;k#u=#b7xiAcWB+_fyPNQKx9 zKuQ!GBB`&;eDT%RZ@wb+FE8SB-v5^uSt0-c5dWJOnOo{wS=m_Xis{$)IzO*iHRVI~2N)a@@^!s!+kix^-X-a6uxJj|PcrM=7Trxx+p%8*1K)8kqY$(~J z=jakCj32o@W`dGw^>N>rn*-1p|UzgzSosv3e{|irrAdi8t@>1(5~dE(UhO{ z@H{Zdv_2m{FBk0=lYBzQrBamDA-^)%=uj6hF^APPt@H6LPH5d?8Tl zk>~Bzg|Q2ML_D6z98)ay_sC$e)i(*MKybpUYKUu32J?D<3rgh!!I?z@&1f5ISp&E` z*W(Vi6b!mG0PExdV^|PsOaL*WFf3GrNv{$lvVgmYuOyQg@_x$YgYVo9FYhR5Xebgs zv4*YBXkk9Gm0D>a@Yz1AIIe$W-ED)!3lnqB&=D;<_@Sp$1N}i)Vy3pA6FvpBrmBH? z*w)~?wQ)*z@@iC@RdKbbvhyJ5Y>-Yiq24>&;G3bI2ZJ7AK4u2YkTsi(>-?66Sd4PG z?0ITS?jc4~RkFGkMgEKEDBbj66n&xOFt872(exNBmK!YJfg=Iv9NuVrt66I zd+J?x?sjz#8Q#wPY%P4$4E%NH(LB6&r^D-Blv9>*MM97VCy&&y&#G8BTHsf(ra3Vg z^|8E2(BbGRI{j6gDCN9K@hqS8AHj~{z64F(*D#ue*BCZ%$`2gnJ0x;)*t2EeY)?vX z8E(Oxv!nHWkeLPQDWsN)r+VL{5~?*&!wYBVE!F0P8k)vmD%L}`GXX>5db^I?!Eitb z;e_DfD+!M|!fl0}xAEvbj5Ct61+ptQ@qOA)ChGdfO25x&Fr|8#bX2=j<2YLmoX8K_ zcSBb0aW09B=IjAcKdjQ;rLlGF*Fm9G!FKy(!fk++dfH!&@+2>s+i6iapu2dyD+n7& zNcODvD8f@~ftvt%D6#(xtKLMMRN2)jMxC_W;=Z<-XYqj6Er)7=;g^S;C0my@DJ766 z*GUL|#5^>vG-b!R-_%~yL+m#Q!tE-92-f0>K$S&|(53)zpHo?+a8hEhAHxn!*3I7! zTJDeoGR12O?<*WV)X7!{>fBs~`BjF#dfv0pmSW0zXd+cK_lTFVL#=MB1yjHD;H!$| z?l9S*wq#dZLEXrCrm?kF-kJgQ9YpYw6U4G3+(N10S4;h_+x|IvW4&86tOZHDl_3NGpU(YvQR^6zO(M-H z+A-bilG|KuDGosWWPjve=CIa(8D-LILALXP+QYTdMhDw9m7sU3Fd44XDH{ux$CQ#` zN+F!S5+t>$&}3v$Ffvp2G>d5GB;q1 zuU#PQW_}LpnCFNFd!bqIuaw)F3V~1!`x0$T$e|$Ti%M0GOed?ASY|vMertZ>VA8n?}w$*tx%zw>8?nBps1%%9Ftq2N7ToX_nr6Qa0C|IG5qPAhI zR7)LU)nr$3P1B_+9^?ukLiN68hs?>!}5p1iFrU&?XXvT(4>-as9N`iLUeS4dGov9|2OUB}+JTJ0WAd4~Io`c72 zNcQ9#Pn#d@Rc5-NCR}5hkI=@iGwBVi37e#-;~gZ!+SCg3;^Iw?|!X> z=dU_R`>KTM>f$_nlClak)arjmtN%t6Tge2p{(@K`;lmXJn*pKE*~m6-heuTq;+`NUqp11&{lNtO2QsK+C4w&#bVEK zv%4dHf_oZRPCu6-Be+>3W&hxQo?!TeV)eAl0LJfW_^57^VsZkuvhFMc+WfdL!&kU7 zzn2z;K8MHMQ*8+@5W~R{xPHkJA0?h{PT}$+i>GfE#`e^#>PJJ4nrUl(NX|pVM3^a=)Lnwme`S;| zE21Jd)(~2tYdPkUz|nveF+m7KnG-il4uSaSiH$FNk|u{p(oX-pLj{dc&mc2vWOe5Hrwf@VU4 zHVSAw;k`xz$}{2XB61VYbFkx;6DBHIIIL)?9^x-F3N(y->$|+XVMw`)0pau?o8Z?f z^O2*<6;Pedi!Ofbc&MDCrX5_WYs(HS!FpM=PWbFviNxgkj!<2l$} z%VveGmU+TjUzkW|`d8Xa{%X9wQB-%=f-vEtY*K(kjs^07+j( zK2c3!YR;wVAyiAw3bSsudOd)r_Qu3Om$9T59w9QNZB~Ov88*?v+!ZmSm-QwbViLq1 zI#>-o)TFPkPOob;?5H0c?5dH4SeGjZnf!r?bb|yxJA+Umm@RTC)w@9`pW~% zu7iKz&N7$OM7{7`9(=;%EACAGhC4LxoJRIWQs7JR(LpZ<30LasQ`U8&yq#cXZksOd z4)p6PaoCcTz;kHM)*OXr@2+H4U`8_B4ns82ldq-whl}`)TWzjAlPW}jMJjWdaz4I5 z$l^~BL>Tq|#Rv!gd0fj(PG8xR+^^F4-DLRg$ z!poZ{VOx>F@A~FBjm@VMmXN_>RVBIZ?Ri+rZu;BveJv7}2pL z5TZC;k;Zsk`7B$jCm^wOPKnSo^k#A9cHPN=$EZzWC{|P2PI{F7xu`dUkbn z*^fku>>GXF`?d(S6!NaqDICz_`!RMQSNp4{!_OmVMi~@b(gn(>BHtIb^gesh2qp(1 zQh||NNMSy<84wQ0B*@wkSaT}Ll!lmplg3c0Uc=JP?PgS;F6BA1K5=xg{4BgQ;J5OM zI}ZQA9TkT++)+I7l7rOt!~h}n+-rtB7BzaWNYaqadl#lPNZ#BXS8bw7Z~$Q=%(qc0 zzc4^Uijda(G0aR&Rc7-#rK_?wzyqj=+|^QQ7j!9)Mhq= zqRrob9(bBOX(p)ShmC1;r+WcGpjuNl$~B2uj~|((D-O+@V`$iPV&qq?E+lr_6bUROBb;uiN!8tty(lM7yX=S={AEsbxYN%Ja=|0VDI zQYC+lI})9*xRZyz*%uIm#SwuXW?EO{0|d^OPxJv1CVZ^~V*wPi(RBV;r2YYj%Tsc# zqvOuOcLQN^7L-_cOcKRO6oV(hJ*>T|N z=D+oP#i4Ku_bP#vcemo!#LPAPTm!*?&wvR+y4TZ{e<;my0ubjED(%7Mb+p~yf)Xu` z8a>G+sqp2L4QduZ-^rCYLYuNPzB5Un+(6;MJm^RxQ9$zpgoAwZz7;c)q4Ml57TWjf zX0Di5+S%=5#J%G0%;$>hq{rrr3SK8#A5@GkZjqD78k;Q}OCnx^+iWmPMM^l87IMXK<`5&q*}&Gi#4X$E_hBo*3CiMKu-ZXE6}X=l=#Mw!x8>m zsom85p;oh~bdyoOXmzhZRV%@$>LlEEy3SOqb`AW^q@Ri8m%qFk?a#DhQ+@caU{2|) zu}S_9+OaY;(>Kvo6#pG~aL|wbb2w))bded}<>sqc1*i^Ft@C`g;qny_Pop6ekwHYZ z&@~p}70eP^5d`c-?PdT$mdn4tC?Dt~0cVpW!Fc zzmvDSn5nWimSCQQNMSVOI5y?5hc{$$>8YAYvq8C7itSn47&^mQNofV8e_dm4xH<(k z!2WRf$u0_zNEnfY&=gc!1*65s^H`_Vd*M{jxIAfMyV5YvTn-xBB*A>o60@e3d9+0h znSbm-k(<1LUr!SVMJC>VY5CzR&77orIsWI$%=GB}jt?!|B?_C+fdi)1p2f$lYg|E{ z=`}eWb@hA=@FFq1ANWC8f}NpQ3T8eGqR^FJc28okaaL;i5#k)9#>*%|6$M+VF-0!L09+o!qK>*M|In zmQN|+#J%McV2@P8I}jBNw4?IqgoVdq4ywa z!GQQC9s%|j4;~)YK81t({pipXgJn8)!fpP@5trFj%JngxYG0a7n3#y$^nR+d;h5bX zrwbw#&KXElW+;b695(ksJNr{v5YZAp5j{5k^y8|zW8Tn{|J8@l9G$1CSZGUEg7pc3 z6^AB>iSOsrkT4jX#j_tsp3QoDtPJ@9M6Ivm(?CDXyr9D8@(&E$BEzh}N z=hkw2ypwHX!wkXnRoCvimSygv%~+=wGcD8HcW*<2`;KpNt(%fC7vIm{^0W)@@=R~U zCCwY)6+@&G79D0Ar+F|cu&jivXt+r=V*$aSriNCLZ z@rfm^HsHVu$!6hPIFz>-N+R-#uzw7qrq?wE2wZBFBdMD~UCdm%J%#mVqWl0R-?Hg* zmmkAXe}8qrePIHBc~!8B_-0J%te>p{h|#ln%LT}eDP6R^U%Rj5P@>4vnrtgg{q;J2 zy@tWLeN)skwH03n6jVZcQ6`=Wx?rpW4uRApVFTIcP}>*WLNOgT%swXe&QylRJcLO# ze_wr!jD}+d#N6TdxCj3 zV_WyznHV+e_8zL46ZyLB@16l6vu1;L-a~ci>s^k^U1cnovE#IZM@ijkOpihLb8Mnk zlNm9DeBlp;q(p;U>;NoMvmV!h%xPejsHYUN99#z8E1sq1NddPXX?@6m2A9d|!0&-^ z(aPP>CAKt#`vE$ycSo6RF40TkTfOCJHc`uMW)Gx#Fx(tOe*o!~sbFJ|b35}E&Veg~ zds3Q5jXZVKoPei6e6DMIemh0|%fZBdHBxLZ%>X zy;5bA8@nf4BAm<(BQh#q`~KiICSxw;g7KUb6}bv2)q$-EE0dH0oTlI1@+IWzsKrF81t&EJ%+Dt; zs?OGbsrYfMm~~sbLP?_Sv@SnFAfP zp$aZM@7<6li)a!l!hRpei?RftHCbFN*8$mD1MLJ*uvEA?RlD8)m7q;i{+%8fdYr(i z)?l})?y^MVvh5N@DP;M!V@oemR7@vMmB|b*+n}$8*qP&lh4V0V!iI5&b^lUfC5~&i zEi{r=`=n0y1Y-!hurVx(Xx40TZ&WdAG4W$F;hg*?`Fu$ima->owI};JxsYkW@`HwX zhg#cRd$7WWif`1gp2@a~slFoEW^O|gsjhi7ai~rt#KN6w#x@jUV{PmdghlQ(7w8fh;ye|e$gLW>Sqv=)f*Kw6VnM!^)hsoe0em05tb@l;YcDgEVKofYxr;ogi8s7>qv&PK4 z?SPB|qN8b%g#@NpfeEmuw`V3D{DU&dMKwP`_^BmAb`WM?#se5n+t1HNqqcgxe3;C` zM?6mHo}_63w4?2yrg_L^hnt)@>F|zfV-5H9*-B>q%ssOM|ACSNdUC>v7VQwb4MC@a znq*N#pqqDO@lhb;@$&1;QzkM$U?>*OK1-wB;W(#p=rI|h-`tsSRh&)vfksdn6&Hsy zB}gZaF#08AumA9w*A9v3r?$sl)#3X!d*zBpcmUvoL?fezKs-hjsvdKCoQi&;)H^dt z)g<%+4!$YN9=R$t2<>_ANOOAABNf{~aQdE-G&W}gV8S8nl|zMRwU!Hd8oMr$eGE^a zsX+lv!KKbD{3*u7aFG37RoxZc5nORyjncJ-=$1m`_DTyK;lZZ_h>f!Z0*8Xa-fl|oFmsk+CErWZ6TH+038=id+6f-adlTsQS#Mmr@?P-C z(?maQ5FfC74~8LJoiNcBNeyx!DVjV4Op%EZOi>|UV}Zs-y_)t#RO9=Q>!m(aYZ}}% zMBx05%e|cj&jwMrslMybcZe6Nh`>*}&xgmV+RvzEs7WTBak!8S2GUU1(5ZcC)}49W zVn!q9Np1+T8et07{&)@;pe8BvMd*I6;Ja9P`J8 zVE(f1*KG5LG08BwvWOO)|N4)5O-3H4M|O=RGEtE z>wAo`t0?41XB6w_nH+sigW^4Ckz?Fk-?!Oj!&?T;2hw0zT9YVu-@8)Kj-tlAd`XC_ zY+xt6fX4JS+I(h6o(vsiTZ;W^v)!qei1T7-l?N8`IeJF_wrn&$k{oeJsE$G9(m|*c zBa9)h>c^%@#qft+j?YyX%vg*%+4EuLQ%2pcC~_lQd%kL=_56H}gd!_KEZU+4XewO6 zVqGa;K^epZ+#&>@0b{Mi@#)V?+|ISvg5R~1yob=~!X~?RC>2WRjzlHtd9t~8Zmsl< zrEUsi;N=@+lU9(Z_J5wP4T2L-^-^J$F|*TEL(cp7*)E_-WeVN?DP*6u6<%^G^hHzpyE6I3X-25>_3lI)&+>pDd^9@Q?k@{oX;f?PxuY+BT5iFH$<2@C;2#Xu= zn~u)4z&Q-R>gbzV{#r-NUjyu{e-}LXUnKPxQ3P!b6m5MCEsaxgu`j1^U;L%qfx^*& zN#NfDT42L_-M&Zn2MWc<=aKp#)CYuz`boCaA3jcGWOQ1TlJvXq^q?>$H5rwJxUev# zuskdT7@11`k64h*(Qhz2z~iqg0*6?59-LHpUBlZt{#YSw+~mLR(u0V<${hS+S-)ST zIU`5@(?vQ*K{{-)9>MjdES}FXe7b78=Ek$O)gF$gau&6I2}L-FYYP>OD#S#~P==W7 zq|oo-Bn6*XapkI!hk{ZofZp|K235yQ9H$Vai1@P&9;|? zOgn0r%SG>e8sTG6T0CU-fgs0OUrQqjo=PVfOQ-wo;#a*Wg%$B3bfuAD*0QWy>y8s8 zoJg#Bj2h-oe!xUeG4`1qa2lOxhy`M};BcugcR*Ip5ym4vN;Cp+d#GuSKwdJZeTvmJ z)HkI0-ZPQNDR-H>1*X)o>gcgJtyXB3yk~&^aR73CT(ojQr0HaQ52e}Z~x$))VZ}jPVP#ufN*5{UW>!nfPZHP7G^d-(Y_gXtIM!71Uxl$x{*P@k;_TtICt{g z24Z3FNw9BbO6v}r@z+I}ty@AIsS=-=8Y#u;_V-||9xM%XwQCAv<7if#7kWG0fchp! zU~|NJN27a&PqZH|-2-}7!l#rWiqnRDrQ<7r3ZCrC_sK{m=(IuB$31DMay@`Vr72!i zSw6oLoU{nC-3uhN%aNk>_n3c2kj*LGyf)T5m6)RTZHSOH0_5a$fcA(t4%1-&Vc@XOdd4iN6R|z(Sf}y zwfz7@uIKYQ`{31$GioR)T*D}(BmXH8DW*?&bRbG(`M}e1+N!9w0LE>ei-P2s4Pixl zop{C*yn=9K6(nP=YOBlqpq?D$l%bGcv&S}Ep|@%-o4a9)z--GE>b^9H=|yYzh8lVo z9i6N%BI}(V5k-$t3Kn=8xFRp75b)pl3U#*mf6he8o_WE&bDY0Ld!+2+TWBDobrn^$PcuL%P4P1`&UgTal)UDwSYZ7@vcjBf6|X!7Ihjc zb)ts;m||}f>wK6_B3Nvj!T%M$di?{(CLvc;XrAEMcD{b=@-ea*(f-^Cux@$}*|<-} zvkajOWWiEJAl@Jru0!|6<9-5OSp!?r%_bZ>tOlt|6=o)--`yulW?=5H=UN>4fRoUh zXp5v_*h>4b3Wu$r$R|6uySpyadua0A5@hQtJCO-RI*HMFE0mo|lg@GM6i#Rb9InqY z214I7p7XdLE$+ld;FL6txzq~=QIzs~g;{bsW@QBvGRzC>lWhB|`WgLS(6*vOMH-2u z$f}2}`7l2A2!<_;%aDw2j#N?4oPlOVRaL5whMq@_<&y3%mAS}jR$@6OC-*^CX?bY5 zgmBReh&F<2=FE`_ygX)p8C!}va!tFyafD}1CSSY)R+X8FEyZ0YsHK*FvhUadWvt7h z*m!~74M8&!KZqR^xo1sK?tfu!u<}YGT$x9^JfW8A*9|b(N`q2zzS+)cXxunh-f*ln z)VmO+=vKuPXJv~wOndq)cJ$#?ax3z>f9lbK0xL!IWHZL}ow6=7Z%goyshYV@V*t+qiw+wt0AjHQ@59rQ%v>eSWV_UubxuB{YS9VZ-{vL$3l07SQT(e6ga2$Za}CYDFu#weNw~z&Df|l|M4%RhBh=*Z zwbzIF_(LVm2cT9@NxWMw92dZ}_7Eym4mMichg}wi!}iA=plAT<@EDrK?WXf7NSN4a zc7Wcvk$cqzFE_=i$%a%pe!q%Rmn%4WmT^r`ji7+JPR5Y>>GiM)$#@7q;TLcQTbr?# zMTQTZDuu~@_#&-N)2YD>D%dN4*ZL|$#71BTSwT?@{hNz`qql}1Hk#MCZCHWk;?Hg% z8N{||*Ucx{m|DPWC}25@9b#&v-v<$YW5?Ylu$wf97v)t8(YRte>900he1Zl?S9Vk> zB+`%HTSnz`!qYOli794HO(|mSRgMU8j6er3*caf_YeK*L+2nO^SekIYFwTmZEhdc6WjG*i@3kwd-+b*9Htpo*Zp%!Ig=^# z{7spC?4%l_nE063gQOC@qi&8%!$FelA0J^ozP-_wyE&(}{cGmp&9r|Q9W3pSIt@xB z006?jkIVci0mVV{AImgy6kdx^>k*!h)#>;h!)+?IYogYaSXKyxu8$#A76=T-;okBV z5#p5M`$7Xv`|Hn6`a;67b}eS+D5w?P7+e>f4%%w%E66pxERhcb3UChVpNRq0KADkM zEA7-|eJ!E8nb=DeC!O*sAKHV7mO+#9?*%-<7do=T&aCFn~#bvL4_F7+8G1Y4m$WojM{sQ_K{3~im8W%#Bi zfo0x1QR*SDRrchbIs%u)dlRp42rAs-vvyTAP7ArToCfwkU16bYW;~w-5wgX@(L7*89B}q>bjW1 z-`{yphGt)kj_;ObjfQCfv0+!F|FDKU3E@f)0b<{FAI6)Zw)bt#HPtQQ!`DU~RF)X= zMC9S%*`T#EkD(&vF*imaYf6!}$@jB9M$pGLUs^~CN87wjpcecD*K^Ch`(2|nCzsnH z5HBO^B=(daqh&>uQ@XQH{TUVROqpXz4{#qTr)4@cS?OIN4a8^PwJ-GhpX3)5;(U^| zj?v67KIxL)oFS_!=tz9r%g4{@2U(c?T2~k0ffoiHMA(HwS6!dxd7*T}v5Tz-v(llp zEVA91Bpr;wo{Lhty4rd03SN$gftImZavqG_k!F$+nT~u!>uLN16Ype-UT&5u^WQIP zC-uva!CHnA2ccl_j0?{@LRcg4Wrgbo#6pbkm&71*=gYMdm4694$S7<+jl^P9Zw}}< z{T}^E9Te`GcARdOZ&lUT@JEq&%ZXqMpT~ln_gaxKOLM@#O*R~>-qXdtWIVOlg$mhb@1T{$(C9b`JkbsCj z^AROa`HZIz$SsLZFo*_BrCXj1>w3<{`|lZoZ5) z=kx|Kn${2=qGW+H$qe}FB#yN;BgzjnVtd#!rP1B3x;Rfp_8jSc2~DH@8=^>-;#KDosA=Fafnw=?WghBxkKkc^I=DL44X&+-;G%vXwCHZ&8KJ4&j);34p@5ZUqJ5Tpdp)lw&tEIlAgPI)S zXh|Naf-cH1^8qJeXc-po-{4ES(%}*f^x?$qaj`Gy@}1mB0kVD7jfl(vdfx6<1EiHA zb4HY)E*vlo9ImBIykwc?bKl8xxijxBc*$$6nKEXU4{D*sgt+0^@RYO;X)-y^tm?TA z1J%Y0C|k~ma2_1cqo`(u`Vyj&SxLUnE=Ysm+b))E+oBf|gJ%f9n+g zrdoKv?l}ZyU?ylcw)zkD+I?fM`UH}}mzoAyFMZ;XL#W3(G?iEOTKdk7R)1?bENErZ zbJN#TG@qL=Op^i+XIj0UEh(G&6vI)I)ZP{@x}Yz6(HCMq701>6ehIO5z{J}TCn2vj z9TJ9x;hE&!ckYN?XVtafQRZf!JM(u!r;|>I5SyvdzMp4{ho&k}^?RNiS6Zc1wT-_n ziwC{3*Grf7PF0B3ZMFEORm$#XYGZ!1m(m$+i&{x*IIb;)aY`)L`uIto6R|r3;w1%9 zq>)n@H&1LOB=sSaNhC0N4i_aQLh6fkjQw)pW-Mm^;|#=;oYJ$Y>?%-mz7Fdf^*0s9 zLN_}=c}1@`t^KtMzv(mee;w8Nr6B*3yhKHRD}d!cvl#b+JC&27;f-9*t}WmcDk6hG z%_-TV5@T5e7q6t7zp}W7xmQ1;7vIcmm)N5N!k$)wx9G45YO2$b|V2!wclGm=}8+noR?}0O1zmk^| z)W68fZ8<6AH}Zl-`;ELLb5ky$UdfAY)Sey5LhTgyV=|Bl&8?FU#s|wt^ZFlk;$=L` zfqcd`WtKV!l(FB-rdtT$CJZ7)c~^opE{JglF1u?5F+%BTFYD!mM`*{JsC=&#x|eXs z6fHQ|zgh&>5hTG9o}p(O!HdTqY0)BMOybs(fo0f>5qPYZT~pjH?^uKvnfM2*pM&Fo z#uGsjQn;w8rr~zPs%mF4R#Irv1`BBQ&EyzZdOUXj241rne}I=XFaW@7eG-7arKW+V=^qu(e=c_Z zhpDX$erqR-WB3PqS^j$_Ok76%+afJ=%fVOk0FMvfRY;B?t^2Kos%$um+sDlOQlJ+$ zv{+(wge)r`AI5DX?+Gpj!@|O_+=MD8h%oMXgVk|o+L$Hb`@S&;!5BquotD~efj3B5 zI^(Q5D~-fm+~H*i5BA*f09-6Fe-QdQdp3eLbL>6#g$ad6SDMEK+J~wC=&A)?ke4J2 zbXS~I26qe;V>wxcUPPddpl+sMLIaP6do;@4VifNEBZ^zeplFr2F|?1Ek^U+vT))$< zj<(H{wcPsy(s{WYz39H@v@&$syX(rMes(#D zfY<)Q?3=b%UWh)%p$L|{Z?m|$7R~_Cnf-q_+b_?R77<_`_`Y3&4MFWsQst2TK0bAKh}tqj z?I0jtk;&c1@k^$b_)T?#uIS3@pH2~+fmb_vN8&1~EJm@wWY}ogo7`M4)b%u6z5zo5 za0ZL8culyFg^77l#mNJl^vK^_udJ(&!+Jh%TrZw&pId<{4qjoqKM9%R=IBhp?PdMQ zj_}X>p);Ol!SC)x+$HtC$Pg`WkF-_I384!z#O#c1(tIycB0z`R3NX=qc{(AL97R`PU&j;!gx_E#E8t{)W<;=CUqk+xc^;%3GmWOt7Q zg%CT)5)@?1nEh*$H>L9D;{zN>kNm|6WxC8M<+i{GP@uKxBoo?iC1sga15;cd7^4G? zOi@~L;iB(8&M%D!UY<{*eED+s_*-lrx;}=5{?#OJX8FTl!b-RQihQHIssP!m!T#P# z|H_a5^Hfk4ef&@1Z+WQ>iLTdEVV7cPhER0@CY!Fn4+fvLT^A2uA|@%yZB@by{GklarbtG0s+r&v;zS~U7xgCkcd=sd3C$CG z+2S|^IK9T|k9}>$c!cCGLdus_%}mzOoo8(fk7Er@+lh-3WRFC$XfRiXp|B1TkQ6D$ z`KyiC`>kZz1gJi^w_Q($Br(_QzFXFiQYp{PvCp6k-IR% zKIonmsvG7pBPn2P~-tWjqM%(dHlhcag z6r{qV3eh5!q&E`dlG7p+;$WriF#JK$(X-)=ED?i5MS^5yWe`PWVt_q)0(tqCtf`7& zLd1iihpDK<=5BgEczW*|<|}y_ye=CYB7;2E-{*CqZ;SooYsicGBZkyTe-!D@hnFL6Itg?;6E|U61YBxZJ#Gof(AlD~NzQ)k5P`KP&a0 ziDdUG#28<0ZT0aZQ_nh|`m}9mm?R!(t7MGo$SGb36olcN@b@inm(4^J4I;X|clwCD zgGh0SPA02Wy4lij_=t104u7RHPXBz~=oCP89pkw?W<%CD`J<%lbo|(vE$JPPp&p*p zPA0pqNkS2oOCAFXt*ad0oS+!ur5hb5ts5_@CHUrX%_n~ZW%N$CY6avWE#qNrcb612 zNN0g|XTfuGfh6tDR0$f8#C&0TNnWyf)B%=L^WXnIh0+wp^)D?$TU5_J*4N>I`9kxP49CJKMc z-D`8$;`T0LrPAIaz{BTLfE*o0|Gg|j{m(>ECLAZD4T$X>o|^d~(~QDnV_iwEXQQF) zD}bS>Sb1RBUc-hr8xvAoLIs)F#Pz<5`BqD;Wp5JpLow?(?u3qI{>jmJ$A)zNbCTuV zbdac|qmB*(bRVSbmdC4vNqHt*z(V*ItTAFKze*UdDO;b1)Fc8@mBIpkQh2%VnGop) z7$gNciB>F~M}JzbNiFhU&yKapQ zSw~)vMd_QGmU|I95MjLT{spj&plWr;hik(=+{j`^Vc%Bx%{afSlVQT{v!r1yuS6oY z=8F&~>@D8@GY97jLpJr1T`%tsRA}5_fJ0V`@p!7LeeCKAO-Q`T({dm&{GeBK$TLKB zvp2k0?^1$MlUXu^Tt1=cSX){4lVU6fE-B@SczHBGa!79UaN?SrQmOqCYVH38wI@$W zzd>zt>p|!%)W&g3O8(za%c4_+vvV1BEyu%{?|XYl4;>Ab`NeDe2WK77aZH5Q1mBb$ z&O16iRqb0J*rGr{Rc($woWx2Qct{|D6&ZKVMnncwM@u}~&!&+U^aB-VAXr1=Dxd}_ zLv-B#WZGt^e==>`8`E;HZyQv5eldv`bB{qEsseiAZA?z+6|fXAuy zP8X&IiIXLFbEObQf1LwsK+5{;p2O6vMj(p|2p75`kGTY2VJG>dHndJWK-8wLS1q-g z-Ece~PxhDS`W@okJe{2!zDuvEceJ!Z_)}X%sBzJ#dAZp{vDxI{Abdv%V2B48 z=yC}Ama?LV9!7{Il`g6rqB2%p3^GA7%Yo-sK^T9LUp=EnVh`vDm(x(V&m<^YI;e5z z=#)CaXa=Mb6lCQSvywC-LqbE-z?kY%N|A}l@!*WgG;Gq`gxoCh+%47J-QRiJ>KR#B z=s4O)*{Mj{I9k&g$w(L!S6S(}sM<&|_V8W0x^JT&&zqAUyOHY^qz!R+$C$p4Gq%=O zlTy*KO_S76G6|NkdU-+Z^9dMyr_1osq4^C1ny%on-iY?iDu39kH06&=F4L4V*SSDF`24mR$PI2rI5?R9}L6@T6tP;<&*8atjJ-;%W;! zTLy#Yp`Ga`ax~G-S>{ct!*a93&eW4w^ySV7ax>*sWi^e zi*g0ZF&e42?LTtXp{O;b(C9PZlMX|J3}D$%6k9;c-V1}1YdBN+mWRa??$OY()4Kcv z0xDi1;5~XL%|9UE2;C|Fg4bgdBL15o4y=Kd*xdM!h)+mN!aKsGJqrl5@tLXi2AI#lZ(s9fa7fjtl5PzQEQT$1^bj*+b5N6HjQ&{P73#i!br_!jykHyBGnNwg zG&+u2FJV=VI$&Vr-7?h}Gj)5y8? z>!sGLZ>E2x)3uTMak&iE?SnvxF{N1{Qt=mxy(H5xuY?2l_4I$`y-=|Np(bwadzSiSuf4Y{du2qufN%a~ZQ!S{| zO&mUs{g6c5UyE}^=~=dhbPLPnF1ZODUlZh$iWM(*zRU7L3w+@8{xw8KSh5650T)AH z2~%q_1DMRBe6)Dy?Z)4C0T#6Qt*>sl3`M78%nsnX0GT?MOuw`bN1P>tKAvbilY4gn z09@2Ji#b7$$K4lu879ch_UH`Iymc0?$WoS%L>rEAh4DBd98AG;n92pRZ~Cz5`C_U7%0q~Axj$J_t@+zb9cKA0pW z`o~mn@96yeuD}~{{65k>1YR#^f1c{kF}45tRh_?1`E%F%n+p9t(mkZ!0sQQi|NHE} zaL%7+f9tIOxv}uAv;Oyy?m-Oo_Y3}0y!dbW?EgCB&wX!ig#o{hbdOK3MFc->!@pba zFDL&$miyLP_j60ZTWj6#BijS%_5G~BoAF;a>VLk%^*4=nKd<<$TjA&W%eQWY-$%9w z)BC?p`tw=y&zGA1m)?cHPW^LXh&w<64+t3>|nWa@ZZilg*L;Z|nZ+M*f&dd=0^rfeJ3{(E?a{uJje^V3j z^J@QJC1;F+Knz6DG&UZ;bVEpZB|kOq6n}-l4@zsn&m|yX( E1Irw-!T + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/scripts/ecmwf-era5/README.md b/scripts/ecmwf-era5/README.md new file mode 100644 index 0000000..11c96cf --- /dev/null +++ b/scripts/ecmwf-era5/README.md @@ -0,0 +1,60 @@ +# ECMWF `ERA5` +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The global `ERA5` dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data # rpp-kshook allocation +/project/rrg-mclark/data/meteorological-data/era5 # rrg-mclark allocation +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data +├── ERA5_merged_195001.nc +├── ERA5_merged_%y%m.nc +├── . +├── . +├── . +└── ERA5_merged_202012.nc +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `ERA5` simulations are `longitude` and `latitude` representing the longitude and latitude points, respectively. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The NetCDF files of the dataset contain 7 variables needed to force hydrological models. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ module load nco/5.0.6 +foo@bar:~$ ncdump -h /project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data/ERA5_merged_195001.nc +``` + +## Spatial Extent +The spatial extent of the `ERA5` is on latitutes from `+90` to `-90` and longitudes from `-180` to `179.75`. The resolution is 0.25 degrees. + +## Temporal Extent +The time-steps are hourly covering from January 1950 to December 2020. + +## Short Description on `ERA5` Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through the current `ERA5` dataset and their details are described in the table below: + +|Variable Name |ERA5 Variable |Original Shortname|Parameter ID|Unit |IPCC abbreviation|Comments | +|---------------------|-------------------|------------------|------------|-----|-----------------|--------------------| +|surface pressure |airpres |sp |134 |Pa |ps | | +|specific humidity @2m|spechum |q |133 |kg/kg|huss | | +|air temperature |airtemp |t |130 |k |tas | | +|wind speed @10m |windspd |u,v |131,132 |m/s |wspd |WIND=SQRT(u2+v2)| +|precipitation |pptrate |mtpr |235055 |mm/hr| |mean total precipitation rate| +|short wave radiation |SWRadAtm |msdwswrf |235035 |W m-2|rsds | | +|long wave radiation |LWRadAtm |msdwlwrd |235036 |W m-2|rlds | | + +For a complete catalog of the dataset, see [here](https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation). + +## Downloading Original ERA5 Data +The scripts to download the original ERA5 data are contained in the [`ERA5_downloads.zip`](./ERA5_downloads.zip) file located in the current directory of the repository. The scripts are written by [Quoqiang Tang](quoqiang.tang@usask.ca) and [Wouter Knoben](wouter.knoben@usask.ca) as part of the [Community Workflows to Advance Reproducibility in Hydrological Modelling (CWARHM)](https://github.com/CH-Earth/CWARHM). "ERA5 data preparation includes interactions between an atmospheric model and a land surface model. ... [Original] ERA5 data is available at 137 different pressure levels (i.e. some height above the surface), as well as at the surface. The lowest atmospheric level is L137, at geopotential and geometric altitude 10m and data here relies only on the atmospheric model. Any variables at a height lower than L137 (i.e., at the surface) are the result of interpolation between atmospheric model and land model. We want to use only the outcomes from the ECMWF atmospheric model [...]. Therefore, we obtain (1) air temperature, (2) wind speed and (3) specific humidity at the lowest pressure level (L137), (4) Precipitation, (5) downward shortwave radiation, (6) downward longwave radiation and (7) air pressure are unaffected by the land model coupling and can be downloaded at the surface level. This is beneficial because surface-level downloads are substantially faster than pressure-level downloads. [^1]" +[^1]: from: https://github.com/CH-Earth/CWARHM/tree/main/3a_forcing#forcing-needed-to-run-summa + diff --git a/scripts/ecmwf-era5/era5.sh b/scripts/ecmwf-era5/era5.sh new file mode 100755 index 0000000..c173383 --- /dev/null +++ b/scripts/ecmwf-era5/era5.sh @@ -0,0 +1,512 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2023, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html +# 2. Drs. Gouqiang Tang and Wouter Knoben provided the downloaded ERA5 dataset files + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n era5 -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # required + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]] || \ + [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ========================== +# Necessary global variables +# ========================== +# the structure of file names is as follows: "ERA5_merged_YYYYMM.nc" +era5Format="%Y%m" # era5 file date format +reportFormat="%Y-%m-%d %H:%M:%S" # report format for manupulations +exportFormat="%Y-%m-%d_%H:%M:%S" # exported file date format +fileStruct="ERA5_merged" # source dataset files' prefix constant + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module load cdo/2.0.4 + module load nco/5.0.6 +} +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#format date string +format_date () { date --date="$1" +"$2"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +####################################### +# extracts file name, date, and time +# from CONUSI file name strings. +# +# Globals: +# fileName: file name of the .nc data +# fileNameDate: date (YYYYMM) +# fileNameYear: year (YYYY) +# fileNameMonth: month (MM) +# +# Arguments: +# 1: -> fName: the +# +# Outputs: +# produces the following global +# variables: +# a) fileName +# b) fileNameDate +# c) fileNameYear +# d) fileNameMonth +####################################### +function extract_filename_info () { + + # define local variable for input argument + local fPath="$1" # format: "/path/to/file/ERA5_merged_YYYYMM.nc" + + # file name + fileName="$(basename $fPath | cut -d '.' -f 1)" # file name + + # file date + fileNameDate="$(echo "$fileName" | cut -d '_' -f 3)" # file date (YYYYMM) + + # year part of the date + fileNameYear="$(echo "$fileNameDate" | cut -c 1-4)" # file year (YYYY) +} + + +###################################### +# populating an array of dates based +# on the input format and time-step +# ranged between the start and end +# points +# +# Globals: +# dateRangeArr: array of dates +# +# Arguments: +# 1: start date +# 2: end date +# 3: format string parsable by `date` +# 4: time-step, e.g., "1hour" +# +# Outputs: +# produces the following variables: +# 5) dateRangeArr +####################################### +function date_range () { + local start=$1 # start date + local end=$2 # end date + local fmt=$3 # format of the ouput dates + local tstep=$4 # the time-step value parsable by bash `date` + + local curr=$start # current time-step + + # make Unix EPOCH time + local currUnix="$(unix_epoch "$curr")" + local endUnix="$(unix_epoch "$end")" + + # a global array variable + dateRangeArr=() + + while [[ "$currUnix" -le "$endUnix" ]]; do + dateRangeArr+=("$(format_date "$curr" "$fmt")") + curr="$(date --date="${curr} ${tstep}")" + + # update $currUnix for the `while` loop + currUnix="$(unix_epoch "$curr")" + done +} + + +####################################### +# end of a chosen time-frame minus +# the value of the time-step, e.g., +# 1. end of the year date based on +# hourly time-steps +# 2. end of the month based on daily +# time-steps +# +# Globals: +# None +# +# Arguments: +# 1. dateStr +# 2. time-frame, i.e., year, mmonth, +# day, hour +# (parsable by GNU date) +# 3. time-step, i.e., year, month, +# day, hour +# (parsable by GNU date) +# +# Outputs: +# prints the end of the time-frame +# at the last time-step to the stdout +####################################### +function time_frame_end () { + local dateStr=$1 # date string + local timeFrame=$2 # time-frame + local timeStep=$3 # time-step + local fmt=$4 # date format + + local dateStrTrim # date string variable + local endDateStr # end date string + + # calculte the last time-step included in the file + # based on the timeframe of the files; + # e.g., ERA5_199201.nc indicates a monthly of that + # has hourly data (ERA5 is hourly). + case "${timeFrame,,}" in + year) + dateStrTrim=$(format_date "$dateStr" "%Y-01-01 00:00:00") + ;; + month) + dateStrTrim=$(format_date "$dateStr" "%Y-%m-01 00:00:00") + ;; + day) + dateStrTrim=$(format_date "$dateStr" "%Y-%m-%d 00:00:00") + ;; + hour) + dateStrTrim=$(format_date "$dateStr" "%Y-%m-%d %H:00:00") + ;; + esac + + local endDateStr="$(date --date="${dateStrTrim} 1${timeFrame} -1${timeStep}" +"$fmt")" + echo $endDateStr +} + + +####################################### +# splitting netCDF files based on the +# tsValue +# +# Globals: +# None +# +# Arguments: +# 1: start date +# 2: end date +# 3: time variable name +# 4: source file path +# 5: destination path +# 6: prefix file string +# 7: date format +# 8: time frame +# 9: time steps +# +# Outputs: +# it splits the netcdf files based +# on the time-steps (argument #9) +# +####################################### +function split_ts () { + # assign local variables + local start=$1 # start date + local end=$2 # end date + local timeVar=$3 # time variable + local sourceFile=$4 # source file + local destDir=$5 # destination directory + local filePrefix=$6 # file prefix + local dateFmt=$7 # date format + local timeFrame=$8 # time frame: + # month, day, etc. + local timeStep=$9 # time step length + + # local variables used in the while loop + local tBegin="$(format_date "$start" "$dateFmt")" + local tEnd + + while [[ "$(unix_epoch "$tBegin")" -le "$(unix_epoch "$end")" ]]; do + tEnd=$(time_frame_end "$tBegin" "$timeFrame" "$timeStep" "$dateFmt") + + if [[ $(unix_epoch "$tEnd") -gt $(unix_epoch "$end") ]]; then + tEnd="$end" + fi + + exportDate="$(format_date "$tBegin" "$exportFormat")" + ncks -d "$timeVar","$tBegin","$tEnd" \ + -d latitude,"$(lims_to_float $latLims)" \ + -d longitude,"$(lims_to_float $lonLims)" \ + -v "$variables" \ + "$sourceFile" "${destDir}/${filePrefix}-${exportDate}.nc" + + tBegin=$(date --date="${tEnd} 1${timeStep}" "+${dateFmt}") + done +} + + +####################################### +# defining start and end point for the +# netcdf file of interest (era5). +# +# Globals: +# None +# +# Arguments: +# 1: file date +# +# Outputs: +# startPoint: start point of the time +# frame +# endPoint: end point of the time +# frame +####################################### +function define_time_points () { + + local fDate=$1 + + local startPoint + local endPoint + local endOfCurrentMonthUnix + local endPoinUnix + + # check dates + if [[ "$fDate" -eq "$(format_date "$startDate" "$era5Format")" ]]; then + endOfCurrentMonthUnix="$(time_frame_end "${fDate}01" "month" "hour" "%s")" # end of month in Unix EPOCH time + if [[ "$endOfCurrentMonthUnix" -le "$(unix_epoch "$endDate")" ]]; then + endPointUnix="$endOfCurrentMonthUnix" + else + endPointUnix="$(unix_epoch "$endDate")" + fi + startPoint="$(format_date "$startDate" "$reportFormat")" + endPoint="$(format_date "@$endPointUnix" "$reportFormat")" + + elif [[ "$fDate" -eq "$(format_date "$endDate" "$era5Format")" ]]; then + startPoint="$(format_date "${fDate}01" "$reportFormat")" + endPoint="$(format_date "${endDate}" "$reportFormat")" + + else + startPoint="$(format_date "${fDate}01" "$reportFormat")" + endPoint="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" + + fi + + timePoints=("$startPoint" "$endPoint") +} + + +# =============== +# Data Processing +# =============== +# display info +echo "$(basename $0): processing ECMWF ERA5..." + +# extract the dates using `date_range` function -> dateRangeArr +date_range "$startDate" "$endDate" "$era5Format" "1hour" # tstep is hard-coded for ERA5 +# extract unique values from $dateRangeArr +uniqueDatesArr=($(echo "${dateRangeArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); + +# creating a temporary directory for temporary files +echo "$(basename $0): creating cache files in $HOME/.temp_gwfdata" +mkdir -p "$cache" # making the directory + +# copy necessary files to the $cache +for ym in "${uniqueDatesArr[@]}"; do + cp "${datasetDir}/${fileStruct}_${ym}.nc" "${cache}/${fileStruct}_${ym}.nc" +done + +# make the output directory +mkdir -p "$outputDir" + +# define empty global array for start and end timePoints +timePoints=() # 0: startPoint, 1: endPoint + +# data files for the current year with extracted $variables +files=($cache/*) + +# if yearly timeScale then make an empty yearsArr +if [[ "${timeScale,,}" == "y" ]]; then + yearsArr=() + datesArr=() +fi + +for f in "${files[@]}"; do + extract_filename_info "$f" # extract file name info + define_time_points "$fileNameDate" # define start & end time points for subsetting + + case "${timeScale,,}" in + h) + split_ts "${timePoints[0]}" "${timePoints[1]}" "time" "$f" "$outputDir" "$prefix" "$reportFormat" "hour" "hour" + ;; + + d) + split_ts "${timePoints[0]}" "${timePoints[1]}" "time" "$f" "$outputDir" "$prefix" "$reportFormat" "day" "hour" + ;; + + m) + exportDate="$(format_date "${timePoints[0]}" "$exportFormat")" + + monthStart="$(format_date "${fDate}01" "$reportFormat")" + monthEnd="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" + if [[ "$monthStart" == "${timePoints[0]}" && "$monthEnd" == "${timePoints[1]}" ]]; then + ncks -d latitude,$(lims_to_float "$latLims") \ + -d longitude,$(lims_to_float "$lonLims") \ + -v "$variables" \ + "$f" "${outputDir}/${prefix}-${exportDate}.nc" + + else + ncks -d time,"${timePoints[0]}","${timePoints[1]}" \ + -d latitude,$(lims_to_float $latLims) \ + -d longitude,$(lims_to_float $lonLims) \ + -v "$variables" \ + "$f" "${outputDir}/${prefix}-${exportDate}.nc" + + fi + ;; + + y) + yearsArr+=("$fileNameYear") + exportDate="$(format_date "${timePoints[0]}" "$exportFormat")" + datesArr+=("$exportDate") + + monthStart="$(format_date "${fDate}01" "$reportFormat")" + monthEnd="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" + if [[ "$monthStart" == "${timePoints[0]}" && "$monthEnd" == "${timePoints[1]}" ]]; then + ncks -d latitude,$(lims_to_float "$latLims") \ + -d longitude,$(lims_to_float "$lonLims") \ + -v "$variables" \ + --mk_rec_dmn time \ + -O \ + "$f" "$f" + + else + ncks -d time,"${timePoints[0]}","${timePoints[1]}" \ + -d latitude,$(lims_to_float $latLims) \ + -d longitude,$(lims_to_float $lonLims) \ + -v "$variables" \ + -O \ + --mk_rec_dmn time \ + "$f" "$f" + + fi + ;; + + esac +done + +if [[ "${timeScale,,}" == "y" ]]; then + # make an array of unique years + uniqueYearsArr=($(echo "${yearsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + + for yr in "${uniqueYearsArr[@]}"; do + # get the first exportDate of each year + idx=0 + for str in "${datesArr[@]}"; do + if [[ "$yr" == $(echo "$str" | cut -d '-' -f 1) ]]; then + break + else + idx=$(($idx + 1)) + fi + done + + exportDate="${datesArr[$idx]}" + ncrcat ${cache}/*${yr}* "${outputDir}/${prefix}-${exportDate}.nc" + done +fi + +rm -r $cache # removing the temporary directory +echo "$(basename $0): temporary files from $cache are removed." +echo "$(basename $0): results are produced under $outputDir." + diff --git a/scripts/ecmwf-era5/era5_simplified.sh b/scripts/ecmwf-era5/era5_simplified.sh new file mode 100755 index 0000000..9200ea6 --- /dev/null +++ b/scripts/ecmwf-era5/era5_simplified.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html +# 2. Drs. Quoqiang Tang and Wouter Knoben downloaded, pre-processed, and +# produced relevant associated scripts. + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n era5 -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]]; then + echo "ERROR $(basename $0): redundant argument (ensemble) provided"; + exit 1; +fi + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary global variables +# ========================== +# the structure of file names is as follows: "ERA5_merged_YYYYMM.nc" +era5Format="%Y%m" # era5 file date format +fileStruct="ERA5_merged" # source dataset files' prefix constant + +latVar="latitude" +lonVar="longitude" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing ECMWF ERA5..." + +# make the output directory +echo "$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" + +toDate="$startDate" +toDateUnix="$(unix_epoch "$toDate")" +endDateUnix="$(unix_epoch "$endDate")" + +# creating yearly directory +mkdir -p "$outputDir" # making the output directory + +# extract variables from the forcing data files +while [[ "$toDateUnix" -le "$endDateUnix" ]]; do + # date manipulations + toDateFormatted=$(date --date "$toDate" +"$era5Format") # current timestamp formatted to conform to CONUSI naming convention + + # creating file name + file="${fileStruct}_${toDateFormatted}.nc" # current file name + + # extracting variables from the files and spatial subsetting + ncks -O -v "$variables" \ + -d latitude,"$(lims_to_float "$latLims")" \ + -d longitude,"$(lims_to_float "$lonLims")" \ + "${datasetDir}/${file}" "${outputDir}/${prefix}${file}" + + [ $( jobs | wc -l ) -ge $( nproc ) ] && wait # forking shell processes + + # increment time-step by one unit + toDate="$(date --date "$toDate 1month")" # current time-step + toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time +done + +# wait to make sure the while loop is finished +wait + +# go to the next year if necessary +if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then + toDate=$(date --date "$toDate 1month") +fi + + +echo "$(basename $0): results are produced under $outputDir." + diff --git a/scripts/gwf-ncar-conus_ii/LICENSE b/scripts/gwf-ncar-conus_ii/LICENSE new file mode 100644 index 0000000..53d1f3d --- /dev/null +++ b/scripts/gwf-ncar-conus_ii/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/scripts/gwf-ncar-conus_ii/README.md b/scripts/gwf-ncar-conus_ii/README.md new file mode 100644 index 0000000..caa9fbe --- /dev/null +++ b/scripts/gwf-ncar-conus_ii/README.md @@ -0,0 +1,121 @@ +# NCAR-GWF `WRF-CONUSII` Simulation Outputs + +In this file, the details of the dataset is explained. + +> [!WARNING] +> It must be noted that the `WRF-CONUSII` dataset are in `.tar` format and the script untars the files automatically. + +> [!CAUTION] +> `WRF-CONUSII` dataset needs extensive I/O operations in the `cache` +> directory. So, in case of submitting `SLURM` jobs, it is recommended to +> use the `$SLURM_TMPDIR` directory as `cache`. This can be provided to +> the main `extract-dataset.sh` script using the `--cache='$SLURM_TMPDIR'` +> option. Please be mindful of **single** quotation marks for the argument. + +## Location of Dataset Files + +The `WRF-CONUSII` simulation outputs are located under the following directory accessible from Compute Canada (CC) Graham Cluster: +``` +/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist +├── 1995 +│ ├── wrf2d_conusii_19950101.tar +│ ├── wrf2d_conusii_19950102.tar +│ ├── . +│ ├── . +│ ├── . +│ └── wrf2d_conusii_19951231.tar +. +. +. +├── %Y +│ ├── wrf2d_conusii_%Y%m%d.tar +│ ├── . +│ ├── . +│ ├── . +│ . +│ . +│ . +. +. +. +└── 2015 + ├── . + ├── . + ├── . + └── wrf2d_conusii_20151231.tar +``` +And, each `.tar` file has the following structure of content: +``` +foo@bar:~$ tar --strip-components=5 -xvf wrf2d_conusii_%Y%m%d.tar > /dev/null +/path/to/tar/extracted/files +└── %Y + ├── wrf2d_d01_%Y-%m-%d_00:00:00 + . + . + . + └── wrf2d_d01_%Y-%m-%d_23:00:00 +``` +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `WRF-CONUSII` simulations are located outside of the main dataset files. The NetCDF file containing the coordinate varibles could be found at the following: +```console +/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc +``` +However, upon many trials by the author, the variables were not concatenated with the main NetCDF files easily. A workaround has been provided to add at least two necessary coordinate variables, i.e., `XLAT` and `XLONG`, to the WRF simulation files. These two coordinates are enough to work with almost all of the meteorological variables included in the dataset. The following scripts are used on Compute Canada (CC) Graham Cluster to produce the substitute NetCDF file containing coordinate variables: +```console +# make a copy of coordinate variable NetCDF file first! +foo@bar:~$ module load cdo/2.0.4; module load nco/5.0.6; # load necessary modules +foo@bar:~$ coordFile="/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc" +foo@bar:~$ ncks -v XLAT,XLONG "$coordFile" coord.nc +foo@bar:~$ nccopy -4 coord.nc coord_new.nc +foo@bar:~$ ncatted -O -a FieldType,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a MemoryOrder,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a stagger,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a coordinates,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a FieldType,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a MemoryOrder,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a stagger,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a coordinates,XLONG,d,, coord_new.nc +foo@bar:~$ ncwa -O -a Time coord_new.nc coord_new.nc +foo@bar:~$ ncrename -a XLONG@description,long_name coord_new.nc +foo@bar:~$ ncrename -a XLAT@description,long_name coord_new.nc +foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc +foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc +foo@bar:~$ ncatted -O -a ,global,d,, coord_new.nc +foo@bar:~$ ncatted -O -h -a license,global,c,c,"GNU General Public License v3 (GPLv3)" coord_new.nc +``` +Furthermore, the substitute NetCDF file containing the coordinate variables are located at `/asset/coord_XLAT_XLONG_conus_i.nc` within this repository. The workaround NetCDF is automatically being used by the script to add the `XLAT` and `XLONG` variables to the final, produced files. + +### Time-stamps +Each hourly, extracted NetCDF files from `.tar` files will have a single time-stamp indicating the time-step of the file. + +## Dataset Variables +The NetCDF files of the dataset contain 187 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ module load nco/5.0.6 +foo@bar:~$ ncdump -h /path/to/extracted/conusii/netcdf/file.nc +``` + +## Spatial Extent +The spatial extent of the `WRF-CONUSII` is on latitutes from `15.02852` to `73.27542` and longitudes from `-156.8242` to `-40.3046`. + +## Temporal Extent +As is obvious from the nomenclature of the dataset files, the time-steps are hourly covering from the January 1995 to December 2015. + +# Short Description on `WRF-CONUSII` Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WRF-CONUSII` dataset and their details are described in the table below: +|Variable Name |WRF-CONUSII Variable|Unit |IPCC abbreviation|Comments | +|---------------------|--------------------|-----|-----------------|--------------------| +|surface pressure |PSFC |Pa |ps | | +|specific humidity @2m|Q2 |1 |huss | | +|air tempreature @2m |T2 |k |tas | | +|wind speed @10m |U10,V10 |m/s |wspd |WIND=SQRT(U102+V102)| +|precipitation |PREC_ACC_NC |mm/hr| |accumulated precipitation over one hour| +|short wave radiation |ACSWDNB |W m-2|rsds | | +|long wave radiation |ACLWDNB |W m-2|rlds | | diff --git a/scripts/gwf-ncar-conus_ii/conus_ii.sh b/scripts/gwf-ncar-conus_ii/conus_ii.sh new file mode 100755 index 0000000..dcfcf84 --- /dev/null +++ b/scripts/gwf-ncar-conus_ii/conus_ii.sh @@ -0,0 +1,552 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html +# 2. Dr. Zhenhua Li provided scripts to extract and process CONUSII datasets +# 3. Dr. Shervan Gharari produced the netCDF file containing XLAT and XLONG +# coordinate variables put under /assets/coord_XLAT_XLONG_conus_i.nc. + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-io DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL]" +} + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n conus-ii -o i:v:o:s:e:t:l:n:c:p:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,cache:,prefix:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # required + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -c | --cache) cacheDir="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]] || \ + [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + + +# ===================== +# Necessary assumptions +# ===================== + +# hard-coding the address of the co-ordinate NetCDF files +# containing XLAT and XLONG variables each having dimensions +# of "south_north" and "west_east". +datatoolPath="$(dirname $0)/../../" # datatool's path +coordMainFile="/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc" +coordEssFile="${datatoolPath}/assets/nc_coords/coord_XLAT_XLONG_conus_ii.nc" +latVar="south_north" +lonVar="west_east" + +# The structure of file names is as follows: "wrf2d_d01_YYYY-MM-DD_HH:MM:SS" (no file extension) +format="%Y-%m-%d_%H:%M:%S" +tarFormat="%Y%m%d" +fileStruct="wrf2d_d01" +tarFileStruct="wrf2d_conusii" +coordIdxScript="${datatoolPath}/assets/ncl_scripts/coord_wrf_idx.ncl" + +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load cdo/2.0.4; + module -q load nco/5.0.6; +} +load_core_modules # load necessary modules + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#format date string +format_date () { date --date="$1" +"$2"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +####################################### +# Implements the necessary netCDF +# operations using CDO and NCO +# +# Globals: +# coordFile: coordinate variables .nc +# file +# lonLims: longitute bounds +# latLims: latitute bounds +# cacheDir: temporary directory for +# file manipulations +# yr: year of selected forcing data +# outputDir: output directory for +# final files +# +# Arguments: +# 1: -> fName: data file name +# 2: -> fDate: date of the forcing +# 3: -> fTime: time of the forcing +####################################### +function generate_netcdf () { + + # defining local variables + local fName="$1" # raw file name string + local fDate="$2" # file string date (YYYY-MM-DD) + local fTime="$3" # file string time (HH:MM:SS) + local fTempDir="$4" # file directory path + local fOutDir="$5" # file output path + local fTimeScale="$6" # fime scale to check the file name + + # add _cat if necessary + if [[ "${fTimeScale,,}" != "h" ]]; then + local fExt="_cat.nc" + fi + + # necessary netCDF operations + ## add coordinate variables: XLAT and XLONG + ncks -A -v XLONG,XLAT "$coordFileSubset" "${fTempDir}/${fName}${fExt}" + ## set time axes + cdo -s -f nc4c -z zip_1 -r settaxis,"$fDate","$fTime",1hour "${fTempDir}/${fName}${fExt}" "${fTempDir}/${fName}_taxis.nc"; + ## rename the `description` attribute + # ncrename -O -a .description,long_name "${fTempDir}/${fName}_taxis.nc" -o "${fOutDir}/${fName}.nc" + mv "${fTempDir}/${fName}_taxis.nc" "${fOutDir}/${prefix}${fName}.nc" +} + + +####################################### +# extracts file name, date, and time +# from CONUSII file name strings. +# +# Globals: +# fileName: file name of the .nc data +# fileNameDate: date (YYYY-MM-DD) +# fileNameYear: year (YYYY) +# fileNameMonth: month (MM) +# fileNameDay: day (DD) +# fileNametime: time (HH:MM:SS) +# +# Arguments: +# 1: -> fName: the +# +# Outputs: +# produces the following global +# variables: +# a) fileName +# b) fileNameDate +# c) fileNameYear +# d) fileNameMonth +# e) fileNameDay +# f) fileNameTime +####################################### +function extract_file_info () { + + # define local variable for input argument + local fPath="$1" # format: "/path/to/file/wrf2d_d01_YYYY-MM-DD_HH:MM:SS" + + # file name + fileName=$(basename "$fPath") # file name + + # file date + fileNameDate=$(echo "$fileName" | cut -d '_' -f 3) # file date (YYYY-MM-DD) + + # parts of the date + fileNameYear=$(echo "$fileNameDate" | cut -d '-' -f 1) # file year (YYYY) + fileNameMonth=$(echo "$fileNameDate" | cut -d '-' -f 2) # file month (MM) + fileNameDay=$(echo "$fileNameDate" | cut -d '-' -f 3) #file name day (DD) + + # file hour + fileNameTime=$(echo "$fileName" | cut -d '_' -f 4) # file time (HH:MM:SS) +} + + +####################################### +# function for extracting the index of +# first match between $str and that of +# the ordered array elements +# +# Globals: +# idx: index of the first match +# +# Arguments: +# 1: the string to be matched with +# 2: the array containing strings to +# be checked +# 3: the position within the matching +# string split by '-' +####################################### +function date_match_idx () { + + # defining local variables + local str="$1" # string to be matched + local matchPos="$2" # the position of the matching string within the "YYYY-MM-DD", + # 1: year, 2: month, 3: day + # 1,2: year and month, 2,3: month and day, 1,3: year and day + # 1-3: complete date + local delim="$3" # delimiter + shift 3 # shift argument positins by 3 + local strArr=("$@") # arrays of string + + # index variable + idx=0 + + # looping through the $strArr + for s in "${strArr[@]}"; do + if [[ "$str" == $(echo "$s" | cut -d ${delim} -f "$matchPos") ]]; then + break + else + idx=`expr $idx + 1` + fi + done +} + + +####################################### +# concatenating files based on a speci- +# temporal scale. +# +# Globals: +# None +# +# Arguments: +# 1: name of the concatenated file +# 2: destination directory +# 3-: array of file paths +# +# Outputs: +# produces $fName_cat.nc under $fDir +# out of all elements of $filesArr +####################################### +function concat_files () { + # defining local variables + local fName="$1" # output file name + local fTempDir="$2" # temporary directory + shift 2 # shift arguments by 2 positions + local filesArr=("$@") # array of file names + + # concatenating $files and producing a single $fName.nc + ncrcat "${filesArr[@]}" "${fTempDir}/${fName}_cat.nc" +} + + +####################################### +# populating arrays with date and time +# values. +# +# Globals: +# datesArr: array of date values +# monthsArr: array of year-month va- +# lues +# timesArr: array of time values +# files: array of file paths +# fileNameDate: date of the current +# filename +# fileNameYear: year of the current +# filename +# fileNameMonth: month of the current +# filename +# uniqueMonthsArr: array of unique +# months +# uniqueDatesArr: array of unique +# dates +# +# Arguments: +# None +# +# Outputs: +# produces the following variables: +# 1) datesArr +# 2) monthsArr +# 3) timesArr +# 4) uniqueMonthsArr +# 5) unqiueDatesArr +####################################### +function populate_date_arrays () { + # defining empty arrays + datesArr=(); + monthsArr=(); + timesArr=(); + + for f in "${files[@]}"; do + extract_file_info "$f" # extract necessary information + + # populate date arrays + datesArr+=(${fileNameDate}); + monthsArr+=("${fileNameYear}-${fileNameMonth}"); + timesArr+=(${fileNameTime}); + done + + uniqueMonthsArr=($(echo "${monthsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); + uniqueDatesArr=($(echo "${datesArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); +} + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing NCAR-GWF CONUSII..." + +# make the output directory +mkdir -p "$outputDir" # create output directory + +# constructing the range of years +startYear=$(date --date="$startDate" "+%Y") # start year (first folder) +endYear=$(date --date="$endDate" "+%Y") # end year (last folder) +yearsRange=$(seq $startYear $endYear) + +# constructing $toDate and $endDate in unix time EPOCH +toDate=$startDate +toDateUnix=$(date --date="$startDate" "+%s") # first date in unix EPOCH time +endDateUnix=$(date --date="$endDate" "+%s") # end date in unix EPOCH time + +# extract the associated indices corresponding to latLims and lonLims +module -q load ncl/6.6.2 +## min and max of latitude and longitude limits +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) +## extract coord +coordIdx="$(ncl -nQ 'coord_file='\"$coordMainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" +lonLimsIdx="$(echo $coordIdx | cut -d ' ' -f 1)" +latLimsIdx="$(echo $coordIdx | cut -d ' ' -f 2)" +module -q unload ncl/6.6.2 +load_core_modules + +# produce subsetted $coordFile as well +mkdir -p "$cacheDir" +coordFileSubset="${cacheDir}/coordFileSubset.nc" +if [[ -f "$coordFileSubset" ]]; then + : +else + ncks -v "XLAT,XLONG" \ + -d "$latVar","$latLimsIdx" \ + -d "$lonVar","$lonLimsIdx" \ + "$coordEssFile" "$coordFileSubset" || true +fi + +# for each year (folder) do the following calculations +for yr in $yearsRange; do + + # creating a temporary directory for temporary files + echo "$(basename $0): creating cache files for year $yr in $cacheDir" + mkdir -p "$cacheDir/$yr" # making the directory + + # setting the end point, either the end of current year, or the $endDate + endOfCurrentYearUnix=$(date --date="$yr-01-01 1year -1day" "+%s") # last time-step of the current year + if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then + endPointUnix=$endOfCurrentYearUnix + else + endPointUnix=$endDateUnix + fi + + # extract variables from the forcing data files + while [[ "$toDateUnix" -le "$endPointUnix" ]]; do + # date manipulations + toDateFormatted=$(date --date "$toDate" "+$tarFormat") # current timestamp formatted to conform to CONUSII naming convention + + # creating file name + file="${tarFileStruct}_${toDateFormatted}.tar" # current file name + + # untar files one day at a time + tar --strip-components=6 -xf "$datasetDir/$yr/$file" -C "$cacheDir/$yr/" + + # parse tar file contents + tarFiles="$(tar -tf $datasetDir/$yr/$file)" + IFS=' ' read -ra tarFiles <<< $(echo $tarFiles) + + # extracting variables from the files and spatial subsetting + for f in "${tarFiles[@]}"; do + f2="$(echo $f | rev | cut -d '/' -f 1 | rev)" + + until ncks -O -v "$variables" \ + -d "$latVar","$latLimsIdx" \ + -d "$lonVar","$lonLimsIdx" \ + "$cacheDir/$yr/$f2" "$cacheDir/$yr/$f2"; do + + echo "$(basename $0): Process killed: restarting process" >$2 + sleep 5; + done + + done + + # increment time-step by one unit + toDate=$(date --date "$toDate 1day") # current time-step + toDateUnix=$(date --date="$toDate" "+%s") # current timestamp in unix EPOCH time + done + + # go to the next year if necessary + if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then + toDate=$(date --date "$toDate 1hour") + fi + + # make the output directory + mkdir -p "$outputDir/$yr/" + + # data files for the current year with extracted $variables + files=($cacheDir/$yr/*) + # sorting files to make sure the time-series is correct + IFS=$'\n' files=($(sort <<<"${files[*]}")); unset IFS + + + # check the $timeScale variable + case "${timeScale,,}" in + + h) + # going through every hourly file + for f in "${files[@]}"; do + # extracting information + extract_file_info "$f" + # necessary NetCDF operations + generate_netcdf "${fileName}" "$fileNameDate" "$fileNameTime" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + d) + # construct the date arrays + populate_date_arrays + + # for each date (i.e., YYYY-MM-DD) + for d in "${uniqueDatesArr[@]}"; do + # find the index of the $timesArr corresponding to $d -> $idx + date_match_idx "$d" "1-3" "-" "${datesArr[@]}" + + # concatenate hourly netCDF files to daily file, i.e., already produces _cat.nc files + dailyFiles=($cacheDir/$yr/${fileStruct}_${d}*) + concat_files "${fileStruct}_${d}" "$cacheDir/$yr/" "${dailyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${d}" "$d" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + m) + # construct the date arrays + populate_date_arrays + + # for each date (i.e., YYYY-MM-DD) + for m in "${uniqueMonthsArr[@]}"; do + # find the index of the $timesArr corresponding to $d -> $idx + # $m is in 'YYYY-MM' format + date_match_idx "$m" "1,2" "-" "${datesArr[@]}" + + # concatenate hourly netCDF files to monthly files, i.e., already produced *_cat.nc files + monthlyFiles=($cacheDir/$yr/${fileStruct}_${m}*) + concat_files "${fileStruct}_${m}" "$cacheDir/$yr/" "${monthlyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${m}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + y) + # construct the date arrays + populate_date_arrays + + # find the index of the $timesArr and $datesArr corresponding to $d -> $idx + date_match_idx "$yr" "1" "-" "${datesArr[@]}" + + # concatenate hourly to yearly files - produced _cat.nc files + yearlyFiles=($cacheDir/$yr/${fileStruct}_${yr}*) + concat_files "${fileStruct}_${yr}" "$cacheDir/$yr/" "${yearlyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${yr}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + ;; + + esac +done + +mkdir "$HOME/empty_dir" +rsync -aP --delete "$HOME/empty_dir/" "$cacheDir" +rm -r "$cacheDir" +echo "$(basename $0): temporary files from $cacheDir are removed." +echo "$(basename $0): results are produced under $outputDir." + diff --git a/scripts/ornl-daymet/LICENSE b/scripts/ornl-daymet/LICENSE new file mode 100644 index 0000000..53d1f3d --- /dev/null +++ b/scripts/ornl-daymet/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/scripts/ornl-daymet/README.md b/scripts/ornl-daymet/README.md new file mode 100644 index 0000000..0f8edee --- /dev/null +++ b/scripts/ornl-daymet/README.md @@ -0,0 +1,93 @@ +# `Daymet` dataset +In this file, the details of the dataset is explained. + +:warning: the dataset files are divided between three different spatial domains: 1) North America (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). For the moment, only the `na` domain is considered in `datatool`. + +## Location of Dataset Files +The global `Daymet` dataset is located under the following directory accessible from Digital Research Alliance of Canada (DRA) Graham cluster: +``` +/project/rpp-kshook/Model_Output/Daymet_Daily_V4R1/data/ +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/Daymet_Daily_V4R1/data/ +├── daymet_v4_daily_hi_dayl_1980.nc +├── daymet_v4_daily_hi_dayl_1981.nc +├── . +├── . +├── . +├── daymet_v4_daily_hi_dayl_2022.nc +├── . +├── . +├── . +├── daymet_v4_daily_hi_vp_1980.nc +├── . +├── . +├── . +├── daymet_v4_daily_hi_vp_2022.nc +├── . +├── . +├── . +├── daymet_v4_daily_{%domain}_{%variable}_{%year}.nc +├── . +├── . +├── . +├── daymet_v4_daily_na_dayl_1980.nc +├── . +├── . +├── . +├── daymet_v4_daily_na_vp_2022.nc +├── daymet_v4_daily_pr_dayl_1950.nc +├── . +├── . +├── . +└── daymet_v4_daily_pr_vp_2022.n +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `Daymet` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. The coordinate system is 2-dimensional and follows a Lambert Conformal Conic projection system. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The NetCDF files of the dataset contain 7 variables. You may see a list of variables by browsing the dataset directory and listing the files. + +## Spatial Extent +The spatial resolutaion of `Daymet` gridded data is 1 $km$. The model files are divided between three different domains: 1) North American (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). Each domains spatial extents are printed in the following table: +|Number |Domain |Latitude extents |Longitude extents | +|:-------------:|:-------------:|:-----------------------------:|:-----------------------------:| +|1 |`na` | `+6.08`° to `+83.79`° | `-180`° to `+180`° | +|2 |`pr` | `+16.85`° to `+19.93`° | `-67.97`° to `-64.13`° | +|3 |`hi` | `+17.96`° to `+23.51`° | `-160.30`° to `-154.78`° | + +:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. + +## Temporal Extent +The time-steps are daily and the temporal extent for each domain is listed in the following table: +|Number |Domain |Time-step interval |Start date |End date | +|:-------------:|:-------------:|:---------------------:|:-------------:|:-------------:| +|1 |`na` |daily |`1980-01-01` |`2022-12-31` | +|2 |`pr` |daily |`1950-01-01` |`2022-12-31` | +|3 |`hi` |daily |`1980-01-01` |`2022-12-31` | + +Also, "[t]he Daymet calendar is based on a standard calendar year. All Daymet years have 1 - 365 days, including leap years. For leap years, the Daymet database includes leap day. Values for December 31 are discarded from leap years to maintain a 365-day year." + +:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. + +## Short Description on `Daymet` Variables +The variables currently available through the `Daymet` dataset and their details are described in the table below, taken from the [source](https://daymet.ornl.gov/overview): + +|Variable Name |`Daymet` Variable |Unit |Comments | +|-----------------------|-----------------------|---------------|-----------------------------------------------------------------------| +|day length |`dayl` |`s/day` |Duration of the daylight period in seconds per day. This calculation is based on the period of the day during which the sun is above a hypothetical flat horizon| +|precipitation |`prcp` |`mm/day` |Daily total precipitation in millimeters per day, sum of all forms converted to water-equivalent. Precipitation occurrence on any given day may be ascertained.| +|shortwave rdiation |`srad` |`W/m2` |Incident shortwave radiation flux density in watts per square meter, taken as an average over the daylight period of the day. NOTE: Daily total radiation `(MJ/m2/day)` can be calculated as follows: `((srad (W/m2) * dayl (s/day)) / l,000,000)`| +|snow water equivalent |`swe ` |`kg/m2` |Snow water equivalent in kilograms per square meter. The amount of water contained within the snowpack.| +|maximum air temperature|`tmax` |`deg C` |Daily maximum 2-meter air temperature in degrees Celsius.| +|minimum air temperature|`tmin` |`deg C` |Daily minimum 2-meter air temperature in degrees Celsius.| +|water vapor pressure |`vp` |`Pa` |Water vapor pressure in pascals. Daily average partial pressure of water vapor.| + +For a complete catalog of the dataset, see [here](https://daymet.ornl.gov/overview). + diff --git a/scripts/ornl-daymet/daymet.sh b/scripts/ornl-daymet/daymet.sh new file mode 100755 index 0000000..962869e --- /dev/null +++ b/scripts/ornl-daymet/daymet.sh @@ -0,0 +1,365 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n daymet -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variable) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]]; then + echo "ERROR $(basename $0): redundant argument (ensemble) provided"; + exit 1; +fi + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + +# create $cache directory +mkdir -p $cache + +# ========================== +# Necessary global variables +# ========================== +# the structure of file names is as follows: "YYYYMMDD12.nc" +daymetDateFormat="%Y" # Daymet dataset date format +daymetPrefixString="daymet_v4_daily" # source dataset files' prefix constant + +# domains of the dataset files - for now, only "na" domain +domains=("na") #na: North America, pr: Peurto Rico, hi: Hawaii + +# spatial 2-dimentional variable included in the dataset netCDF files +latVar="lat" # latitude variable +lonVar="lon" # longitude variable + +# spatial dimension names included in the dataset netCDF files +latDim="y" # latitude dimension +lonDim="x" # longitude dimension + +# paths +datatoolPath="$(dirname $0)/../../" # datatool's path +# daymet index scripts works on RDRSv2.1 grids as well +# and ESPO-G6-R2 has similar grid system as RDRSv2.1 +coordIdxScript="$datatoolPath/assets/ncl_scripts/coord_daymet_idx.ncl" +coordClosestIdxScript="$datatoolPath/assets/ncl_scripts/coord_closest_daymet_idx.ncl" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +## core modules +function load_core_modules () { + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +function unload_core_modules () { + # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT + module -q unload cdo/2.0.4 + module -q unload nco/5.0.6 +} +## ncl modules +function load_ncl_module () { + module -q load ncl/6.6.2 +} +function unload_ncl_module () { + module -q unload ncl/6.6.2 +} + +# loading core modules for the script +load_core_modules + + +# ================= +# Useful one-liners +# ================= +# log date format +log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + +#maximum of a variable in a netcdf file +ncmax () { ncap2 -O -C -v -s "foo=${2}.max();print(foo)" ${1} "$cache/max_$(basename $1)" | cut -f 3- -d ' ' ; } + +#minimum of a variable in a netcdf file +ncmin () { ncap2 -O -C -v -s "foo=${2}.min();print(foo)" ${1} "$cache/min_$(basename $1)" | cut -f 3- -d ' ' ; } + +#minimum of comma delimited string +delim_min () { IFS=', ' read -r -a l <<< "$@"; printf "%s\n" "${l[@]}" | sort -n | head -n1; } + +#maximum of comma delimited string +delim_max () { IFS=', ' read -r -a l <<< "$@"; printf "%s\n" "${l[@]}" | sort -n | tail -n1; } + + +####################################### +# compare float values using basic +# calculator, i.e., `bc` +# +# Arguments: +# 1: -> firstNum: first int/float +# 2: -> SecondNum: second int/float +# 3: -> operator: comparison operator +####################################### +function bc_compare () { + # local variables + local firstNum=$1 + local secondNum=$2 + local operator=$3 + + # implement the comparison + echo "$(bc <<< "$firstNum $operator $secondNum")" +} + +####################################### +# print the full path of the `n`th file +# given the $parentDir and $wildcard +# variables +# +# Arguments: +# 1: -> parentDir: parent directory +# 2: -> wildcard: wildcard to use in +# listing files +# 3: -> nth: nth file to return +####################################### +function nth_file () { + # local variables + local parentDir=$1 + local wildcard=$2 + local nth=$3 + + local fileList + + # listing files + fileList=($parentDir/*${wildcard}*) + # printing nth file + echo "${fileList[$nth]}" +} + + +# =============== +# Data Processing +# =============== +# display info +echo "$(log_date)$(basename $0): processing daymet v4 dataset..." + +# make the output directory +echo "$(log_date)$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" + +echo "$(log_date)$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# define necessary dates +toDate="$startDate" +toDateUnix="$(unix_epoch "$toDate")" +endDateUnix="$(unix_epoch "$endDate")" + +# create empty arrays for included domains and spatial limits +domainsCovered=() # domain strings +domainsLatIdx=() # latitude index +domainLonIdx=() # longitude index + +# parse the upper and lower bounds of a given spatial limit +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) + +# load NCL module +load_ncl_module + +# extract domains that are included in the given spatial limits +for domain in ${domains[@]}; do + # select a representative file (2nd) for each domain + domainFile=$(nth_file $datasetDir $domain 2) + + # check if the input spatial limits overlap with that of domain files + if [[ $(bc_compare "$(delim_min $latLims)" "$(ncmax $domainFile $latVar)" "<=") -eq 1 ]] && \ + [[ $(bc_compare "$(delim_min $lonLims)" "$(ncmax $domainFile $lonVar)" "<=") -eq 1 ]] && \ + [[ $(bc_compare "$(delim_max $latLims)" "$(ncmin $domainFile $latVar)" ">=") -eq 1 ]] && \ + [[ $(bc_compare "$(delim_max $lonLims)" "$(ncmin $domainFile $lonVar)" ">=") -eq 1 ]]; then + + # extract the associated indices corresponding to latLims and lonLims + coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" + + # if spatial index out-of-bound, i.e., 'ERROR' is return + if [[ "${coordIdx}" == "ERROR" ]]; then + # extract the closest index values + coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordClosestIdxScript")" + fi + + # add covered domains + domainsCovered+=($domain) + + # parse the output index for latitude and longitude + lonLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 1)" + latLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 2)" + + # add the limits to the relevant arrays + domainsLatIdx+=("${latLimsIdx}") + domainsLonIdx+=("${lonLimsIdx}") + fi +done + +# check if $domainsCovered is empty +if [[ "${#domainsCovered[@]}" -eq 0 ]]; then + echo -n "$(log_date)$(basename $0): ERROR! The input spatial limits do not " + echo "overlap with the dataset covered area. Try other extents." + exit 1; +fi + +# unload NCl module +unload_ncl_module + +# load core modules again +load_core_modules + +# make array of variable names +IFS=',' read -ra variablesArr <<< "$(echo "$variables")" + +# status update +echo "$(log_date)$(basename $0): extracting daymet v4 netCDF files..." + +# extract files given the time-series extents +while [[ "$toDateUnix" -le "$endDateUnix" ]]; do + + # date format adjustment + toDateFormatted=$(date --date "$toDate" +"$daymetDateFormat") + + # for each overlapped domain + for idx in $(seq 0 $(bc <<< "${#domainsCovered[@]} - 1")); do + + # for each variable + for var in ${variablesArr[@]}; do + + # generating file name + file="${daymetPrefixString}_${domainsCovered[$idx]}_${var}_${toDateFormatted}.nc" + + # extract $file + ncks -O -d "$latDim","${domainsLatIdx[$idx]}" \ + -d "$lonDim","${domainsLonIdx[$idx]}" \ + "${datasetDir}/${file}" "${outputDir}/${prefix}${file}" + + done # for var + done # for domain's index + + # increment time-step by one unit + toDate="$(date --date "$toDate 1year")" # current time-step + toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time + +done + +# wait to assure the `while` loop is done +wait + +# finalizing the workflow +mkdir "$HOME/empty_dir" +echo "$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(basename $0): temporary files from $cache are removed" +echo "$(basename $0): results are produced under $outputDir" + From 7f0d71eb9dd6671cceac916085bf0fb8058924e4 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 17:15:36 -0500 Subject: [PATCH 19/34] Tracking LICENSE of eccc-rdrs --- scripts/eccc-rdrs/LICENSE | 675 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 675 insertions(+) create mode 100644 scripts/eccc-rdrs/LICENSE diff --git a/scripts/eccc-rdrs/LICENSE b/scripts/eccc-rdrs/LICENSE new file mode 100644 index 0000000..53d1f3d --- /dev/null +++ b/scripts/eccc-rdrs/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + From 59bc43443d860bbff73cd950c96bc8e1de292b0c Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 17:16:04 -0500 Subject: [PATCH 20/34] Tracking eccc-rdrs script --- scripts/eccc-rdrs/rdrs.sh | 228 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100755 scripts/eccc-rdrs/rdrs.sh diff --git a/scripts/eccc-rdrs/rdrs.sh b/scripts/eccc-rdrs/rdrs.sh new file mode 100755 index 0000000..3cf239a --- /dev/null +++ b/scripts/eccc-rdrs/rdrs.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n rdrs -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variable) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]] || \ + [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + +# check the prefix of not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary global variables +# ========================== +# the structure of file names is as follows: "YYYYMMDD12.nc" +rdrsFormat="%Y%m%d" # rdrs file date format +exportFormat="%Y%m%d" # exported file date format +fileStruct="" # source dataset files' prefix constant + +latVar="rlat" +lonVar="rlon" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load gcc/9.3.0 + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing ECCC RDRSv2.1..." + +# make the output directory +echo "$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" +echo "$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# define necessary dates +startYear=$(date --date="$startDate" +"%Y") # start year (first folder) +endYear=$(date --date="$endDate" +"%Y") # end year (last folder) +yearsRange=$(seq $startYear $endYear) + +toDate="$startDate" +toDateUnix="$(unix_epoch "$toDate")" +endDateUnix="$(unix_epoch "$endDate")" + +for yr in $yearsRange; do + # creating yearly directory + mkdir -p "$outputDir/$yr" # making the output directory + mkdir -p "$cache/$yr" # making the cache directory + + # setting the end point, either the end of current year, or the $endDate + endOfCurrentYearUnix=$(date --date="$yr-01-01 +1year -1day" "+%s") # last time-step of the current year + if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then + endPointUnix=$endOfCurrentYearUnix + else + endPointUnix=$endDateUnix + fi + + # extract variables from the forcing data files + while [[ "$toDateUnix" -le "$endPointUnix" ]]; do + # date manipulations + toDateFormatted=$(date --date "$toDate" +"$rdrsFormat") # current timestamp formatted to conform to RDRS naming convention + + # creating file name + file="${toDateFormatted}12.nc" # current file name + + # change lon values so the extents are from ~-180 to 0 + ncap2 -s 'where(lon>0) lon=lon-360' "${datasetDir}/${yr}/${file}" "${cache}/${yr}/${file}" + + # extracting variables from the files and spatial subsetting + cdo -z zip -s -L -sellonlatbox,"$lonLims","$latLims" \ + -selvar,"$variables" \ + "${cache}/${yr}/${file}" "${outputDir}/${yr}/${prefix}${file}" + + [ $( jobs | wc -l ) -ge $( nproc ) ] && wait # forking shell processes + + # increment time-step by one unit + toDate="$(date --date "$toDate 1day")" # current time-step + toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time + done + + # wait to make sure the while loop is finished + wait + + # go to the next year if necessary + if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then + toDate=$(date --date "$toDate 1day") + fi + +done + +mkdir "$HOME/empty_dir" +echo "$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(basename $0): temporary files from $cache are removed" +echo "$(basename $0): results are produced under $outputDir" + From d7a9a786804bc5200dd15f3b03cef50303f0e99b Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Mon, 4 Mar 2024 17:16:30 -0500 Subject: [PATCH 21/34] Tracking GWF-NCAR CONUS-I script --- scripts/gwf-ncar-conus_i/LICENSE | 675 ++++++++++++++++++++++++++++ scripts/gwf-ncar-conus_i/README.md | 113 +++++ scripts/gwf-ncar-conus_i/conus_i.sh | 541 ++++++++++++++++++++++ 3 files changed, 1329 insertions(+) create mode 100644 scripts/gwf-ncar-conus_i/LICENSE create mode 100644 scripts/gwf-ncar-conus_i/README.md create mode 100755 scripts/gwf-ncar-conus_i/conus_i.sh diff --git a/scripts/gwf-ncar-conus_i/LICENSE b/scripts/gwf-ncar-conus_i/LICENSE new file mode 100644 index 0000000..53d1f3d --- /dev/null +++ b/scripts/gwf-ncar-conus_i/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + diff --git a/scripts/gwf-ncar-conus_i/README.md b/scripts/gwf-ncar-conus_i/README.md new file mode 100644 index 0000000..802c26e --- /dev/null +++ b/scripts/gwf-ncar-conus_i/README.md @@ -0,0 +1,113 @@ +# NCAR-GWF `WRF-CONUSI` Simulation Outputs + +In this file, the details of the dataset is explained. + +> [!CAUTION] +> `WRF-CONUSI` dataset needs extensive I/O operations in the `cache` +> directory. So, in case of submitting `SLURM` jobs, it is recommended to +> use the `$SLURM_TMPDIR` directory as `cache`. This can be provided to +> the main `extract-dataset.sh` script using the `--cache='$SLURM_TMPDIR'` +> option. Please be mindful of **single** quotation marks for the argument. + +## Location of Dataset Files +The `WRF-CONUSI` simulation outputs are located under the following directory accessible from Compute Canada (CC) Graham Cluster: +``` +/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/ +``` +and the structure of the dataset hourly files is as following: +```console +/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/ +├── 2000 +│ ├── wrf2d_d01_2000-10-01_00:00:00 +│ ├── wrf2d_d01_2000-10-01_01:00:00 +│ ├── . +│ ├── . +│ ├── . +│ └── wrf2d_d01-2000-12-31_23:00:00 +. +. +. +├── %Y +│ ├── wrf2d_d01_%Y-%m-%d_%H:00:00 +│ ├── . +│ ├── . +│ ├── . +│ . +│ . +│ . +. +. +. +└── 2013 + ├── . + ├── . + ├── . + └── wrf2d_d01-2013-12-31_23:00:00 + +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `WRF-CONUSI` simulations are located outside of the main dataset files. The NetCDF file containing the coordinate varibles could be found at the following: +```console +/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/coord.nc +``` +However, upon many trials by the author, the variables were not concatenated with the main NetCDF files easily. A workaround has been provided to add at least two necessary coordinate variables, i.e., `XLAT` and `XLONG`, to the WRF simulation files. These two coordinates are enough to work with almost all of the meteorological variables included in the dataset. The following scripts are used on Compute Canada (CC) Graham Cluster to produce the substitute NetCDF file containing coordinate variables: +```console +# make a copy of coordinate variable netCDF file first +foo@bar:~$ module load cdo/2.0.4; module load nco/5.0.6; +foo@bar:~$ coordFile="/project/6008034/Model_Output/WRF/CONUS/CTRL/coord.nc" +foo@bar:~$ ncks -O -v XLAT,XLONG "$coordFile" coord2.nc +foo@bar:~$ nccopy -4 coord2.nc coord_new.nc +foo@bar:~$ ncatted -O -a FieldType,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a MemoryOrder,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a stagger,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a coordinates,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a sr_x,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a sr_y,XLAT,d,, coord_new.nc +foo@bar:~$ ncatted -O -a FieldType,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a MemoryOrder,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a stagger,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a coordinates,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a sr_x,XLONG,d,, coord_new.nc +foo@bar:~$ ncatted -O -a sr_y,XLONG,d,, coord_new.nc +foo@bar:~$ ncwa -O -a Times coord_new.nc coord_new.nc +foo@bar:~$ ncks -O -v XLAT,XLONG coord_new.nc coord_new.nc +foo@bar:~$ ncrename -a XLONG@description,long_name coord_new.nc +foo@bar:~$ ncrename -a XLAT@description,long_name coord_new.nc +foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc +foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc +foo@bar:~$ ncatted -O -a ,global,d,, coord_new.nc +foo@bar:~$ ncatted -O -h -a license,global,c,c,"GNU General Public License v3 (GPLv3)" coord_new.nc +``` +Furthermore, the substitute NetCDF file containing the coordinate variables are located at `/asset/coord_XLAT_XLONG_conus_i.nc` within this repository. The workaround NetCDF is automatically being used by the script to add the `XLAT` and `XLONG` variables to the final, produced files. + +### Time-stamps +The time-stamp of the time-steps are missing from the dataset NetCDF files. However, the time-stamps for each time-step is obvious from the file names. The time-stamp pattern of the dataset files is as following: `%Y-%m-%d_%H:00:00` which will be changed to `%Y-%m-%s %H:00:00` to be registered as a valid time-stamp in the NetCDF files. The script is able to set the time-stamps for the final produced file(s) automatically. + +## Dataset Variables +The NetCDF files of the dataset contain 281 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: +```console +foo@bar:~$ module load cdo/2.0.4 +foo@bar:~$ module load nco/5.0.6 +foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/2000/wrf2d_d01_2000-10-01_00:00:00 +``` + +## Spatial Extent +The spatial extent of the `WRF-CONUSI` is on latitutes from `18.13629` to `57.91813` and longitudes from `-139.0548` to `-56.94519`. + +## Temporal Extent +As is obvious from the nomenclature of the dataset files, the time-steps are hourly covering from the October 2000 to September 2013. + +# Short Description on `WRF-CONUSI` Variables +In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WRF-CONUSI` dataset and their details are described in the table below: +|Variable Name |WRF-CONUSI Variable|Unit |IPCC abbreviation|Comments | +|---------------------|-------------------|-----|-----------------|--------------------| +|surface pressure |PSFC |Pa |ps | | +|specific humidity @2m|Q2 |1 |huss | | +|air tempreature @2m |T2 |k |tas | | +|wind speed @10m |U10,V10 |m/s |wspd |WIND=SQRT(U102+V102)| +|precipitation |PREC_ACC_NC |mm/hr| |accumulated precipitation over one hour| +|short wave radiation |ACSWDNB |W m-2|rsds | | +|long wave radiation |ACLWDNB |W m-2|rlds | | diff --git a/scripts/gwf-ncar-conus_i/conus_i.sh b/scripts/gwf-ncar-conus_i/conus_i.sh new file mode 100755 index 0000000..db90686 --- /dev/null +++ b/scripts/gwf-ncar-conus_i/conus_i.sh @@ -0,0 +1,541 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2022-2023, University of Saskatchewan +# Copyright (C) 2023-2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from +# https://www.shellscript.sh/tips/getopt/index.html +# 2. Dr. Zhenhua Li provided scripts to extract and process CONUSI datasets +# 3. Dr. Shervan Gharari produced the netCDF file containing XLAT and XLONG +# coordinate variables put under /assets/coord_XLAT_XLONG_conus_i.nc. +# 4. Sorting workflow is taken from the following link: +# https://stackoverflow.com/a/11789688/5188208 + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-io DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL]" +} + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n conus-i -o i:v:o:s:e:t:l:n:c:p:m:S:M: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,cache:,prefix:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variables) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # required + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -c | --cache) cacheDir="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility + -S | --scenario) scenario="$2" ; shift 2 ;; # redundant - added for compatibility + -M | --model) model="$2" ; shift 2 ;; # redundant - added for compatibility + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check if $ensemble is provided +if [[ -n "$ensemble" ]] || \ + [[ -n "$scenario" ]] || \ + [[ -n "$model" ]]; then + echo "ERROR $(basename $0): redundant argument provided"; + exit 1; +fi + + +# ===================== +# Necessary assumptions +# ===================== + +# hard-coding the address of the co-ordinate NetCDF files +# containing XLAT and XLONG variables each having dimensions +# of "south_north" and "west_east". +datatoolPath="$(dirname $0)/../../" # datatool's path +coordMainFile="/project/rpp-kshook/Model_Output/WRF/CONUS/coord.nc" +coordEssFile="${datatoolPath}/assets/nc_coords/coord_XLAT_XLONG_conus_i.nc" +latVar="south_north" +lonVar="west_east" + +# The structure of file names is as follows: "wrf2d_d01_YYYY-MM-DD_HH:MM:SS" (no file extension) +format="%Y-%m-%d_%H:%M:%S" +fileStruct="wrf2d_d01" +coordIdxScript="${datatoolPath}/assets/ncl_scripts/coord_wrf_idx.ncl" + +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# =================== +# Necessary functions +# =================== +# Modules below available on Compute Canada (CC) Graham Cluster Server +function load_core_modules () { + module -q load cdo/2.0.4; + module -q load nco/5.0.6; +} +load_core_modules # load necessary modules + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#format date string +format_date () { date --date="$1" +"$2"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +####################################### +# Implements the necessary netCDF +# operations using CDO and NCO +# +# Globals: +# coordFile: coordinate variables .nc +# file +# lonLims: longitute bounds +# latLims: latitute bounds +# cacheDir: temporary directory for +# file manipulations +# yr: year of selected forcing data +# outputDir: output directory for +# final files +# +# Arguments: +# 1: -> fName: data file name +# 2: -> fDate: date of the forcing +# 3: -> fTime: time of the forcing +####################################### +function generate_netcdf () { + + # defining local variables + local fName="$1" # raw file name string + local fDate="$2" # file string date (YYYY-MM-DD) + local fTime="$3" # file string time (HH:MM:SS) + local fTempDir="$4" # file directory path + local fOutDir="$5" # file output path + local fTimeScale="$6" # fime scale to check the file name + + # add _cat if necessary + if [[ "${fTimeScale,,}" != "h" ]]; then + local fExt="_cat.nc" + fi + + # necessary netCDF operations + ## add coordinate variables: XLAT and XLONG + ncks -A -v XLONG,XLAT "$coordFileSubset" "${fTempDir}/${fName}${fExt}" + ## set time axes + cdo -s -f nc4c -z zip_1 -r settaxis,"$fDate","$fTime",1hour "${fTempDir}/${fName}${fExt}" "${fTempDir}/${fName}_taxis.nc"; + ## rename the `description` attribute + # ncrename -O -a .description,long_name "${fTempDir}/${fName}_taxis.nc" -o "${fOutDir}/${fName}.nc" + mv "${fTempDir}/${fName}_taxis.nc" "${fOutDir}/${prefix}${fName}.nc" +} + + +####################################### +# extracts file name, date, and time +# from CONUSI file name strings. +# +# Globals: +# fileName: file name of the .nc data +# fileNameDate: date (YYYY-MM-DD) +# fileNameYear: year (YYYY) +# fileNameMonth: month (MM) +# fileNameDay: day (DD) +# fileNametime: time (HH:MM:SS) +# +# Arguments: +# 1: -> fName: the +# +# Outputs: +# produces the following global +# variables: +# a) fileName +# b) fileNameDate +# c) fileNameYear +# d) fileNameMonth +# e) fileNameDay +# f) fileNameTime +####################################### +function extract_file_info () { + + # define local variable for input argument + local fPath="$1" # format: "/path/to/file/wrf2d_d01_YYYY-MM-DD_HH:MM:SS" + + # file name + fileName=$(basename "$fPath") # file name + + # file date + fileNameDate=$(echo "$fileName" | cut -d '_' -f 3) # file date (YYYY-MM-DD) + + # parts of the date + fileNameYear=$(echo "$fileNameDate" | cut -d '-' -f 1) # file year (YYYY) + fileNameMonth=$(echo "$fileNameDate" | cut -d '-' -f 2) # file month (MM) + fileNameDay=$(echo "$fileNameDate" | cut -d '-' -f 3) #file name day (DD) + + # file hour + fileNameTime=$(echo "$fileName" | cut -d '_' -f 4) # file time (HH:MM:SS) +} + + +####################################### +# function for extracting the index of +# first match between $str and that of +# the ordered array elements +# +# Globals: +# idx: index of the first match +# +# Arguments: +# 1: the string to be matched with +# 2: the array containing strings to +# be checked +# 3: the position within the matching +# string split by '-' +####################################### +function date_match_idx () { + + # defining local variables + local str="$1" # string to be matched + local matchPos="$2" # the position of the matching string within the "YYYY-MM-DD", + # 1: year, 2: month, 3: day + # 1,2: year and month, 2,3: month and day, 1,3: year and day + # 1-3: complete date + local delim="$3" # delimiter + shift 3 # shift argument positins by 3 + local strArr=("$@") # arrays of string + + # index variable + idx=0 + + # looping through the $strArr + for s in "${strArr[@]}"; do + if [[ "$str" == $(echo "$s" | cut -d ${delim} -f "$matchPos") ]]; then + break + else + idx=`expr $idx + 1` + fi + done +} + + +####################################### +# concatenating files based on a speci- +# temporal scale. +# +# Globals: +# None +# +# Arguments: +# 1: name of the concatenated file +# 2: destination directory +# 3-: array of file paths +# +# Outputs: +# produces $fName_cat.nc under $fDir +# out of all elements of $filesArr +####################################### +function concat_files () { + # defining local variables + local fName="$1" # output file name + local fTempDir="$2" # temporary directory + shift 2 # shift arguments by 2 positions + local filesArr=("$@") # array of file names + + # concatenating $files and producing a single $fName.nc + ncrcat "${filesArr[@]}" "${fTempDir}/${fName}_cat.nc" +} + + +####################################### +# populating arrays with date and time +# values. +# +# Globals: +# datesArr: array of date values +# monthsArr: array of year-month va- +# lues +# timesArr: array of time values +# files: array of file paths +# fileNameDate: date of the current +# filename +# fileNameYear: year of the current +# filename +# fileNameMonth: month of the current +# filename +# uniqueMonthsArr: array of unique +# months +# uniqueDatesArr: array of unique +# dates +# +# Arguments: +# None +# +# Outputs: +# produces the following variables: +# 1) datesArr +# 2) monthsArr +# 3) timesArr +# 4) uniqueMonthsArr +# 5) unqiueDatesArr +####################################### +function populate_date_arrays () { + # defining empty arrays + datesArr=(); + monthsArr=(); + timesArr=(); + + for f in "${files[@]}"; do + extract_file_info "$f" # extract necessary information + + # populate date arrays + datesArr+=(${fileNameDate}); + monthsArr+=("${fileNameYear}-${fileNameMonth}"); + timesArr+=(${fileNameTime}); + done + + uniqueMonthsArr=($(echo "${monthsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); + uniqueDatesArr=($(echo "${datesArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); +} + + +# =============== +# Data processing +# =============== +# display info +echo "$(basename $0): processing NCAR-GWF CONUSI..." + +# make the output directory +mkdir -p "$outputDir" # create output directory + +# constructing the range of years +startYear=$(date --date="$startDate" "+%Y") # start year (first folder) +endYear=$(date --date="$endDate" "+%Y") # end year (last folder) +yearsRange=$(seq $startYear $endYear) + +# constructing $toDate and $endDate in unix time EPOCH +toDate=$startDate +toDateUnix=$(date --date="$startDate" "+%s") # first date in unix EPOCH time +endDateUnix=$(date --date="$endDate" "+%s") # end date in unix EPOCH time + +# extract the associated indices corresponding to latLims and lonLims +module -q load ncl/6.6.2 +## min and max of latitude and longitude limits +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) +## extract coord +coordIdx="$(ncl -nQ 'coord_file='\"$coordMainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" +lonLimsIdx="$(echo $coordIdx | cut -d ' ' -f 1)" +latLimsIdx="$(echo $coordIdx | cut -d ' ' -f 2)" +module -q unload ncl/6.6.2 +load_core_modules + +# produce subsetted $coordFile as well +mkdir -p "$cacheDir" +coordFileSubset="${cacheDir}/coordFileSubset.nc" +# if subsetted coordinate file does not exist, make one +if [[ -f "$coordFileSubset" ]]; then + : +else + ncks -v "XLAT,XLONG" \ + -d "$latVar","$latLimsIdx" \ + -d "$lonVar","$lonLimsIdx" \ + "$coordEssFile" "$coordFileSubset" || true +fi + +# for each year (folder) do the following calculations +for yr in $yearsRange; do + + # creating a temporary directory for temporary files + echo "$(basename $0): creating cache files for year $yr in $cacheDir" + mkdir -p "$cacheDir/$yr" # making the directory + + # setting the end point, either the end of current year, or the $endDate + endOfCurrentYearUnix=$(date --date="$yr-01-01 +1year -1hour" "+%s") # last time-step of the current year + if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then + endPointUnix=$endOfCurrentYearUnix + else + endPointUnix=$endDateUnix + fi + + # extract variables from the forcing data files + while [[ "$toDateUnix" -le "$endPointUnix" ]]; do + # date manipulations + toDateFormatted=$(date --date "$toDate" "+$format") # current timestamp formatted to conform to CONUSI naming convention + + # creating file name + file="${fileStruct}_${toDateFormatted}" # current file name + + # extracting variables from the files and spatial subsetting + ncks -O -v "$variables" \ + -d "$latVar","$latLimsIdx" \ + -d "$lonVar","$lonLimsIdx" \ + "$datasetDir/$yr/$file" "$cacheDir/$yr/$file" & # extracting $variables + [ $( jobs | wc -l ) -ge $( nproc ) ] && wait + + # increment time-step by one unit + toDate=$(date --date "$toDate 1hour") # current time-step + toDateUnix=$(date --date="$toDate" "+%s") # current timestamp in unix EPOCH time + done + + # wait to make sure the while loop is finished + wait + + # go to the next year if necessary + if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then + toDate=$(date --date "$toDate 1hour") + fi + + # make the output directory + mkdir -p "$outputDir/$yr/" + + # data files for the current year with extracted $variables + files=($cacheDir/$yr/*) + # sorting files to make sure the time-series is correct + IFS=$'\n' files=($(sort <<<"${files[*]}")); unset IFS + + # check the $timeScale variable + case "${timeScale,,}" in + + h) + # going through every hourly file + for f in "${files[@]}"; do + # extracting information + extract_file_info "$f" + # necessary NetCDF operations + generate_netcdf "${fileName}" "$fileNameDate" "$fileNameTime" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + d) + # construct the date arrays + populate_date_arrays + + # for each date (i.e., YYYY-MM-DD) + for d in "${uniqueDatesArr[@]}"; do + # find the index of the $timesArr corresponding to $d -> $idx + date_match_idx "$d" "1-3" "-" "${datesArr[@]}" + + # concatenate hourly netCDF files to daily file, i.e., already produces _cat.nc files + dailyFiles=($cacheDir/$yr/${fileStruct}_${d}*) + concat_files "${fileStruct}_${d}" "$cacheDir/$yr/" "${dailyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${d}" "$d" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + m) + # construct the date arrays + populate_date_arrays + + # for each date (i.e., YYYY-MM-DD) + for m in "${uniqueMonthsArr[@]}"; do + # find the index of the $timesArr corresponding to $d -> $idx + # $m is in 'YYYY-MM' format + date_match_idx "$m" "1,2" "-" "${datesArr[@]}" + + # concatenate hourly netCDF files to monthly files, i.e., already produced *_cat.nc files + monthlyFiles=($cacheDir/$yr/${fileStruct}_${m}*) + concat_files "${fileStruct}_${m}" "$cacheDir/$yr/" "${monthlyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${m}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + done + ;; + + y) + # construct the date arrays + populate_date_arrays + + # find the index of the $timesArr and $datesArr corresponding to $d -> $idx + date_match_idx "$yr" "1" "-" "${datesArr[@]}" + + # concatenate hourly to yearly files - produced _cat.nc files + yearlyFiles=($cacheDir/$yr/${fileStruct}_${yr}*) + concat_files "${fileStruct}_${yr}" "$cacheDir/$yr/" "${yearlyFiles[@]}" + + # implement CDO/NCO operations + generate_netcdf "${fileStruct}_${yr}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" + ;; + + esac +done + +mkdir "$HOME/empty_dir" +echo "$(basename $0): deleting temporary files from $cacheDir" +rsync -aP --delete "$HOME/empty_dir/" "$cacheDir" +rm -r "$cacheDir" +echo "$(basename $0): temporary files from $cacheDir are removed" +echo "$(basename $0): results are produced under $outputDir" + From 2e9815f6c855222ef04691ce0813d7e7890a5a68 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 15:03:29 -0500 Subject: [PATCH 22/34] Documentation for NASA's NEX-GDDP-CMIP6 dataset This commit addresses issue #27 by describing the NASA's NEX-GDDP-CMIP^ dataset and relevant scripts for it. Furthermore, it provides necessary information for users to enable them use `datatool` for extracting subsets of the dataset for any temporal and spatial extents. Signed-off-by: Kasra Keshavarz --- scripts/nasa-nex-gddp-cmip6/README.md | 305 ++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 scripts/nasa-nex-gddp-cmip6/README.md diff --git a/scripts/nasa-nex-gddp-cmip6/README.md b/scripts/nasa-nex-gddp-cmip6/README.md new file mode 100644 index 0000000..dfc691a --- /dev/null +++ b/scripts/nasa-nex-gddp-cmip6/README.md @@ -0,0 +1,305 @@ +# NASA NEX-GDDP-CMIP6 Climate Dataset (`nex-gddp-cmip6`) +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `nex-gddp-cmip6` dataset is located under the following directory(s) accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rrg-mclark/data/meteorological-data/nasa-nex-gddp-cmip6/NEX-GDDP-CMIP6 # rrg-mclark allocation +``` + +and the structure of the dataset's yearly files (containing daily time-steps) is as following: +```console +/project/rrg-mclark/data/meteorological-data/nasa-nex-gddp-cmip6/NEX-GDDP-CMIP6/ +├── ACCESS-CM2 +│ ├── historical +│ │ └── r1i1p1f1 +│ │ ├── hurs +│ │ │ ├── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +| | | ├── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950_v1.1.nc +│ │ │ ├── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +│ │ │ ├── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951_v1.1.nc +│ │ │ ├── . +│ │ │ ├── . +│ │ │ ├── . +│ | │ ├── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +│ | │ └── hurs_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014_v1.1.nc +│ | ├── huss +│ | | ├── huss_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── huss_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── huss_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── huss_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── huss_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | ├── pr +│ | | ├── pr_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── pr_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── pr_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── pr_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── pr_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | ├── rlds +│ | | ├── rlds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── rlds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── rlds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── rlds_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── rlds_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | ├── rsds +│ | | ├── rsds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── rsds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── rsds_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── rsds_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── rsds_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | ├── tas +│ | | ├── tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── tas_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | ├── tasmax +│ | | ├── tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | | ├── tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | | ├── tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | | └── tasmax_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +| | └── tasmin +│ | ├── tasmin_day_ACCESS-CM2_historical_r1i1p1f1_gn_1950.nc +│ | ├── tasmin_day_ACCESS-CM2_historical_r1i1p1f1_gn_1951.nc +| | ├── tasmin_day_ACCESS-CM2_historical_r1i1p1f1_gn_1952.nc +| | ├── . +| | ├── . +| | ├── . +| | ├── tasmin_day_ACCESS-CM2_historical_r1i1p1f1_gn_2013.nc +| | └── tasmin_day_ACCESS-CM2_historical_r1i1p1f1_gn_2014.nc +│ ├── ssp126 +│ | └── r1i1p1f1 +│ | ├── hurs +│ | | ├── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2015.nc +│ | | ├── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2015_v1.1.nc +│ | | ├── . +│ | | ├── . +│ | | ├── . +│ | | ├── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2100.nc +│ | | └── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2100_v1.1.nc +| | . +| | . +| | . +| | └── tasmin +| | ├── tasmin_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2015.nc +| | ├── tasmin_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2016.nc +| | ├── . +| | ├── . +| | ├── . +| | └── tasmin_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2100.nc +| . +| . +| . +| ├── %{scenario} +| ├── %{ensemble} +| . . ├── %{var} +| . . . ├── %{var}_day_ACCESS-CM2_%{scenario}_%{ensemble}_gn_%{year}%{version}.nc +| . . . . +| . . . . +| . . . . +| └── ssp585 +│ └── r1i1p1f1 +│ ├── hurs +│ | ├── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2015.nc +│ | ├── . +│ | ├── . +│ | ├── . +│ | └── hurs_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2100.nc +| . +| . +| . +| └── tasmin +| ├── tasmin_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2015.nc +| ├── . +| ├── . +| ├── . +| └── tasmin_day_ACCESS-CM2_ssp126_r1i1p1f1_gn_2100.nc +. +. . +. . +├── %{model} +. ├── %{scenario} +. . └── %{ensemble} +. . ├── %{var} +. . . ├── %{var}_day_%{model}_%{scenario}_%{ensemble}_gn_%{year}%{version}.nc +. . . . +. . . . +. . . . +└── UKESM1-0-LL + ├── historical + | └── r1i1p1f2 + | ├── hurs + | | ├── hurs_day_UKESM1-0-LL_historical_r1i1p1f2_gn_1950.nc + | | ├── hurs_day_UKESM1-0-LL_historical_r1i1p1f2_gn_1950_v1.1.nc + | | ├── . + | | ├── . + | | ├── . + | | ├── hurs_day_UKESM1-0-LL_historical_r1i1p1f2_gn_2014.nc + | | └── hurs_day_UKESM1-0-LL_historical_r1i1p1f2_gn_2014_v1.1.nc + | . . + | . . + | . . + | └── tasmin + | . + | . + | └── tasmin_day_UKESM1-0-LL_historical_r1i1p1f2_gn_2014.nc + . + . + . + └── ssp585 + └── r1i1p1f2 + ├── hurs + | ├── hurs_day_UKESM1-0-LL_ssp585_r1i1p1f2_gn_2015.nc + | ├── . + | ├── . + | ├── . + | └── hurs_day_UKESM1-0-LL_ssp585_r1i1p1f2_gn_2100.nc + . + . + . + └── tasmin + ├── tasmin_day_UKESM1-0-LL_ssp585_r1i1p1f2_gn_2015.nc + . + . + . + └── tasmin_day_UKESM1-0-LL_ssp585_r1i1p1f2_gn_2100.nc +``` + +> [!important] +> Not all models have the same number of scenarios, enesmble members, and +> variables. Each individual model needs to be investigate individually. + +> [!caution] +> Currently, `datatool` is NOT capable of identifying various versions of +> dataset files. In this dataset, as can be observed files for `v1.1` +> (those indicated with a `_v1.1_` in their file names) are ignored. +> This will be addressed in the future versions. + + +## `nex-gddp-cmip6` Climate Models +This dataset offers downscaled outputs of various climate models. Table below +summarizes the models and relevant keywords that could be used with the +main `datatool` script: + +|# |Model (keyword for `--model`) |Scenarios (keyword for `--scenario`) | +|---| -------------------------------|----------------------------------------------------| +|1 |`ACCESS-CM2` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|2 |`ACCESS-ESM1-5` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|3 |`BCC-CSM2-MR` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|4 |`CanESM5` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|5 |`CESM2` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|6 |`CESM2-WACCM` |`historical`, `ssp245`, `ssp585` | +|7 |`CMCC-CM2-SR5` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|8 |`CMCC-ESM2` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|9 |`CNRM-CM6-1` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|10 |`CNRM-ESM2-1` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|11 |`EC-Earth3` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|12 |`EC-Earth3-Veg-LR` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|13 |`FGOALS-g3` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|14 |`GFDL-CM4` |`historical`, `ssp245`, `ssp585` | +|15 |`GFDL-CM4_gr2` |`historical`, `ssp245`, `ssp585` | +|16 |`GFDL-ESM4` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|17 |`GISS-E2-1-G` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|18 |`HadGEM3-GC31-LL` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|19 |`HadGEM3-GC31-MM` |`historical`, `ssp126`, `ssp245`, `ssp585` | +|20 |`IITM-ESM` |`historical`, `ssp126`, `ssp585` | +|21 |`INM-CM4-8` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|22 |`INM-CM5-0` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|23 |`IPSL-CM6A-LR` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|24 |`KACE-1-0-G` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|25 |`KIOST-ESM` |`historical`, `ssp126`, `ssp245`, `ssp585` | +|26 |`MIROC6` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|27 |`MIROC-ES2L` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|28 |`MPI-ESM1-2-HR` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|29 |`MPI-ESM1-2-LR` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|30 |`MRI-ESM2-0` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|31 |`NESM3` |`historical`, `ssp126`, `ssp245`, `ssp585` | +|32 |`NorESM2-LM` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|33 |`NorESM2-MM` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|34 |`TaiESM1` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| +|35 |`UKESM1-0-LL` |`historical`, `ssp126`, `ssp245`, `ssp370`, `ssp585`| + + +## Coordinate Variables, Spatial and Temporal extents, and Time-stamps + +### Coordinate Variables +The coordinate variables of the `nex-gddp-cmip6` climate dataset files are `rlon` and `rlat` representing the longitude and latitude points, respectively. + +### Temporal Extents and Time-stamps +The time-stamps are already included in the original files. The dataset offers +**daily** time-series of climate variables. The following table +describes the temporal extent for senarios included in this dataset: +|# |Scenarios (keyword for `--scenario`) |Temporal extent | +|---|-------------------------------------|----------------------------| +|1 |`historical` |`1950-01-01` to `2014-12-31`| +|2 |`ssp126` |`2015-01-01` to `2100-12-31`| +|3 |`ssp245` |`2015-01-01` to `2100-12-31`| +|4 |`ssp370` |`2015-01-01` to `2100-12-31`| +|5 |`ssp585` |`2015-01-01` to `2100-12-31`| + +> [!Note] +> Values of the `Temporal extent` column are the limits for `--start-date` +> and `--end-date` options with the main `datatool` script. + + +## Dataset Variables +The NetCDF files of the dataset contain various variables. You may see a list of variables by browsing the dataset's directory: +```console +foo@bar:~$ ls /project/rrg-mclark/data/meteorological-data/nasa-nex-gddp-cmip6/NEX-GDDP-CMIP6/ACCESS-CM2/ssp126/r1i1p1f1/ +hurs huss pr rlds rsds sfcWind tas tasmax tasmin +``` + +## Spatial Extent +The `nex-gddp-cmip6` dataset spatial extent is global. + +## Short Description on `nex-gddp-cmip6` Climate Dataset Variables +This dataset offers 9 climate variables: 1) precipitation, 2) mean +air temperature, 3) daily maximum temperature, 4) daily minimum +temperature, 5) specific humidity, 6) relative humidity, 7) shortwave +radiation, 8) longwave radiation, and 9) wind speed. + +Since the frequency of this dataset is daily, including daily +time-series of precipitation and air temperature, it could +be potentially used for forcing conceptual hydrological models that only +need daily time-series of these variables. + +Furthermore, with common existing disaggregation methods existing in the +literature, one can generate sub-daily time-series of each variable and +use them for forcing physically based models that may need more +climate variables as their forcing data. + +The table below, summarizes the variables offered by this dataset: + +|Variable Name |Variable (keyword for `--variable`)|Unit |IPCC Abbreviation|Comments | +|-----------------------|-----------------------------------|------|-----------------|----------------------| +|maximum temperature@2m |`tasmax` |K |tasmax |near-surface 2m level | +|minimum temperature@2m |`tasmin` |K |tasmin |near-surface 2m level | +|preciptiation |`pr` |mm/day|pr |surface level | +|relative humidity |`hurs` |% |hurs |near-surface level | +|specific humidity |`huss` |1 |huss |near-surface 2m level | +|longwave radiation |`rlds` |W m-2 |rlds |surface level | +|shortwave radiation |`rsds` |W m-2 |rsds |surface level | +|wind speed@10m |`sfcWind` |m s-1 | |near-surface 10m level| +|mean air temperature@2m|`tas` |K |tas |near-surface 2m level | + +For the most up-to-date information please visit [NASA's NEX-GDDP-CMIP6 +project website](https://www.nccs.nasa.gov/services/data-collections/land-based-products/nex-gddp-cmip6). From 79de9836abb543b37de09b5967dbabf0047de90e Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 15:05:26 -0500 Subject: [PATCH 23/34] Script for NASA's NEX-GDDP-CMIP6 dataset This commit addresses issue #27 and provides scripts to extract subset from NASA's NEX-GDDP-CMIP6 dataset. This script is capable to work with various models, scenarios, ensemble members, and variables offered by this dataset. Signed-off-by: Kasra Keshavarz --- scripts/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh | 421 ++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100755 scripts/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh diff --git a/scripts/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh b/scripts/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh new file mode 100755 index 0000000..98977fc --- /dev/null +++ b/scripts/nasa-nex-gddp-cmip6/nex-gddp-cmip6.sh @@ -0,0 +1,421 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage Functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variable) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant + -S | --scenario) scenario="$2" ; shift 2 ;; # required + -M | --model) model="$2" ; shift 2 ;; # required + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check the prefix is not set +if [[ -z $prefix ]]; then + prefix="data" +fi + +# check if $model is provided +if [[ -z $model ]]; then + echo "ERROR $(basename $0): --model value(s) missing" + exit 1; +fi + +# useful log date format function +logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + +# check if the dates are within datasets date range +# define $startYear and $endYear +startYear=$(date --date "$startDate" +"%Y") +endYear=$(date --date "$endDate" +"%Y") + + +# ===================== +# Necessary Assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + + +# ========================== +# Necessary Global Variables +# ========================== +latDim="lat" +lonDim="lon" +timeDim="time" +resolution="0.25" + + +# =================== +# Necessary Functions +# =================== +# Modules below available on Digital Research Alliance of Canada's Graham HPC +## core modules +function load_core_modules () { + module -q load gcc/9.3.0 + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +function unload_core_modules () { + # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT + module -q unload cdo/2.0.4 + module -q unload nco/5.0.6 +} +## ncl modules +function load_ncl_module () { + module -q load gcc/9.3.0 + module -q load ncl/6.6.2 +} +function unload_ncl_module () { + module -q unload ncl/6.6.2 +} + +# loading core modules for the script +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + +#offset lims +offset () { float="$1"; offset="$2"; printf "%.1f," $(echo "$float + $offset" | bc) | sed 's/,$//'; } + + +# ================ +# Useful functions +# ================ +####################################### +# expand the upper and lower limits of +# $latLims by the resolution value of +# the dataset - only for the sake of an +# abundance of caution +# +# Arguments: +# 1. lims -> spatial extents in a +# comma-delimited form +# 2. resolution -> resolution of the +# dataset for +# expansion +# +# Globals: +# 1. globalLims -> expanded Lims +####################################### +function expand_lims () { + # local variables + local lims="$1" + local res="$2" + local limArr + + # expansion + IFS=',' read -ra limArr <<< $lims + limArr[0]=$(echo "${limArr[0]} - $res" | bc) + limArr[1]=$(echo "${limArr[1]} + $res" | bc) + echo "$(join_by , ${limArr[@]})" +} + + +# =============== +# Data Processing +# =============== +# create $modelArr array from input comma-delimited values +IFS=',' read -ra modelArr <<< $(echo $model) +# create $scenarioArr array from input comma-delimited values +IFS=',' read -ra scenarioArr <<< $(echo $scenario) +# create $ensembleArr array from input comma-delimited values +IFS=',' read -ra ensembleArr <<< $(echo $ensemble) +# create $variableArr array from input comma-delimited values +IFS=',' read -ra variableArr <<< $(echo $variables) + +# taking care of various possible scenarios for $startDate and $endDate +## #1 if startYear is before 2015, and historical is NOT selected as a +## scenario, issue a WARNING and add historical to $scenarioArr +if [[ "$startYear" -lt 2015 ]] && \ + [[ "${scenarioArr[*]}" == "historical" ]]; then + # issue a warning and add historical to the scenarios + echo "$(logDate)$(basename $0): WARNING! Dates preceeding 2015 belongs to \`hisotrical\` scenario" + echo "$(logDate)$(basename $0): WARNING! \`historical\` is added to \`--scenario\` list" + scenarioArr+=("historical") +fi + +## #2 if endYear is beyond 2014, and SSP scenarios are NOT +## selected, issue an ERROR and terminate with exitcode 1 +if [[ "$endYear" -gt 2014 ]] && \ + [[ "${scenarioArr[*]}" == "ssp" ]]; then # `ssp` is treated as *ssp* + echo "$(logDate)$(basename $0): ERROR! Dates past 2015 belong to \`ssp\` scenarios" + echo "$(logDate)$(basename $0): ERROR! Choose the appropriate date range and try again" + exit 1; +fi + +# display info +echo "$(logDate)$(basename $0): processing NASA NEX-GDDP-CMIP6..." + +# since, the dataset's grid cell system is gaussian, assure to to_float() +# the $latLims and $lonLims values +latLims="$(lims_to_float "$latLims")" +lonLims="$(lims_to_float "$lonLims")" + +# since longitudes are within the [0, 360] range, offset input $lonLims by +# -180, if they are greater than 180. +IFS=',' read -ra lims <<< $lonLims +f_arr=() +for lim in "${lims[@]}"; do + if [[ $(echo "$lim < 0" | bc -l ) ]]; then + f_arr+=($(offset "$lim" 180)) + else + f_arr+=($lim) + fi +done +lonLims="$(join_by , ${f_arr[@]})" + +# expand the upper and lower limits of latLims by the resolution value +latLims=$(expand_lims $latLims $resolution) +lonLims=$(expand_lims $lonLims $resolution) + + +# ============================================ +# Build date arrays for time-series extraction +# ============================================ +# file date intervals in years - dataset's default +interval=1 + +fileDateFormat="%Y" +actualDateFormat="%Y-%m-%d" + +# define needed variables +let "difference = $endYear - $startYear" +let "steps = $difference / $interval" + +# build $startDateFileArr, $endDateFileArr +startDateFileArr=() +endDateFileArr=() +actualStartDateArr=() +actualEndDateArr=() + +# range of jumps +range=$(seq 0 $steps) + +# filling the arrays +for iter in $range; do + # jumps every $interval years + let "jumps = $iter * $interval" + + # current date after necessary jumps + let "toDate = $jumps + $startYear" + + # extract start and end values + startValue="$(date --date "${toDate}0101" +"${fileDateFormat}")" + endValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${fileDateFormat}")" + + # double-check end-date + if [[ "$endValue" -gt 2100 ]]; then + endValue="2100" # irregular last date for dataset files + fi + + # extract start and end values for actual dates + actualStartValue="$(date --date "${toDate}0101" +"${actualDateFormat}")" + actualEndValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${actualDateFormat}")" + + # fill up relevant arrays + startDateFileArr+=("${startValue}") + endDateFileArr+=("${endValue}") + + actualStartDateArr+=("${actualStartValue}") + actualEndDateArr+=("${actualEndValue}") +done + +# build actualStartArr array for temporal subsetting +actualStartDateArr[0]="$(date --date "${startDate}" +"${actualDateFormat}")" + +# and similarly, the actualEndArr array +lastIndex=$(( "${#actualEndDateArr[@]}" - 1 )) +actualEndDateArr[${lastIndex}]="$(date --date "${endDate}" +"${actualDateFormat}")" + + +# ===================== +# Extract dataset files +# ===================== +# Typical directory structure of the dataset is: +# ${datasetDir}/${model}/${scenario}/${ensemble}/${var}/ +# and each ${var} directory contains files in the following nomenclature: +# ${var}_day_${model}_${scenario}_${ensemble}_gn_%Y.nc +# with the %Y year value indicating the starting year of data inside the +# file + +# create dataset directories in $cache and $outputDir +echo "$(logDate)$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" +echo "$(logDate)$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# iterate over models/submodels +for model in "${modelArr[@]}"; do + # extract model and submodel names + modelName=$(echo $model | cut -d '/' -f 1) + + # iterate over scenarios, e.g., ssp126, ssp245, ssp370, ssp585 + for scenario in "${scenarioArr[@]}"; do + + # iterate over ensemble members, e.g., r1p1, r1p2, r1p3 + for ensemble in "${ensembleArr[@]}"; do + + pathTemplate="${modelName}/${scenario}/${ensemble}/" + if [[ -e "${datasetDir}/${pathTemplate}" ]]; then + echo "$(logDate)$(basename $0): processing ${model}.${scenario}.${ensemble} files" + mkdir -p "${cache}/${pathTemplate}" + mkdir -p "${outputDir}/${pathTemplate}" + else + echo "$(logDate)$(basename $0): ERROR! ${model}.${scenario}.${ensemble} does not exist." 2>&1 + break 1; + fi + + # iterate over date range of interest using index + for idx in "${!startDateFileArr[@]}"; do + + # dates for files + fileStartDate="${startDateFileArr[$idx]}" + fileEndDate="${endDateFileArr[$idx]}" + # dates for subsetting + actualStartDate="${actualStartDateArr[$idx]}" + actualEndDate="${actualEndDateArr[$idx]}" + # dates for ncks slabs + actualStartDateFormatted="$(date --date "${actualStartDate}" +'%Y-%m-%d')" + actualEndDateFormatted="$(date --date "${actualEndDate}" +'%Y-%m-%d')" + + # iterate over dataset variables of interest + for var in "${variableArr[@]}"; do + + # define file for further operation + src="${var}_day_${modelName}_${scenario}_${ensemble}_gn_${fileStartDate}.nc" + dst="day_${modelName}_${scenario}_${ensemble}_gn_${fileStartDate}.nc" + + # subsetting variable, spatial extents, and temporal extents + until ncks -A -v ${var} \ + -d "$latDim","${latLims}" \ + -d "$lonDim","${lonLims}" \ + -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ + ${datasetDir}/${pathTemplate}/${var}/${src} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process" 2>&1 + sleep 10; + done # until ncks + + # apply offset to $lonDim values of each NetCDF file to have + # longitude range within [-180, +180] + until ncap2 -O -s "${lonDim}=${lonDim}-180" \ + ${cache}/${pathTemplate}/${dst} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process" 2>&1 + sleep 10; + done # until ncap2 + + # copy the results + cp -r ${cache}/${pathTemplate}/${dst} \ + ${outputDir}/${pathTemplate}/${prefix}${dst}; + + done # for $variableArr + done # for $startDateArr + done # for $ensembleArr + done # for $scenarioArr +done # for $modelArr + +# wait for everything to finish - just in case +sleep 10 + +mkdir "$HOME/empty_dir" +echo "$(logDate)$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(logDate)$(basename $0): temporary files from $cache are removed" +echo "$(logDate)$(basename $0): results are produced under $outputDir" + From e983994c5070450afbe9c46bcfa5a741ed7ce2ea Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 17:30:00 -0500 Subject: [PATCH 24/34] Adding Ouranos ESPO-G6-R2 Dataset Script This commit addresses issue #34 and processes this dataset that contains multiple GCM model outputs, including various sub-models, scenarios, ensemble members, and variables. Signed-off-by: Kasra Keshavarz --- scripts/ouranos-espo-g6-r2/espo-g6-r2.sh | 424 +++++++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100755 scripts/ouranos-espo-g6-r2/espo-g6-r2.sh diff --git a/scripts/ouranos-espo-g6-r2/espo-g6-r2.sh b/scripts/ouranos-espo-g6-r2/espo-g6-r2.sh new file mode 100755 index 0000000..bfdcd7f --- /dev/null +++ b/scripts/ouranos-espo-g6-r2/espo-g6-r2.sh @@ -0,0 +1,424 @@ +#!/bin/bash +# Meteorological Data Processing Workflow +# Copyright (C) 2024, University of Calgary +# +# This file is part of Meteorological Data Processing Workflow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# ========================= +# Credits and contributions +# ========================= +# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html + + +# ================ +# General comments +# ================ +# * All variables are camelCased for distinguishing from function names; +# * function names are all in lower_case with words seperated by underscore for legibility; +# * shell style is based on Google Open Source Projects' +# Style Guide: https://google.github.io/styleguide/shellguide.html + + +# =============== +# Usage functions +# =============== +short_usage() { + echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" +} + + +# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT +parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") +validArguments=$? +if [ "$validArguments" != "0" ]; then + short_usage; + exit 1; +fi + +# check if no options were passed +if [ $# -eq 0 ]; then + echo "ERROR $(basename $0): arguments missing"; + exit 1; +fi + +# check long and short options passed +eval set -- "$parsedArguments" +while : +do + case "$1" in + -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required + -v | --variable) variables="$2" ; shift 2 ;; # required + -o | --output-dir) outputDir="$2" ; shift 2 ;; # required + -s | --start-date) startDate="$2" ; shift 2 ;; # required + -e | --end-date) endDate="$2" ; shift 2 ;; # required + -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility + -l | --lat-lims) latLims="$2" ; shift 2 ;; # required + -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required + -p | --prefix) prefix="$2" ; shift 2 ;; # optional + -c | --cache) cache="$2" ; shift 2 ;; # required + -m | --ensemble) ensemble="$2" ; shift 2 ;; # required + -S | --scenario) scenario="$2" ; shift 2 ;; # required + -M | --model) model="$2" ; shift 2 ;; # required + + # -- means the end of the arguments; drop this, and break out of the while loop + --) shift; break ;; + + # in case of invalid option + *) + echo "ERROR $(basename $0): invalid option '$1'"; + short_usage; exit 1 ;; + esac +done + +# check the prefix is not set +if [[ -z $prefix ]]; then + prefix="data_" +fi + +# useful log date format function +logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } + + +# ================ +# Necessary checks +# ================ + +# check if the dates are within datasets date range +# define $startYear and $endYear +startYear=$(date --date "$startDate" +"%Y") +endYear=$(date --date "$endDate" +"%Y") + +# if $startYear is before 1950 raise a "WARNING" and set startDate +if [[ $startYear -lt 1950 ]]; then + echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" + echo "$(logDate)$(basename $0): WARNING! \`start-date\` is set to 1950-01-01 00:00:00" + startDate="1950-01-01" + startYear="1950" +fi + +# if $endYear is beyond 2100 raise a "WARNING" and set endDate +if [[ $endYear -gt 2100 ]]; then + echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" + echo "$(logDate)$(basename $0): WARNING! \`end-date\` is set to 2100-12-31 00:00:00" + endDate="2100-12-31" + endYear="2100" +fi + +# check if $model, $ensemble, and $scenario is given +if [[ -z $model ]] || \ + [[ -z $ensemble ]] || \ + [[ -z $scenario ]]; then + echo "$(logDate)$(basename $0): ERROR! \`--model\`, \`--ensemble\`, and \`--scenario\` values are required" + exit 1; +fi + + +# ===================== +# Necessary assumptions +# ===================== +# TZ to be set to UTC to avoid invalid dates due to Daylight Saving +alias date='TZ=UTC date' + +# expand aliases for the one stated above +shopt -s expand_aliases + +# paths +datatoolPath="$(dirname $0)/../../" # datatool's path +# daymet index scripts works on RDRSv2.1 grids as well +# and ESPO-G6-R2 has similar grid system as RDRSv2.1 +coordIdxScript="$datatoolPath/assets/ncl_scripts/coord_daymet_idx.ncl" +coordClosestIdxScript="$datatoolPath/assets/ncl_scripts/coord_closest_daymet_idx.ncl" + + +# ========================== +# Necessary global variables +# ========================== +latDim="rlat" +lonDim="rlon" +timeDim="time" + + +# =================== +# Necessary functions +# =================== +# Modules below available on Digital Research Alliance of Canada's Graham HPC +## core modules +function load_core_modules () { + module -q load gcc/9.3.0 + module -q load cdo/2.0.4 + module -q load nco/5.0.6 +} +function unload_core_modules () { + # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT + module -q unload cdo/2.0.4 + module -q unload nco/5.0.6 +} +## ncl modules +function load_ncl_module () { + module -q load gcc/9.3.0 + module -q load ncl/6.6.2 +} +function unload_ncl_module () { + module -q unload ncl/6.6.2 +} + +# loading core modules for the script +load_core_modules + + +# ================= +# Useful one-liners +# ================= +#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 +unix_epoch () { date --date="$@" +"%s"; } + +#check whether the input is float or real +check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } + +#convert to float if the number is 'int' +to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } + +#join array element by the specified delimiter +join_by () { local IFS="$1"; shift; echo "$*"; } + +#to_float the latLims and lonLims, real numbers delimited by ',' +lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } + + +# =============== +# Data processing +# =============== +# display info +echo "$(logDate)$(basename $0): processing Ouranos ESPO-G6-R2..." + +# create $modelArr array from input comma-delimited values +IFS=',' read -ra modelArr <<< $(echo $model) +# create $scenarioArr array from input comma-delimited values +IFS=',' read -ra scenarioArr <<< $(echo $scenario) +# create $ensembleArr array from input comma-delimited values +IFS=',' read -ra ensembleArr <<< $(echo $ensemble) +# create $variableArr array from input comma-delimited values +IFS=',' read -ra variableArr <<< $(echo $variables) + + +# ====================== +# Extract domain extents +# ====================== + +# parse the upper and lower bounds of a given spatial limit +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) + +# unload and load necessary modules +unload_core_modules +load_ncl_module +# choose a sample file as all files share the same grid +domainFile=$(find ${datasetDir} -type f -name "*.nc" | head -n 1) +# parse the upper and lower bounds of a given spatial limit +minLat=$(echo $latLims | cut -d ',' -f 1) +maxLat=$(echo $latLims | cut -d ',' -f 2) +minLon=$(echo $lonLims | cut -d ',' -f 1) +maxLon=$(echo $lonLims | cut -d ',' -f 2) + +# extract the associated indices corresponding to $latLims and $lonLims +coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" + +# if spatial index out-of-bound, i.e., 'ERROR' is return +if [[ "${coordIdx}" == "ERROR" ]]; then + # extract the closest index values + coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordClosestIdxScript")" +fi + +# parse the output index for latitude and longitude +lonLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 1)" +latLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 2)" + +# reload necessary modules +unload_ncl_module +load_core_modules + +# ============================================ +# Build date arrays for time-series extraction +# ============================================ +# file date intervals in years - dataset's default +interval=4 + +startFormat="%Y0101" +endFormat="%Y1231" # will be redefined later depending on the $modelName + +actualFormat='%Y%m%d' + +# define needed variables +let "difference = $endYear - $startYear" +let "steps = $difference / $interval" + +# build $startDateFileArr, $endDateFileArr +startDateFileArr=() +endDateFileArr=() + +# range of jumps +range=$(seq 0 $steps) + +# filling the arrays +for iter in $range; do + # jumps every $interval years + let "jumps = $iter * $interval" + + # current date after necessary jumps + let "toDate = $jumps + $startYear" + + # extract start and end values + startValue="$(date --date "${toDate}0101" +"${startFormat}")" + endValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${endFormat}")" + + # check if endValue is beyond 2100 + endValueYear="$(date --date "${endValue}" +"%Y")" + # double-check end-date + if [[ "$endValueYear" -gt 2100 ]]; then + endValue="21001231" # irregular last date for dataset files + fi + + # fill up relevant arrays + startDateFileArr+=("${startValue}") + endDateFileArr+=("${endValue}") + +done + +# build actualStartArr array for temporal subsetting +actualStartDateArr=("${startDateFileArr[@]}") +actualStartDateArr[0]="$(date --date "${startDate}" +"${actualFormat}")" + +# and similarly, the actualEndArr array +actualEndDateArr=("${endDateFileArr[@]}") +lastIndex=$(( "${#actualEndDateArr[@]}" - 1 )) +actualEndDateArr[${lastIndex}]="$(date --date "${endDate}" +"${actualFormat}")" + + +# ===================== +# Extract dataset files +# ===================== +# Typical directory structure of the dataset is: +# ${datasetDir}/${model}/%submodel/${scenario}/${ensemble}/day/${var}/ +# and each ${var} directory contains files in the following nomenclature: +# ${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${model}_%submodel_${scenario}_${ensemble}_%yyyymmdd-%yyyymmdd.nc +# with the former date value indicating the starting year of data inside the +# file, and the latter demonstrating the ending date of data +# +# NOTE: %submodel must be determined in the upstream caller +# + +# create dataset directories in $cache and $outputDir +echo "$(logDate)$(basename $0): creating output directory under $outputDir" +mkdir -p "$outputDir" +echo "$(logDate)$(basename $0): creating cache directory under $cache" +mkdir -p "$cache" + +# iterate over models/submodels +for model in "${modelArr[@]}"; do + # extract model and submodel names + modelName=$(echo $model | cut -d '/' -f 1) + submodelName=$(echo $model | cut -d '/' -f 2) + + # iterate over scenarios, e.g., ssp245, ssp370, ssp585 + for scenario in "${scenarioArr[@]}"; do + + # iterate over ensemble members, e.g., r1p1, r1p2, etc. + for ensemble in "${ensembleArr[@]}"; do + + pathTemplate="${modelName}/${submodelName}/${scenario}/${ensemble}/day/" + if [[ -e "${datasetDir}/${pathTemplate}" ]]; then + echo "$(logDate)$(basename $0): processing ${model}.${scenario}.${ensemble} files" + mkdir -p "${cache}/${pathTemplate}" + mkdir -p "${outputDir}/${pathTemplate}" + else + echo "$(logDate)$(basename $0): ERROR! ${model}.${scenario}.${ensemble} does not exist." + break 1; + fi + + # iterate over date range of interest using index + for idx in "${!startDateFileArr[@]}"; do + + # dates for files + fileStartDate="${startDateFileArr[$idx]}" + fileEndDate="${endDateFileArr[$idx]}" + # dates for subsetting + actualStartDate="${actualStartDateArr[$idx]}" + actualEndDate="${actualEndDateArr[$idx]}" + # dates for ncks slabs + actualStartDateFormatted="$(date --date $actualStartDate +'%Y-%m-%d')" + actualEndDateFormatted="$(date --date $actualEndDate +'%Y-%m-%d')" + + # destination NetCDF file + dst="day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_${modelName}_${submodelName}_${scenario}_${ensemble}_${actualStartDate}-${actualEndDate}.nc" + + # address inconsistencies with NetCDF file end-date values in the + # dataset + if [[ "$modelName" == "NIMS-KMA" ]] || \ + [[ "$modelName" == "MOHC" ]]; then + fileEndDate=$(date --date "${fileEndDate}" +"%Y1230") + fi + + # iterate over dataset variables of interest + for var in "${variableArr[@]}"; do + + # define file for further operation + # address inconsistencies with NetCDF file name in the dataset + if [[ "$modelName" == "DKRZ" ]]; then + src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" + else + src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${modelName}_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" + fi + + # spatial subsetting + until ncks -A -v ${var} \ + -d "$latDim","${latLimsIdx}" \ + -d "$lonDim","${lonLimsIdx}" \ + -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ + ${datasetDir}/${pathTemplate}/${var}/${src} \ + ${cache}/${pathTemplate}/${dst}; do + echo "$(logDate)$(basename $0): Process killed: restarting process in 10 sec" >&2 + echo "NCKS failed" >&2 + sleep 10; + done # until ncks + done # for $variableArr + + # change lon values so the extents are from ~-180 to 0 + # this is solely for easymore compatibility + until ncap2 -O -s "where(lon>0) lon=lon-360" \ + "${cache}/${pathTemplate}/${dst}" \ + "${outputDir}/${pathTemplate}/${prefix}${dst}"; do + echo "$(logDate)$(basename $0): Process killed: restarting process in 10 sec" >&2 + echo "NCAP2 failed" >&2 + sleep 10; + done # until ncap2 + + done # for $startDateArr + done # for $ensembleArr + done # for $scenarioArr +done # for $modelArr + +# wait for everything to finish - just in case +sleep 10 + +mkdir "$HOME/empty_dir" +echo "$(logDate)$(basename $0): deleting temporary files from $cache" +rsync -aP --delete "$HOME/empty_dir/" "$cache" +rm -r "$cache" +echo "$(logDate)$(basename $0): temporary files from $cache are removed" +echo "$(logDate)$(basename $0): results are produced under $outputDir" + From fd6d96ddebdd0ba6ad0a8bd1a0aa5047bd3bf846 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 17:32:08 -0500 Subject: [PATCH 25/34] Documenting Ouranos ESPO-G6-R2 Dataset script Necessary information to use `datatool` for this script is provided to the user via the README.md file. Signed-off-by: Kasra Keshavarz --- scripts/ouranos-espo-g6-r2/README.md | 164 +++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 scripts/ouranos-espo-g6-r2/README.md diff --git a/scripts/ouranos-espo-g6-r2/README.md b/scripts/ouranos-espo-g6-r2/README.md new file mode 100644 index 0000000..013fa9a --- /dev/null +++ b/scripts/ouranos-espo-g6-r2/README.md @@ -0,0 +1,164 @@ +# Ouranos `ESPO-G6-R2 v1.0.0` dataset +In this file, the details of the dataset is explained. + +## Location of Dataset Files +The `ESPO-G6-R2 v1.0.0` dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: +```console +/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/ouranos-espo-g6-r2 # rpp-kshook allocation +/project/rrg-mclark/data/meteorological-data/ouranos-espo-g6-r2 # rrg-mclark allocation +``` + +and the structure of the dataset hourly files is as following: +```console +/project/rrg-mclark/data/meteorological-data/ouranos-espo-g6-r2 +├── AS-RCEC +│ └── TaiESM1 +│ ├── ssp245 +| | └── r1i1p1f1 +│ | └── day +| | ├── pr +│ | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19500101-19531231.nc +| | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19540101-19571231.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20940101-20971231.nc +| | | └── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20980101-210031231.nc +│ | ├── tasmax +| | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19500101-19531231.nc +| | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19540101-19571231.nc +| | | ├── . +| | | ├── . +| | | ├── . +| | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20940101-20971231.nc +| | | └── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20980101-21001231.nc +| | └── tasmin +| | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19500101-19531231.nc +| | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19540101-19571231.nc +| | ├── . +| | ├── . +| | ├── . +| | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20940101-20971231.nc +| | └── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_20980101-21001231.nc +│ └── ssp370 +| └── r1i1p1f1 +│ └── day +| ├── pr +│ | ├── . +│ | ├── . +│ | └── . +| ├── tasmax +│ | ├── . +│ | ├── . +│ | └── . +| tasmin +│ ├── . +│ ├── . +│ └── . +│ +. +. +. +├── %{model} +| ├── %{submodel} # no need for explicit declaration in the scripts +| | ├── %{scenario} +| | | └── %{ensemble} +| | | └── day +| | | ├── %{var} +| | | | ├── %{var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_%{model}_%{submodel}_%{scenario}_%{ensemble}_%{year}0101_%{year+3}1231.nc +| | | | ├── . +| | | | ├── . +| | | | ├── . +| | | | └── %{var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_%{model}_%{submodel}_%{scenario}_%{ensemble}_%{year}0101_%{year+2}1231.nc +. . . . +. . . . +. . . . +└── NUIST + └── NESM3 + ├── ssp245 + | └── r1i1p1f1 + | └── day + | ├── pr + | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19500101-19531231.nc + | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19540101-19571231.nc + | | ├── . + | | ├── . + | | ├── . + | | ├── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20940101-20971231.nc + | | └── pr_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20980101-210031231.nc + | ├── tasmax + | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19500101-19531231.nc + | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19540101-19571231.nc + | | ├── . + | | ├── . + | | ├── . + | | ├── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20940101-20971231.nc + | | └── tasmax_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20980101-21001231.nc + | └── tasmin + | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19500101-19531231.nc + | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19540101-19571231.nc + | ├── . + | ├── . + | ├── . + | ├── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20940101-20971231.nc + | └── tasmin_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_20980101-21001231.nc + └── ssp370 + └── r1i1p1f1 + └── day + ├── pr + | ├── . + | ├── . + | └── . + ├── tasmax + | ├── . + | ├── . + | └── . + tasmin + ├── . + ├── . + └── . +``` + +## Coordinate Variables and Time-stamps + +### Coordinate Variables +The coordinate variables of the `ESPO-G6-R2 v1.0.0` simulations are `rlon` and `rlat` representing the longitude and latitude points, respectively. +### Time-stamps +The time-stamps are included in the original files. + +## Dataset Variables +The NetCDF files of the dataset contain one variable per file. You may see a list of variables by browsing the dataset files: +```console +foo@bar:~$ ls /project/rrg-mclark/data/meteorological-data/ouranos-espo-g6-r2/ESPO-G6-R2v1.0.0/AS-RCEC/TaiESM1/ssp245/r1i1p1f1/day +``` + +## Spatial Extent +The spatial extent of the `ESPO-G6-R2 v1.0.0` is on latitutes from `+5.75` to `+83.98` and longitudes from `-179.9925` to `179.9728` covering North America. The resolution is 0.09 degrees (~10km). + +## Temporal Extent +The time-stamps are already included in the original files. The dataset offers +**daily** time-series of climate variables. The following table +describes the temporal extent for senarios included in this dataset: +|# |Scenarios (keyword for `--scenario`) |Temporal extent | +|---|-------------------------------------|----------------------------| +|1 |`ssp245` |`2015-01-01` to `2100-12-31`| +|2 |`ssp370` |`2015-01-01` to `2100-12-31`| +|3 |`ssp585` |`2015-01-01` to `2100-12-31`| + + +## Short Description on `ESPO-G6-R2 v1.0.0` Variables +This dataset only offers three climate variables: 1) daily precipitation +time-series (surface level), 2) daily minimum temperature time-series +(@2m, near-surface level), and 3) daily maximum temperature time-series +(@2m, near-surface level). Since the frequency of this dataset is daily, +and only offers precipitation and temperature values, therefore, it could +be potentially used for forcing conceptual hydrological models that only +need daily time-series of these variables. + +The table below, summarizes the variables offered by this dataset: +|Variable Name |Variable (keyword for `--variable`)|Unit |IPCC Abbreviation|Comments | +|----------------------|-----------------------------------|----------|-----------------|----------------------| +|maximum temperature |`tasmax` |K |tasmax |near-surface 2m height| +|minimum temperature |`tasmin` |K |tasmin |near-surface 2m height| +|preciptiation |`pr` |kg m-2 s-1|pr |surface level | + From a4c22fc1558288412cbe88bb7329f19cf24002c6 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 17:40:59 -0500 Subject: [PATCH 26/34] Updating changelog for v0.5.0 --- CHANGELOG | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 9110b1b..b6bc69b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,20 @@ Changelog ========= +[v0.5.0] March 5th, 2024 +# Improvements + * `DATASETS` file now describes all the datasets available in the script + * new parallelization schemes are introduced using models, scenarios, + and ensemble members + * the `assets` directory is now more organized separating common NCL + and bash scripts needed + * styles of the script is updated (not completely) to be more compatible + with Google's shell scripting style guidelines + * Documentations have been updated +# Datasets + * Ouranos ESPO-G6-R2 CMIP6 script added (~9TBs) + * NASA GDDP-NEX-CMIP CMIP6 script added (~37TBs) + * Alberta Governments CMIP6 script added (~0.1TBs) + [v0.4.1] - September 21st, 2023 # Fixed * minor bug fixes From 0946306151b75a3fba9caa5962b608beeb3e801e Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 17:41:52 -0500 Subject: [PATCH 27/34] Adding a section for WIP directories --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8c9d282..7cfce85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ .ipynb_checkpoints .DS_Store *.swp + +# WIP folders +scripts/ouranos-crcm5-cmip6/ From fadeae982e12a44ec593d29d92af42b3e0b9c29f Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 17:42:00 -0500 Subject: [PATCH 28/34] Restructuring script directory With the growing number of scripts, this commit tries to restructure this directory to provide more clarity and organization for the users. Signed-off-by: Kasra Keshavarz --- scripts/bcc_csm2_mr/README.md | 0 scripts/bcc_csm2_mr/bcc_csm2_mr.sh | 197 ----- scripts/canrcm4_wfdei_gem_capa/LICENSE | 675 ------------------ scripts/canrcm4_wfdei_gem_capa/README.md | 91 --- .../canrcm4_wfdei_gem_capa.sh | 188 ----- scripts/cnrm_cm6_1/cnrm_cm6_1.sh | 197 ----- scripts/conus_i/LICENSE | 675 ------------------ scripts/conus_i/README.md | 113 --- scripts/conus_i/conus_i.sh | 532 -------------- scripts/conus_ii/LICENSE | 675 ------------------ scripts/conus_ii/README.md | 121 ---- scripts/conus_ii/conus_ii.sh | 544 -------------- scripts/daymet/LICENSE | 675 ------------------ scripts/daymet/README.md | 93 --- scripts/daymet/daymet.sh | 359 ---------- scripts/ec_earth3_veg/ec_earth3_veg.sh | 197 ----- scripts/era5/ERA5_downloads.zip | Bin 21373 -> 0 bytes scripts/era5/LICENSE | 675 ------------------ scripts/era5/README.md | 60 -- scripts/era5/era5.sh | 507 ------------- scripts/era5/era5_simplified.sh | 195 ----- scripts/espo-g6-r2/README.md | 74 -- scripts/espo-g6-r2/espo-g6-r2.sh | 420 ----------- scripts/gfdl_cm4/gfdl_cm4.sh | 197 ----- scripts/gfdl_esm4/gfdl_esm4.sh | 197 ----- scripts/hybrid_obs/README.md | 59 -- scripts/hybrid_obs/hybrid_obs.sh | 198 ----- scripts/ipsl_cm6a_lr/ipsl_cm6a_lr.sh | 197 ----- scripts/mri_esm2_0/mri_esm2_0.sh | 197 ----- scripts/rdrs/LICENSE | 675 ------------------ scripts/rdrs/README.md | 74 -- scripts/rdrs/rdrs.sh | 220 ------ scripts/wfdei_gem_capa/README.md | 51 -- scripts/wfdei_gem_capa/wfdei_gem_capa.sh | 185 ----- 34 files changed, 9513 deletions(-) delete mode 100644 scripts/bcc_csm2_mr/README.md delete mode 100755 scripts/bcc_csm2_mr/bcc_csm2_mr.sh delete mode 100644 scripts/canrcm4_wfdei_gem_capa/LICENSE delete mode 100644 scripts/canrcm4_wfdei_gem_capa/README.md delete mode 100755 scripts/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh delete mode 100755 scripts/cnrm_cm6_1/cnrm_cm6_1.sh delete mode 100644 scripts/conus_i/LICENSE delete mode 100644 scripts/conus_i/README.md delete mode 100755 scripts/conus_i/conus_i.sh delete mode 100644 scripts/conus_ii/LICENSE delete mode 100644 scripts/conus_ii/README.md delete mode 100755 scripts/conus_ii/conus_ii.sh delete mode 100644 scripts/daymet/LICENSE delete mode 100644 scripts/daymet/README.md delete mode 100755 scripts/daymet/daymet.sh delete mode 100755 scripts/ec_earth3_veg/ec_earth3_veg.sh delete mode 100644 scripts/era5/ERA5_downloads.zip delete mode 100644 scripts/era5/LICENSE delete mode 100644 scripts/era5/README.md delete mode 100755 scripts/era5/era5.sh delete mode 100755 scripts/era5/era5_simplified.sh delete mode 100644 scripts/espo-g6-r2/README.md delete mode 100755 scripts/espo-g6-r2/espo-g6-r2.sh delete mode 100755 scripts/gfdl_cm4/gfdl_cm4.sh delete mode 100755 scripts/gfdl_esm4/gfdl_esm4.sh delete mode 100644 scripts/hybrid_obs/README.md delete mode 100755 scripts/hybrid_obs/hybrid_obs.sh delete mode 100755 scripts/ipsl_cm6a_lr/ipsl_cm6a_lr.sh delete mode 100755 scripts/mri_esm2_0/mri_esm2_0.sh delete mode 100644 scripts/rdrs/LICENSE delete mode 100644 scripts/rdrs/README.md delete mode 100755 scripts/rdrs/rdrs.sh delete mode 100644 scripts/wfdei_gem_capa/README.md delete mode 100755 scripts/wfdei_gem_capa/wfdei_gem_capa.sh diff --git a/scripts/bcc_csm2_mr/README.md b/scripts/bcc_csm2_mr/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/bcc_csm2_mr/bcc_csm2_mr.sh b/scripts/bcc_csm2_mr/bcc_csm2_mr.sh deleted file mode 100755 index 3fdf38a..0000000 --- a/scripts/bcc_csm2_mr/bcc_csm2_mr.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n bcc-cm2-mr -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_BCC-CSM2-MR_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing BCC-CSM2-MR dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/canrcm4_wfdei_gem_capa/LICENSE b/scripts/canrcm4_wfdei_gem_capa/LICENSE deleted file mode 100644 index 53d1f3d..0000000 --- a/scripts/canrcm4_wfdei_gem_capa/LICENSE +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/canrcm4_wfdei_gem_capa/README.md b/scripts/canrcm4_wfdei_gem_capa/README.md deleted file mode 100644 index de8863b..0000000 --- a/scripts/canrcm4_wfdei_gem_capa/README.md +++ /dev/null @@ -1,91 +0,0 @@ -# CCRN `CanRCM4-WFDEI-GEM-CaPA` -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The `CanRCM4-WFDEI-GEM-CaPA` dataset is located under the following directory accessible from Digital Alliance of Canada (formerly Compute Canada) Graham cluster: -``` -/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA -├── r8i2p1r1 -│ ├── hus_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ ├── pr_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ ├── ps_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ ├── rlds_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ ├── rsds_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ ├── ta_r8i2p1r1_z1_1951-2100.Feb29.nc4 -│ └── wind_r8i2p1r1_z1_1951-2100.Feb29.nc4 -. -. -. -├── %ensembleMember -│ ├── hus_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ ├── pr_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ ├── ps_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ ├── rlds_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ ├── rsds_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ ├── ta_%ensembleMember_z1_1951-2100.Feb29.nc4 -│ └── wind_%ensembleMembe_z1_1951-2100.Feb29.nc4 -. -. -. -└── r10i2p1r5 - ├── hus_%ensembleMember_z1_1951-2100.Feb29.nc4 - ├── pr_%ensembleMember_z1_1951-2100.Feb29.nc4 - ├── ps_%ensembleMember_z1_1951-2100.Feb29.nc4 - ├── rlds_%ensembleMember_z1_1951-2100.Feb29.nc4 - ├── rsds_%ensembleMember_z1_1951-2100.Feb29.nc4 - ├── ta_%ensembleMember_z1_1951-2100.Feb29.nc4 - └── wind_%ensembleMembe_z1_1951-2100.Feb29.nc4 -``` - -Below is a list of ensemble members for this datase: -``` -/project/rpp-kshook/Model_Output/280_CanRCM4_Cor_WFDEI-GEM-CaPA -├── r10i2p1r1 -├── r10i2p1r2 -├── r10i2p1r3 -├── r10i2p1r4 -├── r10i2p1r5 -├── r8i2p1r1 -├── r8i2p1r2 -├── r8i2p1r3 -├── r8i2p1r4 -├── r8i2p1r5 -├── r9i2p1r1 -├── r9i2p1r2 -├── r9i2p1r3 -├── r9i2p1r4 -└── r9i2p1r5 -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `CanRCM4-WFDEI-GEM-CaPA` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -Each NetCDF file belongs to a single variable. The list of variables included in the dataset is descriped in [Short Description on Dataset Variables](##short-description-on-dataset-variables) -## Spatial Extent -The spatial extent of the dataset is on latitutes from `31.0625` to `71.9375` and longitudes from `-149.9375` to `-50.0625` covering North America. The resolution is 0.125 degrees. - -## Temporal Extent -The time-steps are 3-hourly covering from `January 1951` to `December 2100`. - -## Short Description on Dataset Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at the Lowest Model Level (sigma=0.995), 2) surface pressure, 3) air temperature at the Lowest Model Level, 4) wind speed at the Lowest Model Level (sigma=0.995), 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `CanRCM4-WFDEI-GEM-CaPA` dataset and their details are described in the table below: - -|Variable Name |Dataset Variable |Unit |IPCC abbreviation|Comments | -|----------------------|-------------------|-----|-----------------|----------------------| -|surface pressure |ps |Pa |ps |surface pressure | -|specific humidity@1.5m|hus |1 |huss |Specific Humidity at Lowest Model Level (sigma=0.995)| -|air tempreature @1.5m |ta |K |tas |Air Temperature at Lowest Model Level (sigma=0.995)| -|wind speed @10m |wind |m s-1|wspd |Wind Modulus at Lowest Model Level (sigma=0.995)| -|precipitation |pr |kg m-2 s-1| |precipitation flux | -|short wave radiation |rsds |W m-2|rsds |Surface Downwelling Shortwave Flux| -|long wave radiation |lsds |W m-2|rlds |Surface Downwelling Longwave Flux| - diff --git a/scripts/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh b/scripts/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh deleted file mode 100755 index e377a79..0000000 --- a/scripts/canrcm4_wfdei_gem_capa/canrcm4_wfdei_gem_capa.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # optional - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - IFS=' ' read -ra ensembleArr <<< $(echo $(cd $datasetDir && ls -d */ | cut -d '/' -f 1)) -fi - -# make array of variable names -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -fileStruct="z1_1951-2100.Feb29.nc4" # source dataset files' suffix constant - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing CCRN CanRCM4-WFDEI-GEM_CaPA..." - -# make the output directory -echo "$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# reformat $startDate and $endDate -startDateFormated="$(date --date="$startDate" +"$format")" # startDate -endDateFormated="$(date --date="$endDate" +"$format")" # endDate - -# extract $startYear and $endYear -startYear="$(date --date="$startDate" +"%Y")" -endYear="$(date --date="$endDate" +"%Y")" - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(basename $0): processing member $member" - mkdir -p "$outputDir/$member" # making the output directory - - # loop over variables - for var in "${variablesArr[@]}"; do - ncks -O -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -d "$timeVar","$startDateFormated","$endDateFormated" \ - "$datasetDir/$member/${var}_${member}_${fileStruct}" "$outputDir/$member/${prefix}${var}_${member}_z1_${startYear}-${endYear}.Feb29.nc4" - done - - # wait to make sure the for loop is finished - wait - -done - - -echo "$(basename $0): results are produced under $outputDir." - diff --git a/scripts/cnrm_cm6_1/cnrm_cm6_1.sh b/scripts/cnrm_cm6_1/cnrm_cm6_1.sh deleted file mode 100755 index 1cc4454..0000000 --- a/scripts/cnrm_cm6_1/cnrm_cm6_1.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n cnrm_cm6_1 -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_CNRM-CM6-1_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing CNRM-CM6-1 dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/conus_i/LICENSE b/scripts/conus_i/LICENSE deleted file mode 100644 index 53d1f3d..0000000 --- a/scripts/conus_i/LICENSE +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/conus_i/README.md b/scripts/conus_i/README.md deleted file mode 100644 index 802c26e..0000000 --- a/scripts/conus_i/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# NCAR-GWF `WRF-CONUSI` Simulation Outputs - -In this file, the details of the dataset is explained. - -> [!CAUTION] -> `WRF-CONUSI` dataset needs extensive I/O operations in the `cache` -> directory. So, in case of submitting `SLURM` jobs, it is recommended to -> use the `$SLURM_TMPDIR` directory as `cache`. This can be provided to -> the main `extract-dataset.sh` script using the `--cache='$SLURM_TMPDIR'` -> option. Please be mindful of **single** quotation marks for the argument. - -## Location of Dataset Files -The `WRF-CONUSI` simulation outputs are located under the following directory accessible from Compute Canada (CC) Graham Cluster: -``` -/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/ -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/ -├── 2000 -│ ├── wrf2d_d01_2000-10-01_00:00:00 -│ ├── wrf2d_d01_2000-10-01_01:00:00 -│ ├── . -│ ├── . -│ ├── . -│ └── wrf2d_d01-2000-12-31_23:00:00 -. -. -. -├── %Y -│ ├── wrf2d_d01_%Y-%m-%d_%H:00:00 -│ ├── . -│ ├── . -│ ├── . -│ . -│ . -│ . -. -. -. -└── 2013 - ├── . - ├── . - ├── . - └── wrf2d_d01-2013-12-31_23:00:00 - -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `WRF-CONUSI` simulations are located outside of the main dataset files. The NetCDF file containing the coordinate varibles could be found at the following: -```console -/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/coord.nc -``` -However, upon many trials by the author, the variables were not concatenated with the main NetCDF files easily. A workaround has been provided to add at least two necessary coordinate variables, i.e., `XLAT` and `XLONG`, to the WRF simulation files. These two coordinates are enough to work with almost all of the meteorological variables included in the dataset. The following scripts are used on Compute Canada (CC) Graham Cluster to produce the substitute NetCDF file containing coordinate variables: -```console -# make a copy of coordinate variable netCDF file first -foo@bar:~$ module load cdo/2.0.4; module load nco/5.0.6; -foo@bar:~$ coordFile="/project/6008034/Model_Output/WRF/CONUS/CTRL/coord.nc" -foo@bar:~$ ncks -O -v XLAT,XLONG "$coordFile" coord2.nc -foo@bar:~$ nccopy -4 coord2.nc coord_new.nc -foo@bar:~$ ncatted -O -a FieldType,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a MemoryOrder,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a stagger,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a coordinates,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a sr_x,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a sr_y,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a FieldType,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a MemoryOrder,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a stagger,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a coordinates,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a sr_x,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a sr_y,XLONG,d,, coord_new.nc -foo@bar:~$ ncwa -O -a Times coord_new.nc coord_new.nc -foo@bar:~$ ncks -O -v XLAT,XLONG coord_new.nc coord_new.nc -foo@bar:~$ ncrename -a XLONG@description,long_name coord_new.nc -foo@bar:~$ ncrename -a XLAT@description,long_name coord_new.nc -foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc -foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc -foo@bar:~$ ncatted -O -a ,global,d,, coord_new.nc -foo@bar:~$ ncatted -O -h -a license,global,c,c,"GNU General Public License v3 (GPLv3)" coord_new.nc -``` -Furthermore, the substitute NetCDF file containing the coordinate variables are located at `/asset/coord_XLAT_XLONG_conus_i.nc` within this repository. The workaround NetCDF is automatically being used by the script to add the `XLAT` and `XLONG` variables to the final, produced files. - -### Time-stamps -The time-stamp of the time-steps are missing from the dataset NetCDF files. However, the time-stamps for each time-step is obvious from the file names. The time-stamp pattern of the dataset files is as following: `%Y-%m-%d_%H:00:00` which will be changed to `%Y-%m-%s %H:00:00` to be registered as a valid time-stamp in the NetCDF files. The script is able to set the time-stamps for the final produced file(s) automatically. - -## Dataset Variables -The NetCDF files of the dataset contain 281 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: -```console -foo@bar:~$ module load cdo/2.0.4 -foo@bar:~$ module load nco/5.0.6 -foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/WRF/CONUS/CTRL/2000/wrf2d_d01_2000-10-01_00:00:00 -``` - -## Spatial Extent -The spatial extent of the `WRF-CONUSI` is on latitutes from `18.13629` to `57.91813` and longitudes from `-139.0548` to `-56.94519`. - -## Temporal Extent -As is obvious from the nomenclature of the dataset files, the time-steps are hourly covering from the October 2000 to September 2013. - -# Short Description on `WRF-CONUSI` Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WRF-CONUSI` dataset and their details are described in the table below: -|Variable Name |WRF-CONUSI Variable|Unit |IPCC abbreviation|Comments | -|---------------------|-------------------|-----|-----------------|--------------------| -|surface pressure |PSFC |Pa |ps | | -|specific humidity @2m|Q2 |1 |huss | | -|air tempreature @2m |T2 |k |tas | | -|wind speed @10m |U10,V10 |m/s |wspd |WIND=SQRT(U102+V102)| -|precipitation |PREC_ACC_NC |mm/hr| |accumulated precipitation over one hour| -|short wave radiation |ACSWDNB |W m-2|rsds | | -|long wave radiation |ACLWDNB |W m-2|rlds | | diff --git a/scripts/conus_i/conus_i.sh b/scripts/conus_i/conus_i.sh deleted file mode 100755 index e78d400..0000000 --- a/scripts/conus_i/conus_i.sh +++ /dev/null @@ -1,532 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html -# 2. Dr. Zhenhua Li provided scripts to extract and process CONUSI datasets -# 3. Dr. Shervan Gharari produced the netCDF file containing XLAT and XLONG -# coordinate variables put under /assets/coord_XLAT_XLONG_conus_i.nc. -# 4. Sorting workflow is taken from the following link; https://stackoverflow.com/a/11789688/5188208 - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-io DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL]" -} - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:c:p:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,cache:,prefix:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # required - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -c | --cache) cacheDir="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - - -# ===================== -# Necessary Assumptions -# ===================== - -# hard-coding the address of the co-ordinate NetCDF files -# containing XLAT and XLONG variables each having dimensions -# of "south_north" and "west_east". -coordMainFile="/project/rpp-kshook/Model_Output/WRF/CONUS/coord.nc" -coordEssFile="$(dirname $0)/../../assets/coord_XLAT_XLONG_conus_i.nc" -latVar="south_north" -lonVar="west_east" - -# The structure of file names is as follows: "wrf2d_d01_YYYY-MM-DD_HH:MM:SS" (no file extension) -format="%Y-%m-%d_%H:%M:%S" -fileStruct="wrf2d_d01" -coordIdxScript="$(dirname $0)/../../assets/coord_wrf_idx.ncl" - -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4; - module -q load nco/5.0.6; -} -load_core_modules # load necessary modules - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#format date string -format_date () { date --date="$1" +"$2"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -####################################### -# Implements the necessary netCDF -# operations using CDO and NCO -# -# Globals: -# coordFile: coordinate variables .nc -# file -# lonLims: longitute bounds -# latLims: latitute bounds -# cacheDir: temporary directory for -# file manipulations -# yr: year of selected forcing data -# outputDir: output directory for -# final files -# -# Arguments: -# 1: -> fName: data file name -# 2: -> fDate: date of the forcing -# 3: -> fTime: time of the forcing -####################################### -generate_netcdf () { - - # defining local variables - local fName="$1" # raw file name string - local fDate="$2" # file string date (YYYY-MM-DD) - local fTime="$3" # file string time (HH:MM:SS) - local fTempDir="$4" # file directory path - local fOutDir="$5" # file output path - local fTimeScale="$6" # fime scale to check the file name - - # add _cat if necessary - if [[ "${fTimeScale,,}" != "h" ]]; then - local fExt="_cat.nc" - fi - - # necessary netCDF operations - ## add coordinate variables: XLAT and XLONG - ncks -A -v XLONG,XLAT "$coordFileSubset" "${fTempDir}/${fName}${fExt}" - ## set time axes - cdo -s -f nc4c -z zip_1 -r settaxis,"$fDate","$fTime",1hour "${fTempDir}/${fName}${fExt}" "${fTempDir}/${fName}_taxis.nc"; - ## rename the `description` attribute - # ncrename -O -a .description,long_name "${fTempDir}/${fName}_taxis.nc" -o "${fOutDir}/${fName}.nc" - mv "${fTempDir}/${fName}_taxis.nc" "${fOutDir}/${prefix}${fName}.nc" -} - - -####################################### -# extracts file name, date, and time -# from CONUSI file name strings. -# -# Globals: -# fileName: file name of the .nc data -# fileNameDate: date (YYYY-MM-DD) -# fileNameYear: year (YYYY) -# fileNameMonth: month (MM) -# fileNameDay: day (DD) -# fileNametime: time (HH:MM:SS) -# -# Arguments: -# 1: -> fName: the -# -# Outputs: -# produces the following global -# variables: -# a) fileName -# b) fileNameDate -# c) fileNameYear -# d) fileNameMonth -# e) fileNameDay -# f) fileNameTime -####################################### -extract_file_info () { - - # define local variable for input argument - local fPath="$1" # format: "/path/to/file/wrf2d_d01_YYYY-MM-DD_HH:MM:SS" - - # file name - fileName=$(basename "$fPath") # file name - - # file date - fileNameDate=$(echo "$fileName" | cut -d '_' -f 3) # file date (YYYY-MM-DD) - - # parts of the date - fileNameYear=$(echo "$fileNameDate" | cut -d '-' -f 1) # file year (YYYY) - fileNameMonth=$(echo "$fileNameDate" | cut -d '-' -f 2) # file month (MM) - fileNameDay=$(echo "$fileNameDate" | cut -d '-' -f 3) #file name day (DD) - - # file hour - fileNameTime=$(echo "$fileName" | cut -d '_' -f 4) # file time (HH:MM:SS) -} - - -####################################### -# function for extracting the index of -# first match between $str and that of -# the ordered array elements -# -# Globals: -# idx: index of the first match -# -# Arguments: -# 1: the string to be matched with -# 2: the array containing strings to -# be checked -# 3: the position within the matching -# string split by '-' -####################################### -date_match_idx () { - - # defining local variables - local str="$1" # string to be matched - local matchPos="$2" # the position of the matching string within the "YYYY-MM-DD", - # 1: year, 2: month, 3: day - # 1,2: year and month, 2,3: month and day, 1,3: year and day - # 1-3: complete date - local delim="$3" # delimiter - shift 3 # shift argument positins by 3 - local strArr=("$@") # arrays of string - - # index variable - idx=0 - - # looping through the $strArr - for s in "${strArr[@]}"; do - if [[ "$str" == $(echo "$s" | cut -d ${delim} -f "$matchPos") ]]; then - break - else - idx=`expr $idx + 1` - fi - done -} - - -####################################### -# concatenating files based on a speci- -# temporal scale. -# -# Globals: -# None -# -# Arguments: -# 1: name of the concatenated file -# 2: destination directory -# 3-: array of file paths -# -# Outputs: -# produces $fName_cat.nc under $fDir -# out of all elements of $filesArr -####################################### -concat_files () { - # defining local variables - local fName="$1" # output file name - local fTempDir="$2" # temporary directory - shift 2 # shift arguments by 2 positions - local filesArr=("$@") # array of file names - - # concatenating $files and producing a single $fName.nc - ncrcat "${filesArr[@]}" "${fTempDir}/${fName}_cat.nc" -} - - -####################################### -# populating arrays with date and time -# values. -# -# Globals: -# datesArr: array of date values -# monthsArr: array of year-month va- -# lues -# timesArr: array of time values -# files: array of file paths -# fileNameDate: date of the current -# filename -# fileNameYear: year of the current -# filename -# fileNameMonth: month of the current -# filename -# uniqueMonthsArr: array of unique -# months -# uniqueDatesArr: array of unique -# dates -# -# Arguments: -# None -# -# Outputs: -# produces the following variables: -# 1) datesArr -# 2) monthsArr -# 3) timesArr -# 4) uniqueMonthsArr -# 5) unqiueDatesArr -####################################### -populate_date_arrays () { - # defining empty arrays - datesArr=(); - monthsArr=(); - timesArr=(); - - for f in "${files[@]}"; do - extract_file_info "$f" # extract necessary information - - # populate date arrays - datesArr+=(${fileNameDate}); - monthsArr+=("${fileNameYear}-${fileNameMonth}"); - timesArr+=(${fileNameTime}); - done - - uniqueMonthsArr=($(echo "${monthsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); - uniqueDatesArr=($(echo "${datesArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); -} - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing NCAR-GWF CONUSI..." - -# make the output directory -mkdir -p "$outputDir" # create output directory - -# constructing the range of years -startYear=$(date --date="$startDate" "+%Y") # start year (first folder) -endYear=$(date --date="$endDate" "+%Y") # end year (last folder) -yearsRange=$(seq $startYear $endYear) - -# constructing $toDate and $endDate in unix time EPOCH -toDate=$startDate -toDateUnix=$(date --date="$startDate" "+%s") # first date in unix EPOCH time -endDateUnix=$(date --date="$endDate" "+%s") # end date in unix EPOCH time - -# extract the associated indices corresponding to latLims and lonLims -module -q load ncl/6.6.2 -## min and max of latitude and longitude limits -minLat=$(echo $latLims | cut -d ',' -f 1) -maxLat=$(echo $latLims | cut -d ',' -f 2) -minLon=$(echo $lonLims | cut -d ',' -f 1) -maxLon=$(echo $lonLims | cut -d ',' -f 2) -## extract coord -coordIdx="$(ncl -nQ 'coord_file='\"$coordMainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" -lonLimsIdx="$(echo $coordIdx | cut -d ' ' -f 1)" -latLimsIdx="$(echo $coordIdx | cut -d ' ' -f 2)" -module -q unload ncl/6.6.2 -load_core_modules - -# produce subsetted $coordFile as well -mkdir -p "$cacheDir" -coordFileSubset="${cacheDir}/coordFileSubset.nc" -# if subsetted coordinate file does not exist, make one -if [[ -f "$coordFileSubset" ]]; then - : -else - ncks -v "XLAT,XLONG" \ - -d "$latVar","$latLimsIdx" \ - -d "$lonVar","$lonLimsIdx" \ - "$coordEssFile" "$coordFileSubset" || true -fi - -# for each year (folder) do the following calculations -for yr in $yearsRange; do - - # creating a temporary directory for temporary files - echo "$(basename $0): creating cache files for year $yr in $cacheDir" - mkdir -p "$cacheDir/$yr" # making the directory - - # setting the end point, either the end of current year, or the $endDate - endOfCurrentYearUnix=$(date --date="$yr-01-01 +1year -1hour" "+%s") # last time-step of the current year - if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then - endPointUnix=$endOfCurrentYearUnix - else - endPointUnix=$endDateUnix - fi - - # extract variables from the forcing data files - while [[ "$toDateUnix" -le "$endPointUnix" ]]; do - # date manipulations - toDateFormatted=$(date --date "$toDate" "+$format") # current timestamp formatted to conform to CONUSI naming convention - - # creating file name - file="${fileStruct}_${toDateFormatted}" # current file name - - # extracting variables from the files and spatial subsetting - ncks -O -v "$variables" \ - -d "$latVar","$latLimsIdx" \ - -d "$lonVar","$lonLimsIdx" \ - "$datasetDir/$yr/$file" "$cacheDir/$yr/$file" & # extracting $variables - [ $( jobs | wc -l ) -ge $( nproc ) ] && wait - - # increment time-step by one unit - toDate=$(date --date "$toDate 1hour") # current time-step - toDateUnix=$(date --date="$toDate" "+%s") # current timestamp in unix EPOCH time - done - - # wait to make sure the while loop is finished - wait - - # go to the next year if necessary - if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then - toDate=$(date --date "$toDate 1hour") - fi - - # make the output directory - mkdir -p "$outputDir/$yr/" - - # data files for the current year with extracted $variables - files=($cacheDir/$yr/*) - # sorting files to make sure the time-series is correct - IFS=$'\n' files=($(sort <<<"${files[*]}")); unset IFS - - # check the $timeScale variable - case "${timeScale,,}" in - - h) - # going through every hourly file - for f in "${files[@]}"; do - # extracting information - extract_file_info "$f" - # necessary NetCDF operations - generate_netcdf "${fileName}" "$fileNameDate" "$fileNameTime" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - d) - # construct the date arrays - populate_date_arrays - - # for each date (i.e., YYYY-MM-DD) - for d in "${uniqueDatesArr[@]}"; do - # find the index of the $timesArr corresponding to $d -> $idx - date_match_idx "$d" "1-3" "-" "${datesArr[@]}" - - # concatenate hourly netCDF files to daily file, i.e., already produces _cat.nc files - dailyFiles=($cacheDir/$yr/${fileStruct}_${d}*) - concat_files "${fileStruct}_${d}" "$cacheDir/$yr/" "${dailyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${d}" "$d" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - m) - # construct the date arrays - populate_date_arrays - - # for each date (i.e., YYYY-MM-DD) - for m in "${uniqueMonthsArr[@]}"; do - # find the index of the $timesArr corresponding to $d -> $idx - # $m is in 'YYYY-MM' format - date_match_idx "$m" "1,2" "-" "${datesArr[@]}" - - # concatenate hourly netCDF files to monthly files, i.e., already produced *_cat.nc files - monthlyFiles=($cacheDir/$yr/${fileStruct}_${m}*) - concat_files "${fileStruct}_${m}" "$cacheDir/$yr/" "${monthlyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${m}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - y) - # construct the date arrays - populate_date_arrays - - # find the index of the $timesArr and $datesArr corresponding to $d -> $idx - date_match_idx "$yr" "1" "-" "${datesArr[@]}" - - # concatenate hourly to yearly files - produced _cat.nc files - yearlyFiles=($cacheDir/$yr/${fileStruct}_${yr}*) - concat_files "${fileStruct}_${yr}" "$cacheDir/$yr/" "${yearlyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${yr}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - ;; - - esac -done - -mkdir "$HOME/empty_dir" -echo "$(basename $0): deleting temporary files from $cacheDir" -rsync -aP --delete "$HOME/empty_dir/" "$cacheDir" -rm -r "$cacheDir" -echo "$(basename $0): temporary files from $cacheDir are removed" -echo "$(basename $0): results are produced under $outputDir" - diff --git a/scripts/conus_ii/LICENSE b/scripts/conus_ii/LICENSE deleted file mode 100644 index 53d1f3d..0000000 --- a/scripts/conus_ii/LICENSE +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/conus_ii/README.md b/scripts/conus_ii/README.md deleted file mode 100644 index caa9fbe..0000000 --- a/scripts/conus_ii/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# NCAR-GWF `WRF-CONUSII` Simulation Outputs - -In this file, the details of the dataset is explained. - -> [!WARNING] -> It must be noted that the `WRF-CONUSII` dataset are in `.tar` format and the script untars the files automatically. - -> [!CAUTION] -> `WRF-CONUSII` dataset needs extensive I/O operations in the `cache` -> directory. So, in case of submitting `SLURM` jobs, it is recommended to -> use the `$SLURM_TMPDIR` directory as `cache`. This can be provided to -> the main `extract-dataset.sh` script using the `--cache='$SLURM_TMPDIR'` -> option. Please be mindful of **single** quotation marks for the argument. - -## Location of Dataset Files - -The `WRF-CONUSII` simulation outputs are located under the following directory accessible from Compute Canada (CC) Graham Cluster: -``` -/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist -├── 1995 -│ ├── wrf2d_conusii_19950101.tar -│ ├── wrf2d_conusii_19950102.tar -│ ├── . -│ ├── . -│ ├── . -│ └── wrf2d_conusii_19951231.tar -. -. -. -├── %Y -│ ├── wrf2d_conusii_%Y%m%d.tar -│ ├── . -│ ├── . -│ ├── . -│ . -│ . -│ . -. -. -. -└── 2015 - ├── . - ├── . - ├── . - └── wrf2d_conusii_20151231.tar -``` -And, each `.tar` file has the following structure of content: -``` -foo@bar:~$ tar --strip-components=5 -xvf wrf2d_conusii_%Y%m%d.tar > /dev/null -/path/to/tar/extracted/files -└── %Y - ├── wrf2d_d01_%Y-%m-%d_00:00:00 - . - . - . - └── wrf2d_d01_%Y-%m-%d_23:00:00 -``` -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `WRF-CONUSII` simulations are located outside of the main dataset files. The NetCDF file containing the coordinate varibles could be found at the following: -```console -/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc -``` -However, upon many trials by the author, the variables were not concatenated with the main NetCDF files easily. A workaround has been provided to add at least two necessary coordinate variables, i.e., `XLAT` and `XLONG`, to the WRF simulation files. These two coordinates are enough to work with almost all of the meteorological variables included in the dataset. The following scripts are used on Compute Canada (CC) Graham Cluster to produce the substitute NetCDF file containing coordinate variables: -```console -# make a copy of coordinate variable NetCDF file first! -foo@bar:~$ module load cdo/2.0.4; module load nco/5.0.6; # load necessary modules -foo@bar:~$ coordFile="/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc" -foo@bar:~$ ncks -v XLAT,XLONG "$coordFile" coord.nc -foo@bar:~$ nccopy -4 coord.nc coord_new.nc -foo@bar:~$ ncatted -O -a FieldType,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a MemoryOrder,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a stagger,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a coordinates,XLAT,d,, coord_new.nc -foo@bar:~$ ncatted -O -a FieldType,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a MemoryOrder,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a stagger,XLONG,d,, coord_new.nc -foo@bar:~$ ncatted -O -a coordinates,XLONG,d,, coord_new.nc -foo@bar:~$ ncwa -O -a Time coord_new.nc coord_new.nc -foo@bar:~$ ncrename -a XLONG@description,long_name coord_new.nc -foo@bar:~$ ncrename -a XLAT@description,long_name coord_new.nc -foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc -foo@bar:~$ ncatted -O -a cell_methods,,d,, coord_new.nc -foo@bar:~$ ncatted -O -a ,global,d,, coord_new.nc -foo@bar:~$ ncatted -O -h -a license,global,c,c,"GNU General Public License v3 (GPLv3)" coord_new.nc -``` -Furthermore, the substitute NetCDF file containing the coordinate variables are located at `/asset/coord_XLAT_XLONG_conus_i.nc` within this repository. The workaround NetCDF is automatically being used by the script to add the `XLAT` and `XLONG` variables to the final, produced files. - -### Time-stamps -Each hourly, extracted NetCDF files from `.tar` files will have a single time-stamp indicating the time-step of the file. - -## Dataset Variables -The NetCDF files of the dataset contain 187 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: -```console -foo@bar:~$ module load cdo/2.0.4 -foo@bar:~$ module load nco/5.0.6 -foo@bar:~$ ncdump -h /path/to/extracted/conusii/netcdf/file.nc -``` - -## Spatial Extent -The spatial extent of the `WRF-CONUSII` is on latitutes from `15.02852` to `73.27542` and longitudes from `-156.8242` to `-40.3046`. - -## Temporal Extent -As is obvious from the nomenclature of the dataset files, the time-steps are hourly covering from the January 1995 to December 2015. - -# Short Description on `WRF-CONUSII` Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WRF-CONUSII` dataset and their details are described in the table below: -|Variable Name |WRF-CONUSII Variable|Unit |IPCC abbreviation|Comments | -|---------------------|--------------------|-----|-----------------|--------------------| -|surface pressure |PSFC |Pa |ps | | -|specific humidity @2m|Q2 |1 |huss | | -|air tempreature @2m |T2 |k |tas | | -|wind speed @10m |U10,V10 |m/s |wspd |WIND=SQRT(U102+V102)| -|precipitation |PREC_ACC_NC |mm/hr| |accumulated precipitation over one hour| -|short wave radiation |ACSWDNB |W m-2|rsds | | -|long wave radiation |ACLWDNB |W m-2|rlds | | diff --git a/scripts/conus_ii/conus_ii.sh b/scripts/conus_ii/conus_ii.sh deleted file mode 100755 index 1581cfe..0000000 --- a/scripts/conus_ii/conus_ii.sh +++ /dev/null @@ -1,544 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html -# 2. Dr. Zhenhua Li provided scripts to extract and process CONUSII datasets -# 3. Dr. Shervan Gharari produced the netCDF file containing XLAT and XLONG -# coordinate variables put under /assets/coord_XLAT_XLONG_conus_i.nc. - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-io DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL]" -} - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:c:p:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,cache:,prefix:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # required - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -c | --cache) cacheDir="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - -# ===================== -# Necessary Assumptions -# ===================== - -# hard-coding the address of the co-ordinate NetCDF files -# containing XLAT and XLONG variables each having dimensions -# of "south_north" and "west_east". -coordMainFile="/project/rpp-kshook/Model_Output/wrf-conus/CONUSII/hist/wrf04km_coord.nc" -coordEssFile="$(dirname $0)/../../assets/coord_XLAT_XLONG_conus_ii.nc" -latVar="south_north" -lonVar="west_east" - -# The structure of file names is as follows: "wrf2d_d01_YYYY-MM-DD_HH:MM:SS" (no file extension) -format="%Y-%m-%d_%H:%M:%S" -tarFormat="%Y%m%d" -fileStruct="wrf2d_d01" -tarFileStruct="wrf2d_conusii" -coordIdxScript="$(dirname $0)/../../assets/coord_wrf_idx.ncl" - -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4; - module -q load nco/5.0.6; -} -load_core_modules # load necessary modules - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#format date string -format_date () { date --date="$1" +"$2"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -####################################### -# Implements the necessary netCDF -# operations using CDO and NCO -# -# Globals: -# coordFile: coordinate variables .nc -# file -# lonLims: longitute bounds -# latLims: latitute bounds -# cacheDir: temporary directory for -# file manipulations -# yr: year of selected forcing data -# outputDir: output directory for -# final files -# -# Arguments: -# 1: -> fName: data file name -# 2: -> fDate: date of the forcing -# 3: -> fTime: time of the forcing -####################################### -generate_netcdf () { - - # defining local variables - local fName="$1" # raw file name string - local fDate="$2" # file string date (YYYY-MM-DD) - local fTime="$3" # file string time (HH:MM:SS) - local fTempDir="$4" # file directory path - local fOutDir="$5" # file output path - local fTimeScale="$6" # fime scale to check the file name - - # add _cat if necessary - if [[ "${fTimeScale,,}" != "h" ]]; then - local fExt="_cat.nc" - fi - - # necessary netCDF operations - ## add coordinate variables: XLAT and XLONG - ncks -A -v XLONG,XLAT "$coordFileSubset" "${fTempDir}/${fName}${fExt}" - ## set time axes - cdo -s -f nc4c -z zip_1 -r settaxis,"$fDate","$fTime",1hour "${fTempDir}/${fName}${fExt}" "${fTempDir}/${fName}_taxis.nc"; - ## rename the `description` attribute - # ncrename -O -a .description,long_name "${fTempDir}/${fName}_taxis.nc" -o "${fOutDir}/${fName}.nc" - mv "${fTempDir}/${fName}_taxis.nc" "${fOutDir}/${prefix}${fName}.nc" -} - - -####################################### -# extracts file name, date, and time -# from CONUSII file name strings. -# -# Globals: -# fileName: file name of the .nc data -# fileNameDate: date (YYYY-MM-DD) -# fileNameYear: year (YYYY) -# fileNameMonth: month (MM) -# fileNameDay: day (DD) -# fileNametime: time (HH:MM:SS) -# -# Arguments: -# 1: -> fName: the -# -# Outputs: -# produces the following global -# variables: -# a) fileName -# b) fileNameDate -# c) fileNameYear -# d) fileNameMonth -# e) fileNameDay -# f) fileNameTime -####################################### -extract_file_info () { - - # define local variable for input argument - local fPath="$1" # format: "/path/to/file/wrf2d_d01_YYYY-MM-DD_HH:MM:SS" - - # file name - fileName=$(basename "$fPath") # file name - - # file date - fileNameDate=$(echo "$fileName" | cut -d '_' -f 3) # file date (YYYY-MM-DD) - - # parts of the date - fileNameYear=$(echo "$fileNameDate" | cut -d '-' -f 1) # file year (YYYY) - fileNameMonth=$(echo "$fileNameDate" | cut -d '-' -f 2) # file month (MM) - fileNameDay=$(echo "$fileNameDate" | cut -d '-' -f 3) #file name day (DD) - - # file hour - fileNameTime=$(echo "$fileName" | cut -d '_' -f 4) # file time (HH:MM:SS) -} - - -####################################### -# function for extracting the index of -# first match between $str and that of -# the ordered array elements -# -# Globals: -# idx: index of the first match -# -# Arguments: -# 1: the string to be matched with -# 2: the array containing strings to -# be checked -# 3: the position within the matching -# string split by '-' -####################################### -date_match_idx () { - - # defining local variables - local str="$1" # string to be matched - local matchPos="$2" # the position of the matching string within the "YYYY-MM-DD", - # 1: year, 2: month, 3: day - # 1,2: year and month, 2,3: month and day, 1,3: year and day - # 1-3: complete date - local delim="$3" # delimiter - shift 3 # shift argument positins by 3 - local strArr=("$@") # arrays of string - - # index variable - idx=0 - - # looping through the $strArr - for s in "${strArr[@]}"; do - if [[ "$str" == $(echo "$s" | cut -d ${delim} -f "$matchPos") ]]; then - break - else - idx=`expr $idx + 1` - fi - done -} - - -####################################### -# concatenating files based on a speci- -# temporal scale. -# -# Globals: -# None -# -# Arguments: -# 1: name of the concatenated file -# 2: destination directory -# 3-: array of file paths -# -# Outputs: -# produces $fName_cat.nc under $fDir -# out of all elements of $filesArr -####################################### -concat_files () { - # defining local variables - local fName="$1" # output file name - local fTempDir="$2" # temporary directory - shift 2 # shift arguments by 2 positions - local filesArr=("$@") # array of file names - - # concatenating $files and producing a single $fName.nc - ncrcat "${filesArr[@]}" "${fTempDir}/${fName}_cat.nc" -} - - -####################################### -# populating arrays with date and time -# values. -# -# Globals: -# datesArr: array of date values -# monthsArr: array of year-month va- -# lues -# timesArr: array of time values -# files: array of file paths -# fileNameDate: date of the current -# filename -# fileNameYear: year of the current -# filename -# fileNameMonth: month of the current -# filename -# uniqueMonthsArr: array of unique -# months -# uniqueDatesArr: array of unique -# dates -# -# Arguments: -# None -# -# Outputs: -# produces the following variables: -# 1) datesArr -# 2) monthsArr -# 3) timesArr -# 4) uniqueMonthsArr -# 5) unqiueDatesArr -####################################### -populate_date_arrays () { - # defining empty arrays - datesArr=(); - monthsArr=(); - timesArr=(); - - for f in "${files[@]}"; do - extract_file_info "$f" # extract necessary information - - # populate date arrays - datesArr+=(${fileNameDate}); - monthsArr+=("${fileNameYear}-${fileNameMonth}"); - timesArr+=(${fileNameTime}); - done - - uniqueMonthsArr=($(echo "${monthsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); - uniqueDatesArr=($(echo "${datesArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); -} - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing NCAR-GWF CONUSII..." - -# make the output directory -mkdir -p "$outputDir" # create output directory - -# constructing the range of years -startYear=$(date --date="$startDate" "+%Y") # start year (first folder) -endYear=$(date --date="$endDate" "+%Y") # end year (last folder) -yearsRange=$(seq $startYear $endYear) - -# constructing $toDate and $endDate in unix time EPOCH -toDate=$startDate -toDateUnix=$(date --date="$startDate" "+%s") # first date in unix EPOCH time -endDateUnix=$(date --date="$endDate" "+%s") # end date in unix EPOCH time - -# extract the associated indices corresponding to latLims and lonLims -module -q load ncl/6.6.2 -## min and max of latitude and longitude limits -minLat=$(echo $latLims | cut -d ',' -f 1) -maxLat=$(echo $latLims | cut -d ',' -f 2) -minLon=$(echo $lonLims | cut -d ',' -f 1) -maxLon=$(echo $lonLims | cut -d ',' -f 2) -## extract coord -coordIdx="$(ncl -nQ 'coord_file='\"$coordMainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" -lonLimsIdx="$(echo $coordIdx | cut -d ' ' -f 1)" -latLimsIdx="$(echo $coordIdx | cut -d ' ' -f 2)" -module -q unload ncl/6.6.2 -load_core_modules - -# produce subsetted $coordFile as well -mkdir -p "$cacheDir" -coordFileSubset="${cacheDir}/coordFileSubset.nc" -if [[ -f "$coordFileSubset" ]]; then - : -else - ncks -v "XLAT,XLONG" \ - -d "$latVar","$latLimsIdx" \ - -d "$lonVar","$lonLimsIdx" \ - "$coordEssFile" "$coordFileSubset" || true -fi - -# for each year (folder) do the following calculations -for yr in $yearsRange; do - - # creating a temporary directory for temporary files - echo "$(basename $0): creating cache files for year $yr in $cacheDir" - mkdir -p "$cacheDir/$yr" # making the directory - - # setting the end point, either the end of current year, or the $endDate - endOfCurrentYearUnix=$(date --date="$yr-01-01 1year -1day" "+%s") # last time-step of the current year - if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then - endPointUnix=$endOfCurrentYearUnix - else - endPointUnix=$endDateUnix - fi - - # extract variables from the forcing data files - while [[ "$toDateUnix" -le "$endPointUnix" ]]; do - # date manipulations - toDateFormatted=$(date --date "$toDate" "+$tarFormat") # current timestamp formatted to conform to CONUSII naming convention - - # creating file name - file="${tarFileStruct}_${toDateFormatted}.tar" # current file name - - # untar files one day at a time - tar --strip-components=6 -xf "$datasetDir/$yr/$file" -C "$cacheDir/$yr/" - - # parse tar file contents - tarFiles="$(tar -tf $datasetDir/$yr/$file)" - IFS=' ' read -ra tarFiles <<< $(echo $tarFiles) - - # extracting variables from the files and spatial subsetting - for f in "${tarFiles[@]}"; do - f2="$(echo $f | rev | cut -d '/' -f 1 | rev)" - - until ncks -O -v "$variables" \ - -d "$latVar","$latLimsIdx" \ - -d "$lonVar","$lonLimsIdx" \ - "$cacheDir/$yr/$f2" "$cacheDir/$yr/$f2"; do - - echo "$(basename $0): Process killed: restarting process" >$2 - sleep 5; - done - - done - - # increment time-step by one unit - toDate=$(date --date "$toDate 1day") # current time-step - toDateUnix=$(date --date="$toDate" "+%s") # current timestamp in unix EPOCH time - done - - # go to the next year if necessary - if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then - toDate=$(date --date "$toDate 1hour") - fi - - # make the output directory - mkdir -p "$outputDir/$yr/" - - # data files for the current year with extracted $variables - files=($cacheDir/$yr/*) - # sorting files to make sure the time-series is correct - IFS=$'\n' files=($(sort <<<"${files[*]}")); unset IFS - - - # check the $timeScale variable - case "${timeScale,,}" in - - h) - # going through every hourly file - for f in "${files[@]}"; do - # extracting information - extract_file_info "$f" - # necessary NetCDF operations - generate_netcdf "${fileName}" "$fileNameDate" "$fileNameTime" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - d) - # construct the date arrays - populate_date_arrays - - # for each date (i.e., YYYY-MM-DD) - for d in "${uniqueDatesArr[@]}"; do - # find the index of the $timesArr corresponding to $d -> $idx - date_match_idx "$d" "1-3" "-" "${datesArr[@]}" - - # concatenate hourly netCDF files to daily file, i.e., already produces _cat.nc files - dailyFiles=($cacheDir/$yr/${fileStruct}_${d}*) - concat_files "${fileStruct}_${d}" "$cacheDir/$yr/" "${dailyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${d}" "$d" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - m) - # construct the date arrays - populate_date_arrays - - # for each date (i.e., YYYY-MM-DD) - for m in "${uniqueMonthsArr[@]}"; do - # find the index of the $timesArr corresponding to $d -> $idx - # $m is in 'YYYY-MM' format - date_match_idx "$m" "1,2" "-" "${datesArr[@]}" - - # concatenate hourly netCDF files to monthly files, i.e., already produced *_cat.nc files - monthlyFiles=($cacheDir/$yr/${fileStruct}_${m}*) - concat_files "${fileStruct}_${m}" "$cacheDir/$yr/" "${monthlyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${m}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - done - ;; - - y) - # construct the date arrays - populate_date_arrays - - # find the index of the $timesArr and $datesArr corresponding to $d -> $idx - date_match_idx "$yr" "1" "-" "${datesArr[@]}" - - # concatenate hourly to yearly files - produced _cat.nc files - yearlyFiles=($cacheDir/$yr/${fileStruct}_${yr}*) - concat_files "${fileStruct}_${yr}" "$cacheDir/$yr/" "${yearlyFiles[@]}" - - # implement CDO/NCO operations - generate_netcdf "${fileStruct}_${yr}" "${datesArr[$idx]}" "${timesArr[$idx]}" "$cacheDir/$yr/" "$outputDir/$yr/" "$timeScale" - ;; - - esac -done - -mkdir "$HOME/empty_dir" -rsync -aP --delete "$HOME/empty_dir/" "$cacheDir" -rm -r "$cacheDir" -echo "$(basename $0): temporary files from $cacheDir are removed." -echo "$(basename $0): results are produced under $outputDir." - diff --git a/scripts/daymet/LICENSE b/scripts/daymet/LICENSE deleted file mode 100644 index 53d1f3d..0000000 --- a/scripts/daymet/LICENSE +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/daymet/README.md b/scripts/daymet/README.md deleted file mode 100644 index 0f8edee..0000000 --- a/scripts/daymet/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# `Daymet` dataset -In this file, the details of the dataset is explained. - -:warning: the dataset files are divided between three different spatial domains: 1) North America (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). For the moment, only the `na` domain is considered in `datatool`. - -## Location of Dataset Files -The global `Daymet` dataset is located under the following directory accessible from Digital Research Alliance of Canada (DRA) Graham cluster: -``` -/project/rpp-kshook/Model_Output/Daymet_Daily_V4R1/data/ -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/Daymet_Daily_V4R1/data/ -├── daymet_v4_daily_hi_dayl_1980.nc -├── daymet_v4_daily_hi_dayl_1981.nc -├── . -├── . -├── . -├── daymet_v4_daily_hi_dayl_2022.nc -├── . -├── . -├── . -├── daymet_v4_daily_hi_vp_1980.nc -├── . -├── . -├── . -├── daymet_v4_daily_hi_vp_2022.nc -├── . -├── . -├── . -├── daymet_v4_daily_{%domain}_{%variable}_{%year}.nc -├── . -├── . -├── . -├── daymet_v4_daily_na_dayl_1980.nc -├── . -├── . -├── . -├── daymet_v4_daily_na_vp_2022.nc -├── daymet_v4_daily_pr_dayl_1950.nc -├── . -├── . -├── . -└── daymet_v4_daily_pr_vp_2022.n -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `Daymet` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. The coordinate system is 2-dimensional and follows a Lambert Conformal Conic projection system. -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -The NetCDF files of the dataset contain 7 variables. You may see a list of variables by browsing the dataset directory and listing the files. - -## Spatial Extent -The spatial resolutaion of `Daymet` gridded data is 1 $km$. The model files are divided between three different domains: 1) North American (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). Each domains spatial extents are printed in the following table: -|Number |Domain |Latitude extents |Longitude extents | -|:-------------:|:-------------:|:-----------------------------:|:-----------------------------:| -|1 |`na` | `+6.08`° to `+83.79`° | `-180`° to `+180`° | -|2 |`pr` | `+16.85`° to `+19.93`° | `-67.97`° to `-64.13`° | -|3 |`hi` | `+17.96`° to `+23.51`° | `-160.30`° to `-154.78`° | - -:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. - -## Temporal Extent -The time-steps are daily and the temporal extent for each domain is listed in the following table: -|Number |Domain |Time-step interval |Start date |End date | -|:-------------:|:-------------:|:---------------------:|:-------------:|:-------------:| -|1 |`na` |daily |`1980-01-01` |`2022-12-31` | -|2 |`pr` |daily |`1950-01-01` |`2022-12-31` | -|3 |`hi` |daily |`1980-01-01` |`2022-12-31` | - -Also, "[t]he Daymet calendar is based on a standard calendar year. All Daymet years have 1 - 365 days, including leap years. For leap years, the Daymet database includes leap day. Values for December 31 are discarded from leap years to maintain a 365-day year." - -:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. - -## Short Description on `Daymet` Variables -The variables currently available through the `Daymet` dataset and their details are described in the table below, taken from the [source](https://daymet.ornl.gov/overview): - -|Variable Name |`Daymet` Variable |Unit |Comments | -|-----------------------|-----------------------|---------------|-----------------------------------------------------------------------| -|day length |`dayl` |`s/day` |Duration of the daylight period in seconds per day. This calculation is based on the period of the day during which the sun is above a hypothetical flat horizon| -|precipitation |`prcp` |`mm/day` |Daily total precipitation in millimeters per day, sum of all forms converted to water-equivalent. Precipitation occurrence on any given day may be ascertained.| -|shortwave rdiation |`srad` |`W/m2` |Incident shortwave radiation flux density in watts per square meter, taken as an average over the daylight period of the day. NOTE: Daily total radiation `(MJ/m2/day)` can be calculated as follows: `((srad (W/m2) * dayl (s/day)) / l,000,000)`| -|snow water equivalent |`swe ` |`kg/m2` |Snow water equivalent in kilograms per square meter. The amount of water contained within the snowpack.| -|maximum air temperature|`tmax` |`deg C` |Daily maximum 2-meter air temperature in degrees Celsius.| -|minimum air temperature|`tmin` |`deg C` |Daily minimum 2-meter air temperature in degrees Celsius.| -|water vapor pressure |`vp` |`Pa` |Water vapor pressure in pascals. Daily average partial pressure of water vapor.| - -For a complete catalog of the dataset, see [here](https://daymet.ornl.gov/overview). - diff --git a/scripts/daymet/daymet.sh b/scripts/daymet/daymet.sh deleted file mode 100755 index 12d569c..0000000 --- a/scripts/daymet/daymet.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n daymet -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variable) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - -# create $cache directory -mkdir -p $cache - -# ========================== -# Necessary Global Variables -# ========================== -# the structure of file names is as follows: "YYYYMMDD12.nc" -daymetDateFormat="%Y" # Daymet dataset date format -daymetPrefixString="daymet_v4_daily" # source dataset files' prefix constant - -# domains of the dataset files - for now, only "na" domain -domains=("na") #na: North America, pr: Peurto Rico, hi: Hawaii - -# spatial 2-dimentional variable included in the dataset netCDF files -latVar="lat" # latitude variable -lonVar="lon" # longitude variable - -# spatial dimension names included in the dataset netCDF files -latDim="y" # latitude dimension -lonDim="x" # longitude dimension - -# spatial extraction script address -coordIdxScript="$(dirname $0)/../../assets/coord_daymet_idx.ncl" -coordClosestIdxScript="$(dirname $0)/../../assets/coord_closest_daymet_idx.ncl" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -## core modules -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -unload_core_modules () { - # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT - module -q unload cdo/2.0.4 - module -q unload nco/5.0.6 -} -## ncl modules -load_ncl_module () { - module -q load ncl/6.6.2 -} -unload_ncl_module () { - module -q unload ncl/6.6.2 -} - -# loading core modules for the script -load_core_modules - - -####################################### -# useful one-liners -####################################### -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -#maximum of a variable in a netcdf file -ncmax () { ncap2 -O -C -v -s "foo=${2}.max();print(foo)" ${1} "$cache/max_$(basename $1)" | cut -f 3- -d ' ' ; } - -#minimum of a variable in a netcdf file -ncmin () { ncap2 -O -C -v -s "foo=${2}.min();print(foo)" ${1} "$cache/min_$(basename $1)" | cut -f 3- -d ' ' ; } - -#minimum of comma delimited string -delim_min () { IFS=', ' read -r -a l <<< "$@"; printf "%s\n" "${l[@]}" | sort -n | head -n1; } - -#maximum of comma delimited string -delim_max () { IFS=', ' read -r -a l <<< "$@"; printf "%s\n" "${l[@]}" | sort -n | tail -n1; } - - -####################################### -# compare float values using basic -# calculator, i.e., `bc` -# -# Arguments: -# 1: -> firstNum: first int/float -# 2: -> SecondNum: second int/float -# 3: -> operator: comparison operator -####################################### -bc_compare () { - # local variables - local firstNum=$1 - local secondNum=$2 - local operator=$3 - - # implement the comparison - echo "$(bc <<< "$firstNum $operator $secondNum")" -} - -####################################### -# print the full path of the `n`th file -# given the $parentDir and $wildcard -# variables -# -# Arguments: -# 1: -> parentDir: parent directory -# 2: -> wildcard: wildcard to use in -# listing files -# 3: -> nth: nth file to return -####################################### -nth_file () { - # local variables - local parentDir=$1 - local wildcard=$2 - local nth=$3 - - local fileList - - # listing files - fileList=($parentDir/*${wildcard}*) - # printing nth file - echo "${fileList[$nth]}" -} - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing daymet v4 dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -echo "$(log_date)$(basename $0): creating cache directory under $cache" -mkdir -p "$cache" - -# define necessary dates -toDate="$startDate" -toDateUnix="$(unix_epoch "$toDate")" -endDateUnix="$(unix_epoch "$endDate")" - -# create empty arrays for included domains and spatial limits -domainsCovered=() # domain strings -domainsLatIdx=() # latitude index -domainLonIdx=() # longitude index - -# parse the upper and lower bounds of a given spatial limit -minLat=$(echo $latLims | cut -d ',' -f 1) -maxLat=$(echo $latLims | cut -d ',' -f 2) -minLon=$(echo $lonLims | cut -d ',' -f 1) -maxLon=$(echo $lonLims | cut -d ',' -f 2) - -# load NCL module -load_ncl_module - -# extract domains that are included in the given spatial limits -for domain in ${domains[@]}; do - # select a representative file (2nd) for each domain - domainFile=$(nth_file $datasetDir $domain 2) - - # check if the input spatial limits overlap with that of domain files - if [[ $(bc_compare "$(delim_min $latLims)" "$(ncmax $domainFile $latVar)" "<=") -eq 1 ]] && \ - [[ $(bc_compare "$(delim_min $lonLims)" "$(ncmax $domainFile $lonVar)" "<=") -eq 1 ]] && \ - [[ $(bc_compare "$(delim_max $latLims)" "$(ncmin $domainFile $latVar)" ">=") -eq 1 ]] && \ - [[ $(bc_compare "$(delim_max $lonLims)" "$(ncmin $domainFile $lonVar)" ">=") -eq 1 ]]; then - - # extract the associated indices corresponding to latLims and lonLims - coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" - - # if spatial index out-of-bound, i.e., 'ERROR' is return - if [[ "${coordIdx}" == "ERROR" ]]; then - # extract the closest index values - coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordClosestIdxScript")" - fi - - # add covered domains - domainsCovered+=($domain) - - # parse the output index for latitude and longitude - lonLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 1)" - latLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 2)" - - # add the limits to the relevant arrays - domainsLatIdx+=("${latLimsIdx}") - domainsLonIdx+=("${lonLimsIdx}") - fi -done - -# check if $domainsCovered is empty -if [[ "${#domainsCovered[@]}" -eq 0 ]]; then - echo -n "$(log_date)$(basename $0): ERROR! The input spatial limits do not " - echo "overlap with the dataset covered area. Try other extents." - exit 1; -fi - -# unload NCl module -unload_ncl_module - -# load core modules again -load_core_modules - -# make array of variable names -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" - -# status update -echo "$(log_date)$(basename $0): extracting daymet v4 netCDF files..." - -# extract files given the time-series extents -while [[ "$toDateUnix" -le "$endDateUnix" ]]; do - - # date format adjustment - toDateFormatted=$(date --date "$toDate" +"$daymetDateFormat") - - # for each overlapped domain - for idx in $(seq 0 $(bc <<< "${#domainsCovered[@]} - 1")); do - - # for each variable - for var in ${variablesArr[@]}; do - - # generating file name - file="${daymetPrefixString}_${domainsCovered[$idx]}_${var}_${toDateFormatted}.nc" - - # extract $file - ncks -O -d "$latDim","${domainsLatIdx[$idx]}" \ - -d "$lonDim","${domainsLonIdx[$idx]}" \ - "${datasetDir}/${file}" "${outputDir}/${prefix}${file}" - - done # for var - done # for domain's index - - # increment time-step by one unit - toDate="$(date --date "$toDate 1year")" # current time-step - toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time - -done - -# wait to assure the `while` loop is done -wait - -# finalizing the workflow -mkdir "$HOME/empty_dir" -echo "$(basename $0): deleting temporary files from $cache" -rsync -aP --delete "$HOME/empty_dir/" "$cache" -rm -r "$cache" -echo "$(basename $0): temporary files from $cache are removed" -echo "$(basename $0): results are produced under $outputDir" - diff --git a/scripts/ec_earth3_veg/ec_earth3_veg.sh b/scripts/ec_earth3_veg/ec_earth3_veg.sh deleted file mode 100755 index f74b9d6..0000000 --- a/scripts/ec_earth3_veg/ec_earth3_veg.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n ec_earth3_veg -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_EC-Earth3-Veg_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing EC-Earth3-Veg dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/era5/ERA5_downloads.zip b/scripts/era5/ERA5_downloads.zip deleted file mode 100644 index 8db3e96afb4fc77e5891810eb75ef9051efed4b9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21373 zcmbT8WmFwV)~;~~?iO5vy9Wyp+%3V~Ex5b8ySux)OK^90g1gIwp6TgyPo}?_Tj596 zI%`qX*}Lj}YD-Cp0)rp}K)n7Fc4(6U_~*C+0sz1Uzyr|vW@l#dO;bmmUxtTCU0?T` z`8R7_GiyUl6KZn@c{vyW@FuAm4cjnxO?w-6O&fbC06>r%KmdT(UkL!9D>ELrNDJ?L z^EE$v4Fs|5NQHX#6Rrrv5}zo()IGY6gK5w}Bt;30_k`QYK&uKFQ)I-K&4e=pw2fdG z$7)EpOhU{~!1U><5cymcxW3(CVaI*EU>BM;_LnTWV@HhM0Rzw&3zSQGc*nT`(aq+G zTA$3usbiy%sH+@)7FJ$SB~1IG)R8pX7cAW_Dq(AGR!QDz9$2N*D)<4Iy^=55YdJOf zLhMOI!dWnZVFW(k09^n!qZtWl)`J0azG6R;SQySzQ7FcI=--ea`5v&T9+A_RmUWH( znR)S4L`-QR@McuKE%{4;*76-CJLd4xp!^mTF!}P5cXiFFklH3xpi-omob4)&Y}kZ| z71T$QvB70u1{`R_T;nr}=Wj>eHds6)ra_(`u~&%c0vO@l7V^L&1gd^GmSQ?*EV@@< zB|2Gzy9&gHNySP!wE9K%Aw}bABz1M6Uc*V9=GqTApmK<&DwAGsr+9@(7VD<&CN-~h zz-Dnue84;;KFb+Zn0WFYw(6yY_|DVdvk%sjQhL>xil^A%b(_y!B9ZRE>z> z0RNq{ra=p@!kJx3d|c@`+ze?+2)hp}8ADr;_AtgMW;k#u=#b7xiAcWB+_fyPNQKx9 zKuQ!GBB`&;eDT%RZ@wb+FE8SB-v5^uSt0-c5dWJOnOo{wS=m_Xis{$)IzO*iHRVI~2N)a@@^!s!+kix^-X-a6uxJj|PcrM=7Trxx+p%8*1K)8kqY$(~J z=jakCj32o@W`dGw^>N>rn*-1p|UzgzSosv3e{|irrAdi8t@>1(5~dE(UhO{ z@H{Zdv_2m{FBk0=lYBzQrBamDA-^)%=uj6hF^APPt@H6LPH5d?8Tl zk>~Bzg|Q2ML_D6z98)ay_sC$e)i(*MKybpUYKUu32J?D<3rgh!!I?z@&1f5ISp&E` z*W(Vi6b!mG0PExdV^|PsOaL*WFf3GrNv{$lvVgmYuOyQg@_x$YgYVo9FYhR5Xebgs zv4*YBXkk9Gm0D>a@Yz1AIIe$W-ED)!3lnqB&=D;<_@Sp$1N}i)Vy3pA6FvpBrmBH? z*w)~?wQ)*z@@iC@RdKbbvhyJ5Y>-Yiq24>&;G3bI2ZJ7AK4u2YkTsi(>-?66Sd4PG z?0ITS?jc4~RkFGkMgEKEDBbj66n&xOFt872(exNBmK!YJfg=Iv9NuVrt66I zd+J?x?sjz#8Q#wPY%P4$4E%NH(LB6&r^D-Blv9>*MM97VCy&&y&#G8BTHsf(ra3Vg z^|8E2(BbGRI{j6gDCN9K@hqS8AHj~{z64F(*D#ue*BCZ%$`2gnJ0x;)*t2EeY)?vX z8E(Oxv!nHWkeLPQDWsN)r+VL{5~?*&!wYBVE!F0P8k)vmD%L}`GXX>5db^I?!Eitb z;e_DfD+!M|!fl0}xAEvbj5Ct61+ptQ@qOA)ChGdfO25x&Fr|8#bX2=j<2YLmoX8K_ zcSBb0aW09B=IjAcKdjQ;rLlGF*Fm9G!FKy(!fk++dfH!&@+2>s+i6iapu2dyD+n7& zNcODvD8f@~ftvt%D6#(xtKLMMRN2)jMxC_W;=Z<-XYqj6Er)7=;g^S;C0my@DJ766 z*GUL|#5^>vG-b!R-_%~yL+m#Q!tE-92-f0>K$S&|(53)zpHo?+a8hEhAHxn!*3I7! zTJDeoGR12O?<*WV)X7!{>fBs~`BjF#dfv0pmSW0zXd+cK_lTFVL#=MB1yjHD;H!$| z?l9S*wq#dZLEXrCrm?kF-kJgQ9YpYw6U4G3+(N10S4;h_+x|IvW4&86tOZHDl_3NGpU(YvQR^6zO(M-H z+A-bilG|KuDGosWWPjve=CIa(8D-LILALXP+QYTdMhDw9m7sU3Fd44XDH{ux$CQ#` zN+F!S5+t>$&}3v$Ffvp2G>d5GB;q1 zuU#PQW_}LpnCFNFd!bqIuaw)F3V~1!`x0$T$e|$Ti%M0GOed?ASY|vMertZ>VA8n?}w$*tx%zw>8?nBps1%%9Ftq2N7ToX_nr6Qa0C|IG5qPAhI zR7)LU)nr$3P1B_+9^?ukLiN68hs?>!}5p1iFrU&?XXvT(4>-as9N`iLUeS4dGov9|2OUB}+JTJ0WAd4~Io`c72 zNcQ9#Pn#d@Rc5-NCR}5hkI=@iGwBVi37e#-;~gZ!+SCg3;^Iw?|!X> z=dU_R`>KTM>f$_nlClak)arjmtN%t6Tge2p{(@K`;lmXJn*pKE*~m6-heuTq;+`NUqp11&{lNtO2QsK+C4w&#bVEK zv%4dHf_oZRPCu6-Be+>3W&hxQo?!TeV)eAl0LJfW_^57^VsZkuvhFMc+WfdL!&kU7 zzn2z;K8MHMQ*8+@5W~R{xPHkJA0?h{PT}$+i>GfE#`e^#>PJJ4nrUl(NX|pVM3^a=)Lnwme`S;| zE21Jd)(~2tYdPkUz|nveF+m7KnG-il4uSaSiH$FNk|u{p(oX-pLj{dc&mc2vWOe5Hrwf@VU4 zHVSAw;k`xz$}{2XB61VYbFkx;6DBHIIIL)?9^x-F3N(y->$|+XVMw`)0pau?o8Z?f z^O2*<6;Pedi!Ofbc&MDCrX5_WYs(HS!FpM=PWbFviNxgkj!<2l$} z%VveGmU+TjUzkW|`d8Xa{%X9wQB-%=f-vEtY*K(kjs^07+j( zK2c3!YR;wVAyiAw3bSsudOd)r_Qu3Om$9T59w9QNZB~Ov88*?v+!ZmSm-QwbViLq1 zI#>-o)TFPkPOob;?5H0c?5dH4SeGjZnf!r?bb|yxJA+Umm@RTC)w@9`pW~% zu7iKz&N7$OM7{7`9(=;%EACAGhC4LxoJRIWQs7JR(LpZ<30LasQ`U8&yq#cXZksOd z4)p6PaoCcTz;kHM)*OXr@2+H4U`8_B4ns82ldq-whl}`)TWzjAlPW}jMJjWdaz4I5 z$l^~BL>Tq|#Rv!gd0fj(PG8xR+^^F4-DLRg$ z!poZ{VOx>F@A~FBjm@VMmXN_>RVBIZ?Ri+rZu;BveJv7}2pL z5TZC;k;Zsk`7B$jCm^wOPKnSo^k#A9cHPN=$EZzWC{|P2PI{F7xu`dUkbn z*^fku>>GXF`?d(S6!NaqDICz_`!RMQSNp4{!_OmVMi~@b(gn(>BHtIb^gesh2qp(1 zQh||NNMSy<84wQ0B*@wkSaT}Ll!lmplg3c0Uc=JP?PgS;F6BA1K5=xg{4BgQ;J5OM zI}ZQA9TkT++)+I7l7rOt!~h}n+-rtB7BzaWNYaqadl#lPNZ#BXS8bw7Z~$Q=%(qc0 zzc4^Uijda(G0aR&Rc7-#rK_?wzyqj=+|^QQ7j!9)Mhq= zqRrob9(bBOX(p)ShmC1;r+WcGpjuNl$~B2uj~|((D-O+@V`$iPV&qq?E+lr_6bUROBb;uiN!8tty(lM7yX=S={AEsbxYN%Ja=|0VDI zQYC+lI})9*xRZyz*%uIm#SwuXW?EO{0|d^OPxJv1CVZ^~V*wPi(RBV;r2YYj%Tsc# zqvOuOcLQN^7L-_cOcKRO6oV(hJ*>T|N z=D+oP#i4Ku_bP#vcemo!#LPAPTm!*?&wvR+y4TZ{e<;my0ubjED(%7Mb+p~yf)Xu` z8a>G+sqp2L4QduZ-^rCYLYuNPzB5Un+(6;MJm^RxQ9$zpgoAwZz7;c)q4Ml57TWjf zX0Di5+S%=5#J%G0%;$>hq{rrr3SK8#A5@GkZjqD78k;Q}OCnx^+iWmPMM^l87IMXK<`5&q*}&Gi#4X$E_hBo*3CiMKu-ZXE6}X=l=#Mw!x8>m zsom85p;oh~bdyoOXmzhZRV%@$>LlEEy3SOqb`AW^q@Ri8m%qFk?a#DhQ+@caU{2|) zu}S_9+OaY;(>Kvo6#pG~aL|wbb2w))bded}<>sqc1*i^Ft@C`g;qny_Pop6ekwHYZ z&@~p}70eP^5d`c-?PdT$mdn4tC?Dt~0cVpW!Fc zzmvDSn5nWimSCQQNMSVOI5y?5hc{$$>8YAYvq8C7itSn47&^mQNofV8e_dm4xH<(k z!2WRf$u0_zNEnfY&=gc!1*65s^H`_Vd*M{jxIAfMyV5YvTn-xBB*A>o60@e3d9+0h znSbm-k(<1LUr!SVMJC>VY5CzR&77orIsWI$%=GB}jt?!|B?_C+fdi)1p2f$lYg|E{ z=`}eWb@hA=@FFq1ANWC8f}NpQ3T8eGqR^FJc28okaaL;i5#k)9#>*%|6$M+VF-0!L09+o!qK>*M|In zmQN|+#J%McV2@P8I}jBNw4?IqgoVdq4ywa z!GQQC9s%|j4;~)YK81t({pipXgJn8)!fpP@5trFj%JngxYG0a7n3#y$^nR+d;h5bX zrwbw#&KXElW+;b695(ksJNr{v5YZAp5j{5k^y8|zW8Tn{|J8@l9G$1CSZGUEg7pc3 z6^AB>iSOsrkT4jX#j_tsp3QoDtPJ@9M6Ivm(?CDXyr9D8@(&E$BEzh}N z=hkw2ypwHX!wkXnRoCvimSygv%~+=wGcD8HcW*<2`;KpNt(%fC7vIm{^0W)@@=R~U zCCwY)6+@&G79D0Ar+F|cu&jivXt+r=V*$aSriNCLZ z@rfm^HsHVu$!6hPIFz>-N+R-#uzw7qrq?wE2wZBFBdMD~UCdm%J%#mVqWl0R-?Hg* zmmkAXe}8qrePIHBc~!8B_-0J%te>p{h|#ln%LT}eDP6R^U%Rj5P@>4vnrtgg{q;J2 zy@tWLeN)skwH03n6jVZcQ6`=Wx?rpW4uRApVFTIcP}>*WLNOgT%swXe&QylRJcLO# ze_wr!jD}+d#N6TdxCj3 zV_WyznHV+e_8zL46ZyLB@16l6vu1;L-a~ci>s^k^U1cnovE#IZM@ijkOpihLb8Mnk zlNm9DeBlp;q(p;U>;NoMvmV!h%xPejsHYUN99#z8E1sq1NddPXX?@6m2A9d|!0&-^ z(aPP>CAKt#`vE$ycSo6RF40TkTfOCJHc`uMW)Gx#Fx(tOe*o!~sbFJ|b35}E&Veg~ zds3Q5jXZVKoPei6e6DMIemh0|%fZBdHBxLZ%>X zy;5bA8@nf4BAm<(BQh#q`~KiICSxw;g7KUb6}bv2)q$-EE0dH0oTlI1@+IWzsKrF81t&EJ%+Dt; zs?OGbsrYfMm~~sbLP?_Sv@SnFAfP zp$aZM@7<6li)a!l!hRpei?RftHCbFN*8$mD1MLJ*uvEA?RlD8)m7q;i{+%8fdYr(i z)?l})?y^MVvh5N@DP;M!V@oemR7@vMmB|b*+n}$8*qP&lh4V0V!iI5&b^lUfC5~&i zEi{r=`=n0y1Y-!hurVx(Xx40TZ&WdAG4W$F;hg*?`Fu$ima->owI};JxsYkW@`HwX zhg#cRd$7WWif`1gp2@a~slFoEW^O|gsjhi7ai~rt#KN6w#x@jUV{PmdghlQ(7w8fh;ye|e$gLW>Sqv=)f*Kw6VnM!^)hsoe0em05tb@l;YcDgEVKofYxr;ogi8s7>qv&PK4 z?SPB|qN8b%g#@NpfeEmuw`V3D{DU&dMKwP`_^BmAb`WM?#se5n+t1HNqqcgxe3;C` zM?6mHo}_63w4?2yrg_L^hnt)@>F|zfV-5H9*-B>q%ssOM|ACSNdUC>v7VQwb4MC@a znq*N#pqqDO@lhb;@$&1;QzkM$U?>*OK1-wB;W(#p=rI|h-`tsSRh&)vfksdn6&Hsy zB}gZaF#08AumA9w*A9v3r?$sl)#3X!d*zBpcmUvoL?fezKs-hjsvdKCoQi&;)H^dt z)g<%+4!$YN9=R$t2<>_ANOOAABNf{~aQdE-G&W}gV8S8nl|zMRwU!Hd8oMr$eGE^a zsX+lv!KKbD{3*u7aFG37RoxZc5nORyjncJ-=$1m`_DTyK;lZZ_h>f!Z0*8Xa-fl|oFmsk+CErWZ6TH+038=id+6f-adlTsQS#Mmr@?P-C z(?maQ5FfC74~8LJoiNcBNeyx!DVjV4Op%EZOi>|UV}Zs-y_)t#RO9=Q>!m(aYZ}}% zMBx05%e|cj&jwMrslMybcZe6Nh`>*}&xgmV+RvzEs7WTBak!8S2GUU1(5ZcC)}49W zVn!q9Np1+T8et07{&)@;pe8BvMd*I6;Ja9P`J8 zVE(f1*KG5LG08BwvWOO)|N4)5O-3H4M|O=RGEtE z>wAo`t0?41XB6w_nH+sigW^4Ckz?Fk-?!Oj!&?T;2hw0zT9YVu-@8)Kj-tlAd`XC_ zY+xt6fX4JS+I(h6o(vsiTZ;W^v)!qei1T7-l?N8`IeJF_wrn&$k{oeJsE$G9(m|*c zBa9)h>c^%@#qft+j?YyX%vg*%+4EuLQ%2pcC~_lQd%kL=_56H}gd!_KEZU+4XewO6 zVqGa;K^epZ+#&>@0b{Mi@#)V?+|ISvg5R~1yob=~!X~?RC>2WRjzlHtd9t~8Zmsl< zrEUsi;N=@+lU9(Z_J5wP4T2L-^-^J$F|*TEL(cp7*)E_-WeVN?DP*6u6<%^G^hHzpyE6I3X-25>_3lI)&+>pDd^9@Q?k@{oX;f?PxuY+BT5iFH$<2@C;2#Xu= zn~u)4z&Q-R>gbzV{#r-NUjyu{e-}LXUnKPxQ3P!b6m5MCEsaxgu`j1^U;L%qfx^*& zN#NfDT42L_-M&Zn2MWc<=aKp#)CYuz`boCaA3jcGWOQ1TlJvXq^q?>$H5rwJxUev# zuskdT7@11`k64h*(Qhz2z~iqg0*6?59-LHpUBlZt{#YSw+~mLR(u0V<${hS+S-)ST zIU`5@(?vQ*K{{-)9>MjdES}FXe7b78=Ek$O)gF$gau&6I2}L-FYYP>OD#S#~P==W7 zq|oo-Bn6*XapkI!hk{ZofZp|K235yQ9H$Vai1@P&9;|? zOgn0r%SG>e8sTG6T0CU-fgs0OUrQqjo=PVfOQ-wo;#a*Wg%$B3bfuAD*0QWy>y8s8 zoJg#Bj2h-oe!xUeG4`1qa2lOxhy`M};BcugcR*Ip5ym4vN;Cp+d#GuSKwdJZeTvmJ z)HkI0-ZPQNDR-H>1*X)o>gcgJtyXB3yk~&^aR73CT(ojQr0HaQ52e}Z~x$))VZ}jPVP#ufN*5{UW>!nfPZHP7G^d-(Y_gXtIM!71Uxl$x{*P@k;_TtICt{g z24Z3FNw9BbO6v}r@z+I}ty@AIsS=-=8Y#u;_V-||9xM%XwQCAv<7if#7kWG0fchp! zU~|NJN27a&PqZH|-2-}7!l#rWiqnRDrQ<7r3ZCrC_sK{m=(IuB$31DMay@`Vr72!i zSw6oLoU{nC-3uhN%aNk>_n3c2kj*LGyf)T5m6)RTZHSOH0_5a$fcA(t4%1-&Vc@XOdd4iN6R|z(Sf}y zwfz7@uIKYQ`{31$GioR)T*D}(BmXH8DW*?&bRbG(`M}e1+N!9w0LE>ei-P2s4Pixl zop{C*yn=9K6(nP=YOBlqpq?D$l%bGcv&S}Ep|@%-o4a9)z--GE>b^9H=|yYzh8lVo z9i6N%BI}(V5k-$t3Kn=8xFRp75b)pl3U#*mf6he8o_WE&bDY0Ld!+2+TWBDobrn^$PcuL%P4P1`&UgTal)UDwSYZ7@vcjBf6|X!7Ihjc zb)ts;m||}f>wK6_B3Nvj!T%M$di?{(CLvc;XrAEMcD{b=@-ea*(f-^Cux@$}*|<-} zvkajOWWiEJAl@Jru0!|6<9-5OSp!?r%_bZ>tOlt|6=o)--`yulW?=5H=UN>4fRoUh zXp5v_*h>4b3Wu$r$R|6uySpyadua0A5@hQtJCO-RI*HMFE0mo|lg@GM6i#Rb9InqY z214I7p7XdLE$+ld;FL6txzq~=QIzs~g;{bsW@QBvGRzC>lWhB|`WgLS(6*vOMH-2u z$f}2}`7l2A2!<_;%aDw2j#N?4oPlOVRaL5whMq@_<&y3%mAS}jR$@6OC-*^CX?bY5 zgmBReh&F<2=FE`_ygX)p8C!}va!tFyafD}1CSSY)R+X8FEyZ0YsHK*FvhUadWvt7h z*m!~74M8&!KZqR^xo1sK?tfu!u<}YGT$x9^JfW8A*9|b(N`q2zzS+)cXxunh-f*ln z)VmO+=vKuPXJv~wOndq)cJ$#?ax3z>f9lbK0xL!IWHZL}ow6=7Z%goyshYV@V*t+qiw+wt0AjHQ@59rQ%v>eSWV_UubxuB{YS9VZ-{vL$3l07SQT(e6ga2$Za}CYDFu#weNw~z&Df|l|M4%RhBh=*Z zwbzIF_(LVm2cT9@NxWMw92dZ}_7Eym4mMichg}wi!}iA=plAT<@EDrK?WXf7NSN4a zc7Wcvk$cqzFE_=i$%a%pe!q%Rmn%4WmT^r`ji7+JPR5Y>>GiM)$#@7q;TLcQTbr?# zMTQTZDuu~@_#&-N)2YD>D%dN4*ZL|$#71BTSwT?@{hNz`qql}1Hk#MCZCHWk;?Hg% z8N{||*Ucx{m|DPWC}25@9b#&v-v<$YW5?Ylu$wf97v)t8(YRte>900he1Zl?S9Vk> zB+`%HTSnz`!qYOli794HO(|mSRgMU8j6er3*caf_YeK*L+2nO^SekIYFwTmZEhdc6WjG*i@3kwd-+b*9Htpo*Zp%!Ig=^# z{7spC?4%l_nE063gQOC@qi&8%!$FelA0J^ozP-_wyE&(}{cGmp&9r|Q9W3pSIt@xB z006?jkIVci0mVV{AImgy6kdx^>k*!h)#>;h!)+?IYogYaSXKyxu8$#A76=T-;okBV z5#p5M`$7Xv`|Hn6`a;67b}eS+D5w?P7+e>f4%%w%E66pxERhcb3UChVpNRq0KADkM zEA7-|eJ!E8nb=DeC!O*sAKHV7mO+#9?*%-<7do=T&aCFn~#bvL4_F7+8G1Y4m$WojM{sQ_K{3~im8W%#Bi zfo0x1QR*SDRrchbIs%u)dlRp42rAs-vvyTAP7ArToCfwkU16bYW;~w-5wgX@(L7*89B}q>bjW1 z-`{yphGt)kj_;ObjfQCfv0+!F|FDKU3E@f)0b<{FAI6)Zw)bt#HPtQQ!`DU~RF)X= zMC9S%*`T#EkD(&vF*imaYf6!}$@jB9M$pGLUs^~CN87wjpcecD*K^Ch`(2|nCzsnH z5HBO^B=(daqh&>uQ@XQH{TUVROqpXz4{#qTr)4@cS?OIN4a8^PwJ-GhpX3)5;(U^| zj?v67KIxL)oFS_!=tz9r%g4{@2U(c?T2~k0ffoiHMA(HwS6!dxd7*T}v5Tz-v(llp zEVA91Bpr;wo{Lhty4rd03SN$gftImZavqG_k!F$+nT~u!>uLN16Ype-UT&5u^WQIP zC-uva!CHnA2ccl_j0?{@LRcg4Wrgbo#6pbkm&71*=gYMdm4694$S7<+jl^P9Zw}}< z{T}^E9Te`GcARdOZ&lUT@JEq&%ZXqMpT~ln_gaxKOLM@#O*R~>-qXdtWIVOlg$mhb@1T{$(C9b`JkbsCj z^AROa`HZIz$SsLZFo*_BrCXj1>w3<{`|lZoZ5) z=kx|Kn${2=qGW+H$qe}FB#yN;BgzjnVtd#!rP1B3x;Rfp_8jSc2~DH@8=^>-;#KDosA=Fafnw=?WghBxkKkc^I=DL44X&+-;G%vXwCHZ&8KJ4&j);34p@5ZUqJ5Tpdp)lw&tEIlAgPI)S zXh|Naf-cH1^8qJeXc-po-{4ES(%}*f^x?$qaj`Gy@}1mB0kVD7jfl(vdfx6<1EiHA zb4HY)E*vlo9ImBIykwc?bKl8xxijxBc*$$6nKEXU4{D*sgt+0^@RYO;X)-y^tm?TA z1J%Y0C|k~ma2_1cqo`(u`Vyj&SxLUnE=Ysm+b))E+oBf|gJ%f9n+g zrdoKv?l}ZyU?ylcw)zkD+I?fM`UH}}mzoAyFMZ;XL#W3(G?iEOTKdk7R)1?bENErZ zbJN#TG@qL=Op^i+XIj0UEh(G&6vI)I)ZP{@x}Yz6(HCMq701>6ehIO5z{J}TCn2vj z9TJ9x;hE&!ckYN?XVtafQRZf!JM(u!r;|>I5SyvdzMp4{ho&k}^?RNiS6Zc1wT-_n ziwC{3*Grf7PF0B3ZMFEORm$#XYGZ!1m(m$+i&{x*IIb;)aY`)L`uIto6R|r3;w1%9 zq>)n@H&1LOB=sSaNhC0N4i_aQLh6fkjQw)pW-Mm^;|#=;oYJ$Y>?%-mz7Fdf^*0s9 zLN_}=c}1@`t^KtMzv(mee;w8Nr6B*3yhKHRD}d!cvl#b+JC&27;f-9*t}WmcDk6hG z%_-TV5@T5e7q6t7zp}W7xmQ1;7vIcmm)N5N!k$)wx9G45YO2$b|V2!wclGm=}8+noR?}0O1zmk^| z)W68fZ8<6AH}Zl-`;ELLb5ky$UdfAY)Sey5LhTgyV=|Bl&8?FU#s|wt^ZFlk;$=L` zfqcd`WtKV!l(FB-rdtT$CJZ7)c~^opE{JglF1u?5F+%BTFYD!mM`*{JsC=&#x|eXs z6fHQ|zgh&>5hTG9o}p(O!HdTqY0)BMOybs(fo0f>5qPYZT~pjH?^uKvnfM2*pM&Fo z#uGsjQn;w8rr~zPs%mF4R#Irv1`BBQ&EyzZdOUXj241rne}I=XFaW@7eG-7arKW+V=^qu(e=c_Z zhpDX$erqR-WB3PqS^j$_Ok76%+afJ=%fVOk0FMvfRY;B?t^2Kos%$um+sDlOQlJ+$ zv{+(wge)r`AI5DX?+Gpj!@|O_+=MD8h%oMXgVk|o+L$Hb`@S&;!5BquotD~efj3B5 zI^(Q5D~-fm+~H*i5BA*f09-6Fe-QdQdp3eLbL>6#g$ad6SDMEK+J~wC=&A)?ke4J2 zbXS~I26qe;V>wxcUPPddpl+sMLIaP6do;@4VifNEBZ^zeplFr2F|?1Ek^U+vT))$< zj<(H{wcPsy(s{WYz39H@v@&$syX(rMes(#D zfY<)Q?3=b%UWh)%p$L|{Z?m|$7R~_Cnf-q_+b_?R77<_`_`Y3&4MFWsQst2TK0bAKh}tqj z?I0jtk;&c1@k^$b_)T?#uIS3@pH2~+fmb_vN8&1~EJm@wWY}ogo7`M4)b%u6z5zo5 za0ZL8culyFg^77l#mNJl^vK^_udJ(&!+Jh%TrZw&pId<{4qjoqKM9%R=IBhp?PdMQ zj_}X>p);Ol!SC)x+$HtC$Pg`WkF-_I384!z#O#c1(tIycB0z`R3NX=qc{(AL97R`PU&j;!gx_E#E8t{)W<;=CUqk+xc^;%3GmWOt7Q zg%CT)5)@?1nEh*$H>L9D;{zN>kNm|6WxC8M<+i{GP@uKxBoo?iC1sga15;cd7^4G? zOi@~L;iB(8&M%D!UY<{*eED+s_*-lrx;}=5{?#OJX8FTl!b-RQihQHIssP!m!T#P# z|H_a5^Hfk4ef&@1Z+WQ>iLTdEVV7cPhER0@CY!Fn4+fvLT^A2uA|@%yZB@by{GklarbtG0s+r&v;zS~U7xgCkcd=sd3C$CG z+2S|^IK9T|k9}>$c!cCGLdus_%}mzOoo8(fk7Er@+lh-3WRFC$XfRiXp|B1TkQ6D$ z`KyiC`>kZz1gJi^w_Q($Br(_QzFXFiQYp{PvCp6k-IR% zKIonmsvG7pBPn2P~-tWjqM%(dHlhcag z6r{qV3eh5!q&E`dlG7p+;$WriF#JK$(X-)=ED?i5MS^5yWe`PWVt_q)0(tqCtf`7& zLd1iihpDK<=5BgEczW*|<|}y_ye=CYB7;2E-{*CqZ;SooYsicGBZkyTe-!D@hnFL6Itg?;6E|U61YBxZJ#Gof(AlD~NzQ)k5P`KP&a0 ziDdUG#28<0ZT0aZQ_nh|`m}9mm?R!(t7MGo$SGb36olcN@b@inm(4^J4I;X|clwCD zgGh0SPA02Wy4lij_=t104u7RHPXBz~=oCP89pkw?W<%CD`J<%lbo|(vE$JPPp&p*p zPA0pqNkS2oOCAFXt*ad0oS+!ur5hb5ts5_@CHUrX%_n~ZW%N$CY6avWE#qNrcb612 zNN0g|XTfuGfh6tDR0$f8#C&0TNnWyf)B%=L^WXnIh0+wp^)D?$TU5_J*4N>I`9kxP49CJKMc z-D`8$;`T0LrPAIaz{BTLfE*o0|Gg|j{m(>ECLAZD4T$X>o|^d~(~QDnV_iwEXQQF) zD}bS>Sb1RBUc-hr8xvAoLIs)F#Pz<5`BqD;Wp5JpLow?(?u3qI{>jmJ$A)zNbCTuV zbdac|qmB*(bRVSbmdC4vNqHt*z(V*ItTAFKze*UdDO;b1)Fc8@mBIpkQh2%VnGop) z7$gNciB>F~M}JzbNiFhU&yKapQ zSw~)vMd_QGmU|I95MjLT{spj&plWr;hik(=+{j`^Vc%Bx%{afSlVQT{v!r1yuS6oY z=8F&~>@D8@GY97jLpJr1T`%tsRA}5_fJ0V`@p!7LeeCKAO-Q`T({dm&{GeBK$TLKB zvp2k0?^1$MlUXu^Tt1=cSX){4lVU6fE-B@SczHBGa!79UaN?SrQmOqCYVH38wI@$W zzd>zt>p|!%)W&g3O8(za%c4_+vvV1BEyu%{?|XYl4;>Ab`NeDe2WK77aZH5Q1mBb$ z&O16iRqb0J*rGr{Rc($woWx2Qct{|D6&ZKVMnncwM@u}~&!&+U^aB-VAXr1=Dxd}_ zLv-B#WZGt^e==>`8`E;HZyQv5eldv`bB{qEsseiAZA?z+6|fXAuy zP8X&IiIXLFbEObQf1LwsK+5{;p2O6vMj(p|2p75`kGTY2VJG>dHndJWK-8wLS1q-g z-Ece~PxhDS`W@okJe{2!zDuvEceJ!Z_)}X%sBzJ#dAZp{vDxI{Abdv%V2B48 z=yC}Ama?LV9!7{Il`g6rqB2%p3^GA7%Yo-sK^T9LUp=EnVh`vDm(x(V&m<^YI;e5z z=#)CaXa=Mb6lCQSvywC-LqbE-z?kY%N|A}l@!*WgG;Gq`gxoCh+%47J-QRiJ>KR#B z=s4O)*{Mj{I9k&g$w(L!S6S(}sM<&|_V8W0x^JT&&zqAUyOHY^qz!R+$C$p4Gq%=O zlTy*KO_S76G6|NkdU-+Z^9dMyr_1osq4^C1ny%on-iY?iDu39kH06&=F4L4V*SSDF`24mR$PI2rI5?R9}L6@T6tP;<&*8atjJ-;%W;! zTLy#Yp`Ga`ax~G-S>{ct!*a93&eW4w^ySV7ax>*sWi^e zi*g0ZF&e42?LTtXp{O;b(C9PZlMX|J3}D$%6k9;c-V1}1YdBN+mWRa??$OY()4Kcv z0xDi1;5~XL%|9UE2;C|Fg4bgdBL15o4y=Kd*xdM!h)+mN!aKsGJqrl5@tLXi2AI#lZ(s9fa7fjtl5PzQEQT$1^bj*+b5N6HjQ&{P73#i!br_!jykHyBGnNwg zG&+u2FJV=VI$&Vr-7?h}Gj)5y8? z>!sGLZ>E2x)3uTMak&iE?SnvxF{N1{Qt=mxy(H5xuY?2l_4I$`y-=|Np(bwadzSiSuf4Y{du2qufN%a~ZQ!S{| zO&mUs{g6c5UyE}^=~=dhbPLPnF1ZODUlZh$iWM(*zRU7L3w+@8{xw8KSh5650T)AH z2~%q_1DMRBe6)Dy?Z)4C0T#6Qt*>sl3`M78%nsnX0GT?MOuw`bN1P>tKAvbilY4gn z09@2Ji#b7$$K4lu879ch_UH`Iymc0?$WoS%L>rEAh4DBd98AG;n92pRZ~Cz5`C_U7%0q~Axj$J_t@+zb9cKA0pW z`o~mn@96yeuD}~{{65k>1YR#^f1c{kF}45tRh_?1`E%F%n+p9t(mkZ!0sQQi|NHE} zaL%7+f9tIOxv}uAv;Oyy?m-Oo_Y3}0y!dbW?EgCB&wX!ig#o{hbdOK3MFc->!@pba zFDL&$miyLP_j60ZTWj6#BijS%_5G~BoAF;a>VLk%^*4=nKd<<$TjA&W%eQWY-$%9w z)BC?p`tw=y&zGA1m)?cHPW^LXh&w<64+t3>|nWa@ZZilg*L;Z|nZ+M*f&dd=0^rfeJ3{(E?a{uJje^V3j z^J@QJC1;F+Knz6DG&UZ;bVEpZB|kOq6n}-l4@zsn&m|yX( E1Irw-!T - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/era5/README.md b/scripts/era5/README.md deleted file mode 100644 index 11c96cf..0000000 --- a/scripts/era5/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# ECMWF `ERA5` -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The global `ERA5` dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: -```console -/project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data # rpp-kshook allocation -/project/rrg-mclark/data/meteorological-data/era5 # rrg-mclark allocation -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data -├── ERA5_merged_195001.nc -├── ERA5_merged_%y%m.nc -├── . -├── . -├── . -└── ERA5_merged_202012.nc -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `ERA5` simulations are `longitude` and `latitude` representing the longitude and latitude points, respectively. -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -The NetCDF files of the dataset contain 7 variables needed to force hydrological models. You may see a list of variables by using the `ncdump -h` command on one of the files: -```console -foo@bar:~$ module load cdo/2.0.4 -foo@bar:~$ module load nco/5.0.6 -foo@bar:~$ ncdump -h /project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data/ERA5_merged_195001.nc -``` - -## Spatial Extent -The spatial extent of the `ERA5` is on latitutes from `+90` to `-90` and longitudes from `-180` to `179.75`. The resolution is 0.25 degrees. - -## Temporal Extent -The time-steps are hourly covering from January 1950 to December 2020. - -## Short Description on `ERA5` Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 2 meters, 2) surface pressure, 3) air temperature at 2 meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through the current `ERA5` dataset and their details are described in the table below: - -|Variable Name |ERA5 Variable |Original Shortname|Parameter ID|Unit |IPCC abbreviation|Comments | -|---------------------|-------------------|------------------|------------|-----|-----------------|--------------------| -|surface pressure |airpres |sp |134 |Pa |ps | | -|specific humidity @2m|spechum |q |133 |kg/kg|huss | | -|air temperature |airtemp |t |130 |k |tas | | -|wind speed @10m |windspd |u,v |131,132 |m/s |wspd |WIND=SQRT(u2+v2)| -|precipitation |pptrate |mtpr |235055 |mm/hr| |mean total precipitation rate| -|short wave radiation |SWRadAtm |msdwswrf |235035 |W m-2|rsds | | -|long wave radiation |LWRadAtm |msdwlwrd |235036 |W m-2|rlds | | - -For a complete catalog of the dataset, see [here](https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation). - -## Downloading Original ERA5 Data -The scripts to download the original ERA5 data are contained in the [`ERA5_downloads.zip`](./ERA5_downloads.zip) file located in the current directory of the repository. The scripts are written by [Quoqiang Tang](quoqiang.tang@usask.ca) and [Wouter Knoben](wouter.knoben@usask.ca) as part of the [Community Workflows to Advance Reproducibility in Hydrological Modelling (CWARHM)](https://github.com/CH-Earth/CWARHM). "ERA5 data preparation includes interactions between an atmospheric model and a land surface model. ... [Original] ERA5 data is available at 137 different pressure levels (i.e. some height above the surface), as well as at the surface. The lowest atmospheric level is L137, at geopotential and geometric altitude 10m and data here relies only on the atmospheric model. Any variables at a height lower than L137 (i.e., at the surface) are the result of interpolation between atmospheric model and land model. We want to use only the outcomes from the ECMWF atmospheric model [...]. Therefore, we obtain (1) air temperature, (2) wind speed and (3) specific humidity at the lowest pressure level (L137), (4) Precipitation, (5) downward shortwave radiation, (6) downward longwave radiation and (7) air pressure are unaffected by the land model coupling and can be downloaded at the surface level. This is beneficial because surface-level downloads are substantially faster than pressure-level downloads. [^1]" -[^1]: from: https://github.com/CH-Earth/CWARHM/tree/main/3a_forcing#forcing-needed-to-run-summa - diff --git a/scripts/era5/era5.sh b/scripts/era5/era5.sh deleted file mode 100755 index 8111fa6..0000000 --- a/scripts/era5/era5.sh +++ /dev/null @@ -1,507 +0,0 @@ -#!/bin/bash -# Global Water Futures (GWF) Meteorological Data Processing Workflow -# Copyright (C) 2022, Global Water Futures (GWF), University of Saskatchewan -# -# This file is part of GWF Meteorological Data Processing Workflow -# -# For more information see: https://gwf.usask.ca/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html -# 2. Dr. Gouqiang Tang provided the downloaded ERA5 dataset files - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # required - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - -# ========================== -# Necessary Global Variables -# ========================== -# the structure of file names is as follows: "ERA5_merged_YYYYMM.nc" -era5Format="%Y%m" # era5 file date format -reportFormat="%Y-%m-%d %H:%M:%S" # report format for manupulations -exportFormat="%Y-%m-%d_%H:%M:%S" # exported file date format -fileStruct="ERA5_merged" # source dataset files' prefix constant - - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -module load cdo/2.0.4 -module load nco/5.0.6 - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#format date string -format_date () { date --date="$1" +"$2"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -####################################### -# extracts file name, date, and time -# from CONUSI file name strings. -# -# Globals: -# fileName: file name of the .nc data -# fileNameDate: date (YYYYMM) -# fileNameYear: year (YYYY) -# fileNameMonth: month (MM) -# -# Arguments: -# 1: -> fName: the -# -# Outputs: -# produces the following global -# variables: -# a) fileName -# b) fileNameDate -# c) fileNameYear -# d) fileNameMonth -####################################### -extract_filename_info () { - - # define local variable for input argument - local fPath="$1" # format: "/path/to/file/ERA5_merged_YYYYMM.nc" - - # file name - fileName="$(basename $fPath | cut -d '.' -f 1)" # file name - - # file date - fileNameDate="$(echo "$fileName" | cut -d '_' -f 3)" # file date (YYYYMM) - - # year part of the date - fileNameYear="$(echo "$fileNameDate" | cut -c 1-4)" # file year (YYYY) -} - - -###################################### -# populating an array of dates based -# on the input format and time-step -# ranged between the start and end -# points -# -# Globals: -# dateRangeArr: array of dates -# -# Arguments: -# 1: start date -# 2: end date -# 3: format string parsable by `date` -# 4: time-step, e.g., "1hour" -# -# Outputs: -# produces the following variables: -# 5) dateRangeArr -####################################### -date_range () { - local start=$1 # start date - local end=$2 # end date - local fmt=$3 # format of the ouput dates - local tstep=$4 # the time-step value parsable by bash `date` - - local curr=$start # current time-step - - # make Unix EPOCH time - local currUnix="$(unix_epoch "$curr")" - local endUnix="$(unix_epoch "$end")" - - # a global array variable - dateRangeArr=() - - while [[ "$currUnix" -le "$endUnix" ]]; do - dateRangeArr+=("$(format_date "$curr" "$fmt")") - curr="$(date --date="${curr} ${tstep}")" - - # update $currUnix for the `while` loop - currUnix="$(unix_epoch "$curr")" - done -} - - -####################################### -# end of a chosen time-frame minus -# the value of the time-step, e.g., -# 1. end of the year date based on -# hourly time-steps -# 2. end of the month based on daily -# time-steps -# -# Globals: -# None -# -# Arguments: -# 1. dateStr -# 2. time-frame, i.e., year, mmonth, -# day, hour -# (parsable by GNU date) -# 3. time-step, i.e., year, month, -# day, hour -# (parsable by GNU date) -# -# Outputs: -# prints the end of the time-frame -# at the last time-step to the stdout -####################################### -time_frame_end () { - local dateStr=$1 # date string - local timeFrame=$2 # time-frame - local timeStep=$3 # time-step - local fmt=$4 # date format - - local dateStrTrim # date string variable - local endDateStr # end date string - - # calculte the last time-step included in the file - # based on the timeframe of the files; - # e.g., ERA5_199201.nc indicates a monthly of that - # has hourly data (ERA5 is hourly). - case "${timeFrame,,}" in - year) - dateStrTrim=$(format_date "$dateStr" "%Y-01-01 00:00:00") - ;; - month) - dateStrTrim=$(format_date "$dateStr" "%Y-%m-01 00:00:00") - ;; - day) - dateStrTrim=$(format_date "$dateStr" "%Y-%m-%d 00:00:00") - ;; - hour) - dateStrTrim=$(format_date "$dateStr" "%Y-%m-%d %H:00:00") - ;; - esac - - local endDateStr="$(date --date="${dateStrTrim} 1${timeFrame} -1${timeStep}" +"$fmt")" - echo $endDateStr -} - - -####################################### -# splitting netCDF files based on the -# tsValue -# -# Globals: -# None -# -# Arguments: -# 1: start date -# 2: end date -# 3: time variable name -# 4: source file path -# 5: destination path -# 6: prefix file string -# 7: date format -# 8: time frame -# 9: time steps -# -# Outputs: -# it splits the netcdf files based -# on the time-steps (argument #9) -# -####################################### -split_ts () { - # assign local variables - local start=$1 # start date - local end=$2 # end date - local timeVar=$3 # time variable - local sourceFile=$4 # source file - local destDir=$5 # destination directory - local filePrefix=$6 # file prefix - local dateFmt=$7 # date format - local timeFrame=$8 # time frame: - # month, day, etc. - local timeStep=$9 # time step length - - # local variables used in the while loop - local tBegin="$(format_date "$start" "$dateFmt")" - local tEnd - - while [[ "$(unix_epoch "$tBegin")" -le "$(unix_epoch "$end")" ]]; do - tEnd=$(time_frame_end "$tBegin" "$timeFrame" "$timeStep" "$dateFmt") - - if [[ $(unix_epoch "$tEnd") -gt $(unix_epoch "$end") ]]; then - tEnd="$end" - fi - - exportDate="$(format_date "$tBegin" "$exportFormat")" - ncks -d "$timeVar","$tBegin","$tEnd" \ - -d latitude,"$(lims_to_float $latLims)" \ - -d longitude,"$(lims_to_float $lonLims)" \ - -v "$variables" \ - "$sourceFile" "${destDir}/${filePrefix}-${exportDate}.nc" - - tBegin=$(date --date="${tEnd} 1${timeStep}" "+${dateFmt}") - done -} - - -####################################### -# defining start and end point for the -# netcdf file of interest (era5). -# -# Globals: -# None -# -# Arguments: -# 1: file date -# -# Outputs: -# startPoint: start point of the time -# frame -# endPoint: end point of the time -# frame -####################################### -define_time_points () { - - local fDate=$1 - - local startPoint - local endPoint - local endOfCurrentMonthUnix - local endPoinUnix - - # check dates - if [[ "$fDate" -eq "$(format_date "$startDate" "$era5Format")" ]]; then - endOfCurrentMonthUnix="$(time_frame_end "${fDate}01" "month" "hour" "%s")" # end of month in Unix EPOCH time - if [[ "$endOfCurrentMonthUnix" -le "$(unix_epoch "$endDate")" ]]; then - endPointUnix="$endOfCurrentMonthUnix" - else - endPointUnix="$(unix_epoch "$endDate")" - fi - startPoint="$(format_date "$startDate" "$reportFormat")" - endPoint="$(format_date "@$endPointUnix" "$reportFormat")" - - elif [[ "$fDate" -eq "$(format_date "$endDate" "$era5Format")" ]]; then - startPoint="$(format_date "${fDate}01" "$reportFormat")" - endPoint="$(format_date "${endDate}" "$reportFormat")" - - else - startPoint="$(format_date "${fDate}01" "$reportFormat")" - endPoint="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" - - fi - - timePoints=("$startPoint" "$endPoint") -} - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing ECMWF ERA5..." - -# extract the dates using `date_range` function -> dateRangeArr -date_range "$startDate" "$endDate" "$era5Format" "1hour" # tstep is hard-coded for ERA5 -# extract unique values from $dateRangeArr -uniqueDatesArr=($(echo "${dateRangeArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')); - -# creating a temporary directory for temporary files -echo "$(basename $0): creating cache files in $HOME/.temp_gwfdata" -mkdir -p "$cache" # making the directory - -# copy necessary files to the $cache -for ym in "${uniqueDatesArr[@]}"; do - cp "${datasetDir}/${fileStruct}_${ym}.nc" "${cache}/${fileStruct}_${ym}.nc" -done - -# make the output directory -mkdir -p "$outputDir" - -# define empty global array for start and end timePoints -timePoints=() # 0: startPoint, 1: endPoint - -# data files for the current year with extracted $variables -files=($cache/*) - -# if yearly timeScale then make an empty yearsArr -if [[ "${timeScale,,}" == "y" ]]; then - yearsArr=() - datesArr=() -fi - -for f in "${files[@]}"; do - extract_filename_info "$f" # extract file name info - define_time_points "$fileNameDate" # define start & end time points for subsetting - - case "${timeScale,,}" in - h) - split_ts "${timePoints[0]}" "${timePoints[1]}" "time" "$f" "$outputDir" "$prefix" "$reportFormat" "hour" "hour" - ;; - - d) - split_ts "${timePoints[0]}" "${timePoints[1]}" "time" "$f" "$outputDir" "$prefix" "$reportFormat" "day" "hour" - ;; - - m) - exportDate="$(format_date "${timePoints[0]}" "$exportFormat")" - - monthStart="$(format_date "${fDate}01" "$reportFormat")" - monthEnd="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" - if [[ "$monthStart" == "${timePoints[0]}" && "$monthEnd" == "${timePoints[1]}" ]]; then - ncks -d latitude,$(lims_to_float "$latLims") \ - -d longitude,$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$f" "${outputDir}/${prefix}-${exportDate}.nc" - - else - ncks -d time,"${timePoints[0]}","${timePoints[1]}" \ - -d latitude,$(lims_to_float $latLims) \ - -d longitude,$(lims_to_float $lonLims) \ - -v "$variables" \ - "$f" "${outputDir}/${prefix}-${exportDate}.nc" - - fi - ;; - - y) - yearsArr+=("$fileNameYear") - exportDate="$(format_date "${timePoints[0]}" "$exportFormat")" - datesArr+=("$exportDate") - - monthStart="$(format_date "${fDate}01" "$reportFormat")" - monthEnd="$(time_frame_end "${fDate}01" "month" "hour" "$reportFormat")" - if [[ "$monthStart" == "${timePoints[0]}" && "$monthEnd" == "${timePoints[1]}" ]]; then - ncks -d latitude,$(lims_to_float "$latLims") \ - -d longitude,$(lims_to_float "$lonLims") \ - -v "$variables" \ - --mk_rec_dmn time \ - -O \ - "$f" "$f" - - else - ncks -d time,"${timePoints[0]}","${timePoints[1]}" \ - -d latitude,$(lims_to_float $latLims) \ - -d longitude,$(lims_to_float $lonLims) \ - -v "$variables" \ - -O \ - --mk_rec_dmn time \ - "$f" "$f" - - fi - ;; - - esac -done - -if [[ "${timeScale,,}" == "y" ]]; then - # make an array of unique years - uniqueYearsArr=($(echo "${yearsArr[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - - for yr in "${uniqueYearsArr[@]}"; do - # get the first exportDate of each year - idx=0 - for str in "${datesArr[@]}"; do - if [[ "$yr" == $(echo "$str" | cut -d '-' -f 1) ]]; then - break - else - idx=$(($idx + 1)) - fi - done - - exportDate="${datesArr[$idx]}" - ncrcat ${cache}/*${yr}* "${outputDir}/${prefix}-${exportDate}.nc" - done -fi - -rm -r $cache # removing the temporary directory -echo "$(basename $0): temporary files from $cache are removed." -echo "$(basename $0): results are produced under $outputDir." - diff --git a/scripts/era5/era5_simplified.sh b/scripts/era5/era5_simplified.sh deleted file mode 100755 index b0bf729..0000000 --- a/scripts/era5/era5_simplified.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html -# 2. Dr. Quoqiang Tang downloaded, pre-processed, and produced relevant associated scripts. - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -# the structure of file names is as follows: "ERA5_merged_YYYYMM.nc" -era5Format="%Y%m" # era5 file date format -fileStruct="ERA5_merged" # source dataset files' prefix constant - -latVar="latitude" -lonVar="longitude" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing ECMWF ERA5..." - -# make the output directory -echo "$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -toDate="$startDate" -toDateUnix="$(unix_epoch "$toDate")" -endDateUnix="$(unix_epoch "$endDate")" - -# creating yearly directory -mkdir -p "$outputDir" # making the output directory - -# extract variables from the forcing data files -while [[ "$toDateUnix" -le "$endDateUnix" ]]; do - # date manipulations - toDateFormatted=$(date --date "$toDate" +"$era5Format") # current timestamp formatted to conform to CONUSI naming convention - - # creating file name - file="${fileStruct}_${toDateFormatted}.nc" # current file name - - # extracting variables from the files and spatial subsetting - ncks -O -v "$variables" \ - -d latitude,"$(lims_to_float "$latLims")" \ - -d longitude,"$(lims_to_float "$lonLims")" \ - "${datasetDir}/${file}" "${outputDir}/${prefix}${file}" - - [ $( jobs | wc -l ) -ge $( nproc ) ] && wait # forking shell processes - - # increment time-step by one unit - toDate="$(date --date "$toDate 1month")" # current time-step - toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time -done - -# wait to make sure the while loop is finished -wait - -# go to the next year if necessary -if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then - toDate=$(date --date "$toDate 1month") -fi - - -echo "$(basename $0): results are produced under $outputDir." - diff --git a/scripts/espo-g6-r2/README.md b/scripts/espo-g6-r2/README.md deleted file mode 100644 index 6adf7e0..0000000 --- a/scripts/espo-g6-r2/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# ECCC `RDRS` v2.1 -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The `RDRS` v2.1 dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: -```console -/project/rpp-kshook/Model_Output/RDRSv2.1 # rpp-kshook allocation -/project/rrg-mclark/data/meteorological-data/rdrsv2.1 # rrg-mclark allocation -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/RDRSv2.1 -├── 1980 -│ ├── 1980010112.nc -│ ├── 1980010212.nc -│ ├── 1980010312.nc -│ ├── . -│ ├── . -│ ├── . -│ └── 1980123112.nc -. -. -. -├── %Y -│ ├── %Y010112.nc -│ ├── . -│ ├── . -│ ├── %Y%m%d12.nc -│ ├── . -│ ├── . -│ └── %Y123112.nc -. -. -. -└── 2018 - ├── 2018010112.nc - ├── . - ├── . - ├── . - └── 2018123112.nc -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `ERA5` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -The NetCDF files of the dataset contain 28 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: -```console -foo@bar:~$ module load cdo/2.0.4 -foo@bar:~$ module load nco/5.0.6 -foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/RDRSv2.1/1980/1980010112.nc -``` - -## Spatial Extent -The spatial extent of the `RDRS` v2.1 is on latitutes from `+5.75` to `+64.75` and longitudes from `-179.9925` to `179.9728` covering North America. The resolution is 0.09 degrees (~10km). - -## Temporal Extent -The time-steps are hourly covering from `January 1980` to `December 2018`. - -## Short Description on `RDRS` v2.1 Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 1.5 (or 2) meters, 2) surface pressure, 3) air temperature at 1.5 (or 2) meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `RDRS` v2.1 dataset and their details are described in the table below: -|Variable Name |RDRSv2.1 Variable |Unit |IPCC abbreviation|Comments | -|----------------------|-------------------|-----|-----------------|----------------------| -|surface pressure |RDRS_v2.1_P_P0_SFC |mb |ps | | -|specific humidity@1.5m|RDRS_v2.1_P_HU_1.5m|1 |huss | | -|air tempreature @1.5m |RDRS_v2.1_P_TT_1.5m|C |tas | | -|wind speed @10m |RDRS_v2.1_P_UVC_10m|kts |wspd |WIND=SQRT(U102+V102)| -|precipitation |RDRS_v2.1_A_PR0_SFC|m/hr | |CaPA outputs | -|short wave radiation |RDRS_v2.1_P_FB_SFC |W m-2|rsds |Downward solar flux | -|long wave radiation |RDRS_v2.1_P_FI_SFC |W m-2|rlds |Downward infrared flux| diff --git a/scripts/espo-g6-r2/espo-g6-r2.sh b/scripts/espo-g6-r2/espo-g6-r2.sh deleted file mode 100755 index b327c77..0000000 --- a/scripts/espo-g6-r2/espo-g6-r2.sh +++ /dev/null @@ -1,420 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2024, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR] [-MmS STR[,...]]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m:S:M: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble:,scenario:,model: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variable) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - -S | --scenario) scenario="$2" ; shift 2 ;; # required - -M | --model) model="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix is not set -if [[ -z $prefix ]]; then - prefix="data_" -fi - -# useful log date format function -logDate () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# ================ -# Necessary checks -# ================ - -# check if the dates are within datasets date range -# define $startYear and $endYear -startYear=$(date --date "$startDate" +"%Y") -endYear=$(date --date "$endDate" +"%Y") - -# if $startYear is before 1950 raise a "WARNING" and set startDate -if [[ $startYear -lt 1950 ]]; then - echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" - echo "$(logDate)$(basename $0): WARNING! \`start-date\` is set to 1950-01-01 00:00:00" - startDate="1950-01-01" - startYear="1950" -fi - -# if $endYear is beyond 2100 raise a "WARNING" and set endDate -if [[ $endYear -gt 2100 ]]; then - echo "$(logDate)$(basename $0): WARNING! The date range of the dataset is between 1950-01-01 and 2100-12-31" - echo "$(logDate)$(basename $0): WARNING! \`end-date\` is set to 2100-12-31 00:00:00" - endDate="2100-12-31" - endYear="2100" -fi - -# check if $model, $ensemble, and $scenario is given -if [[ -z $model ]] || \ - [[ -z $ensemble ]] || \ - [[ -z $scenario ]]; then - echo "$(logDate)$(basename $0): ERROR! \`--model\`, \`--ensemble\`, and \`--scenario\` values are required" - exit 1; -fi - - -# ===================== -# Necessary assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - -# paths -datatoolPath="$(dirname $0)/../../" # datatool's path -# daymet index scripts works on RDRSv2.1 grids as well -# and ESPO-G6-R2 has similar grid system as RDRSv2.1 -coordIdxScript="$datatoolPath/assets/ncl_scripts/coord_daymet_idx.ncl" -coordClosestIdxScript="$datatoolPath/assets/ncl_scripts/coord_closest_daymet_idx.ncl" - - -# ========================== -# Necessary global variables -# ========================== -latDim="rlat" -lonDim="rlon" -timeDim="time" - - -# =================== -# Necessary functions -# =================== -# Modules below available on Digital Research Alliance of Canada's Graham HPC -## core modules -function load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -function unload_core_modules () { - # WARNING: DO NOT USE IF YOU ARE NOT SURE HOW TO URE IT - module -q unload cdo/2.0.4 - module -q unload nco/5.0.6 -} -## ncl modules -function load_ncl_module () { - module -q load ncl/6.6.2 -} -function unload_ncl_module () { - module -q unload ncl/6.6.2 -} - -# loading core modules for the script -load_core_modules - - -# ================= -# Useful one-liners -# ================= -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -# =============== -# Data processing -# =============== -# display info -echo "$(logDate)$(basename $0): processing Ouranos ESPO-G6-R2..." - -# create $modelArr array from input comma-delimited values -IFS=',' read -ra modelArr <<< $(echo $model) -# create $scenarioArr array from input comma-delimited values -IFS=',' read -ra scenarioArr <<< $(echo $scenario) -# create $ensembleArr array from input comma-delimited values -IFS=',' read -ra ensembleArr <<< $(echo $ensemble) -# create $variableArr array from input comma-delimited values -IFS=',' read -ra variableArr <<< $(echo $variables) - - -# ====================== -# Extract domain extents -# ====================== - -# parse the upper and lower bounds of a given spatial limit -minLat=$(echo $latLims | cut -d ',' -f 1) -maxLat=$(echo $latLims | cut -d ',' -f 2) -minLon=$(echo $lonLims | cut -d ',' -f 1) -maxLon=$(echo $lonLims | cut -d ',' -f 2) - -# unload and load necessary modules -unload_core_modules -load_ncl_module -# choose a sample file as all files share the same grid -domainFile=$(find ${datasetDir} -type f -name "*.nc" | head -n 1) -# parse the upper and lower bounds of a given spatial limit -minLat=$(echo $latLims | cut -d ',' -f 1) -maxLat=$(echo $latLims | cut -d ',' -f 2) -minLon=$(echo $lonLims | cut -d ',' -f 1) -maxLon=$(echo $lonLims | cut -d ',' -f 2) - -# extract the associated indices corresponding to $latLims and $lonLims -coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordIdxScript")" - -# if spatial index out-of-bound, i.e., 'ERROR' is return -if [[ "${coordIdx}" == "ERROR" ]]; then - # extract the closest index values - coordIdx="$(ncl -nQ 'coord_file='\"$domainFile\" 'minlat='"$minLat" 'maxlat='"$maxLat" 'minlon='"$minLon" 'maxlon='"$maxLon" "$coordClosestIdxScript")" -fi - -# parse the output index for latitude and longitude -lonLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 1)" -latLimsIdx+="$(echo $coordIdx | cut -d ' ' -f 2)" - -# reload necessary modules -unload_ncl_module -load_core_modules - -# ============================================ -# Build date arrays for time-series extraction -# ============================================ -# file date intervals in years - dataset's default -interval=4 - -startFormat="%Y0101" -endFormat="%Y1231" # will be redefined later depending on the $modelName - -actualFormat='%Y%m%d' - -# define needed variables -let "difference = $endYear - $startYear" -let "steps = $difference / $interval" - -# build $startDateFileArr, $endDateFileArr -startDateFileArr=() -endDateFileArr=() - -# range of jumps -range=$(seq 0 $steps) - -# filling the arrays -for iter in $range; do - # jumps every $interval years - let "jumps = $iter * $interval" - - # current date after necessary jumps - let "toDate = $jumps + $startYear" - - # extract start and end values - startValue="$(date --date "${toDate}0101" +"${startFormat}")" - endValue="$(date --date "${toDate}0101 +${interval}years -1days" +"${endFormat}")" - - # check if endValue is beyond 2100 - endValueYear="$(date --date "${endValue}" +"%Y")" - # double-check end-date - if [[ "$endValueYear" -gt 2100 ]]; then - endValue="21001231" # irregular last date for dataset files - fi - - # fill up relevant arrays - startDateFileArr+=("${startValue}") - endDateFileArr+=("${endValue}") - -done - -# build actualStartArr array for temporal subsetting -actualStartDateArr=("${startDateFileArr[@]}") -actualStartDateArr[0]="$(date --date "${startDate}" +"${actualFormat}")" - -# and similarly, the actualEndArr array -actualEndDateArr=("${endDateFileArr[@]}") -lastIndex=$(( "${#actualEndDateArr[@]}" - 1 )) -actualEndDateArr[${lastIndex}]="$(date --date "${endDate}" +"${actualFormat}")" - - -# ===================== -# Extract dataset files -# ===================== -# Typical directory structure of the dataset is: -# ${datasetDir}/${model}/%submodel/${scenario}/${ensemble}/day/${var}/ -# and each ${var} directory contains files in the following nomenclature: -# ${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${model}_%submodel_${scenario}_${ensemble}_%yyyymmdd-%yyyymmdd.nc -# with the former date value indicating the starting year of data inside the -# file, and the latter demonstrating the ending date of data -# -# NOTE: %submodel must be determined in the upstream caller -# - -# create dataset directories in $cache and $outputDir -echo "$(logDate)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" -echo "$(logDate)$(basename $0): creating cache directory under $cache" -mkdir -p "$cache" - -# iterate over models/submodels -for model in "${modelArr[@]}"; do - # extract model and submodel names - modelName=$(echo $model | cut -d '/' -f 1) - submodelName=$(echo $model | cut -d '/' -f 2) - - # iterate over scenarios, e.g., ssp245, ssp370, ssp585 - for scenario in "${scenarioArr[@]}"; do - - # iterate over ensemble members, e.g., r1p1, r1p2, etc. - for ensemble in "${ensembleArr[@]}"; do - - pathTemplate="${modelName}/${submodelName}/${scenario}/${ensemble}/day/" - if [[ -e "${datasetDir}/${pathTemplate}" ]]; then - echo "$(logDate)$(basename $0): processing ${model}.${scenario}.${ensemble} files" - mkdir -p "${cache}/${pathTemplate}" - mkdir -p "${outputDir}/${pathTemplate}" - else - echo "$(logDate)$(basename $0): ERROR! ${model}.${scenario}.${ensemble} does not exist." - break 1; - fi - - # iterate over date range of interest using index - for idx in "${!startDateFileArr[@]}"; do - - # dates for files - fileStartDate="${startDateFileArr[$idx]}" - fileEndDate="${endDateFileArr[$idx]}" - # dates for subsetting - actualStartDate="${actualStartDateArr[$idx]}" - actualEndDate="${actualEndDateArr[$idx]}" - # dates for ncks slabs - actualStartDateFormatted="$(date --date $actualStartDate +'%Y-%m-%d')" - actualEndDateFormatted="$(date --date $actualEndDate +'%Y-%m-%d')" - - # destination NetCDF file - dst="day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_${modelName}_${submodelName}_${scenario}_${ensemble}_${actualStartDate}-${actualEndDate}.nc" - - # address inconsistencies with NetCDF file end-date values in the - # dataset - if [[ "$modelName" == "NIMS-KMA" ]] || \ - [[ "$modelName" == "MOHC" ]]; then - fileEndDate=$(date --date "${fileEndDate}" +"%Y1230") - fi - - # iterate over dataset variables of interest - for var in "${variableArr[@]}"; do - - # define file for further operation - # address inconsistencies with NetCDF file name in the dataset - if [[ "$modelName" == "DKRZ" ]]; then - src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" - else - src="${var}_day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_${modelName}_${submodelName}_${scenario}_${ensemble}_${fileStartDate}-${fileEndDate}.nc" - fi - - # spatial subsetting - until ncks -A -v ${var} \ - -d "$latDim","${latLimsIdx}" \ - -d "$lonDim","${lonLimsIdx}" \ - -d "$timeDim","${actualStartDateFormatted}","${actualEndDateFormatted}" \ - ${datasetDir}/${pathTemplate}/${var}/${src} \ - ${cache}/${pathTemplate}/${dst}; do - echo "$(logDate)$(basename $0): Process killed: restarting process" >$2 - sleep 10; - done # until ncks - done # for $variableArr - - # change lon values so the extents are from ~-180 to 0 - # this is solely for easymore compatibility - until ncap2 -O -s 'where(lon>0) lon=lon-360' \ - "${cache}/${pathTemplate}/${dst}" \ - "${outputDir}/${pathTemplate}/${prefix}${dst}"; do - echo "$(logDate)$(basename $0): Process killed: restarting process" >$2 - sleep 10; - done # until ncap2 - - done # for $startDateArr - done # for $ensembleArr - done # for $scenarioArr -done # for $modelArr - -# wait for everything to finish - just in case -sleep 10 - -mkdir "$HOME/empty_dir" -echo "$(logDate)$(basename $0): deleting temporary files from $cache" -rsync -aP --delete "$HOME/empty_dir/" "$cache" -rm -r "$cache" -echo "$(logDate)$(basename $0): temporary files from $cache are removed" -echo "$(logDate)$(basename $0): results are produced under $outputDir" - diff --git a/scripts/gfdl_cm4/gfdl_cm4.sh b/scripts/gfdl_cm4/gfdl_cm4.sh deleted file mode 100755 index c058faa..0000000 --- a/scripts/gfdl_cm4/gfdl_cm4.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n gdfl_cm4 -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_GFDL-CM4_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing GDFL-CM4 dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/gfdl_esm4/gfdl_esm4.sh b/scripts/gfdl_esm4/gfdl_esm4.sh deleted file mode 100755 index 53c9c58..0000000 --- a/scripts/gfdl_esm4/gfdl_esm4.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n gdfl_esm4 -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_GFDL-ESM4_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing GDFL-ESM4 dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/hybrid_obs/README.md b/scripts/hybrid_obs/README.md deleted file mode 100644 index f16e2ad..0000000 --- a/scripts/hybrid_obs/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Alberta Government `Hybrid Observation` Dataset -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The downscaled `Hybrid observation` dataset is located under the following directory accessible from Digital Research Alliance of Canada (DRA) Graham Cluster: -``` -/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/hybrid_obs -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/CompHydCore/climateForcingData/ERA5/ERA5_for_SUMMA/2_merged_data -├── Hybrid_Daily_BCABSK_US_pr_1950.nc -├── . -├── . -├── . -├── Hybrid_Daily_BCABSK_US_pr_2019.nc -├── Hybrid_Daily_BCABSK_US_tmax_1950.nc -├── . -├── . -├── . -├── Hybrid_Daily_BCABSK_US_%var_%yr.nc -├── . -├── . -├── . -├── Hybrid_Daily_BCABSK_US_tmax_2019.nc -├── Hybrid_Daily_BCABSK_US_tmin_1950.nc -├── . -├── . -├── . -└── Hybrid_Daily_BCABSK_US_tmin_2019.nc -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `Hybrid Observation` datasets are `lon` and `lat` representing the longitude and latitude points, respectively. -### Time-stamps -The time-stamps are included in the original files. The data is avilable on a daily time-scale. - -## Dataset Variables -The NetCDF files of the dataset contain 1 variable. You may see a list of variables by browsing the files. - -## Spatial Extent -The spatial extent of the `Hybrid Observation` is on latitutes from `+45.95` to `60.25` and longitudes from `-128.05` to `-106.05`. The resolution is 0.1 degrees. - -## Temporal Extent -The time-steps are daily covering from January 1950 to December 2019. - -## Short Description on `ERA5` Variables -|Variable Name |ERA5 Variable |Unit |Comments | -|---------------------|-------------------|-------|--------------------| -|precipitation |pr |mm/day | | -|maximum temperature |tmax |degC | | -|minimum temperature |tmin |degC | | - -For a complete description of the dataset, see [here](https://doi.org/10.5194/hess-23-5151-2019). - -## Downloading Original `Hybrid Observation` Data -The data can be requested to download from 'hyung.eum AT gov.ab.ca'. diff --git a/scripts/hybrid_obs/hybrid_obs.sh b/scripts/hybrid_obs/hybrid_obs.sh deleted file mode 100755 index 1b33ee5..0000000 --- a/scripts/hybrid_obs/hybrid_obs.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n hybrid_obs -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if ensemble is given -if [[ -n $ensemble ]]; then - echo "$(basename $0): ERROR! This dataset dos not have any ensemble" \ - "members." -fi - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Hybrid_Daily_BCABSK_US" # source dataset files' suffix constant - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing hybrid_obs dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - : -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variableArr of the comma-separated values of variables -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" - -# loop over variables -for var in "${variablesArr[@]}"; do - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$var" \ - "$datasetDir/${filePrefix}_${var}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${var}_${yr}.nc" - done - - # wait to assure the background processes in `for` loop is finished - wait - -done - -# wait to assure the background processes in `for` loop is finished -wait - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/ipsl_cm6a_lr/ipsl_cm6a_lr.sh b/scripts/ipsl_cm6a_lr/ipsl_cm6a_lr.sh deleted file mode 100755 index 23e82c7..0000000 --- a/scripts/ipsl_cm6a_lr/ipsl_cm6a_lr.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n ipsl_cm6a_lr -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_IPSL-CM6A-LR_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing IPSL-CM6A-LR dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/mri_esm2_0/mri_esm2_0.sh b/scripts/mri_esm2_0/mri_esm2_0.sh deleted file mode 100755 index 968b288..0000000 --- a/scripts/mri_esm2_0/mri_esm2_0.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# Copyright (C) 2023, University of Calgary -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n mri_esm2_0 -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # redundant - added for compatibility - -m | --ensemble) ensemble="$2" ; shift 2 ;; # required - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -format="%Y-%m-%dT%H:%M:%S" # date format -filePrefix="Downscaled_MRI-ESM2-0_MBCDS" # source dataset files' suffix constant -fileSuffix="pr_tmn_tmx" # suffix before the date format - -latVar="lat" -lonVar="lon" -timeVar="time" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - -# log date format -log_date () { echo "($(date +"%Y-%m-%d %H:%M:%S")) "; } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(log_date)$(basename $0): processing MRI-ESM2-0 dataset..." - -# make the output directory -echo "$(log_date)$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# make array of ensemble members -if [[ -n "$ensemble" ]]; then - IFS=',' read -ra ensembleArr <<< "$(echo "$ensemble")" # comma separated input -else - # if nothing has been entred, throw an error and exit - echo "$(log_date)$(basename $0): ERROR! --ensemble argument does not" \ - "have valid value(s)" - # exit the script - exit 1; -fi - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year -endYear=$(date --date="$endDate" +"%Y") # end year -yearsRange=$(seq $startYear $endYear) - -# make variable string for output file creation -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" # array for vars -varStr=$(join_by "_" "${variablesArr[@]}") - -for member in "${ensembleArr[@]}"; do - # creating yearly directory - echo "$(log_date)$(basename $0): processing member $member" - - # loop over years - for yr in $yearsRange; do - # extract variables and spatially and temporally subset - ncks -O \ - -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -v "$variables" \ - "$datasetDir/${filePrefix}_${member}_${fileSuffix}_${yr}.nc" \ - "$outputDir/${prefix}${filePrefix}_${member}_${varStr}_${yr}.nc" - done - - # wait to assure the `for` loop is finished - wait - -done - -# printing final prompt -echo "$(log_date)$(basename $0): results are produced under $outputDir." - diff --git a/scripts/rdrs/LICENSE b/scripts/rdrs/LICENSE deleted file mode 100644 index 53d1f3d..0000000 --- a/scripts/rdrs/LICENSE +++ /dev/null @@ -1,675 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. - diff --git a/scripts/rdrs/README.md b/scripts/rdrs/README.md deleted file mode 100644 index 6adf7e0..0000000 --- a/scripts/rdrs/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# ECCC `RDRS` v2.1 -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The `RDRS` v2.1 dataset is located under the following directory accessible from Compute Canada (CC) Graham Cluster: -```console -/project/rpp-kshook/Model_Output/RDRSv2.1 # rpp-kshook allocation -/project/rrg-mclark/data/meteorological-data/rdrsv2.1 # rrg-mclark allocation -``` -and the structure of the dataset hourly files is as following: -```console -/project/rpp-kshook/Model_Output/RDRSv2.1 -├── 1980 -│ ├── 1980010112.nc -│ ├── 1980010212.nc -│ ├── 1980010312.nc -│ ├── . -│ ├── . -│ ├── . -│ └── 1980123112.nc -. -. -. -├── %Y -│ ├── %Y010112.nc -│ ├── . -│ ├── . -│ ├── %Y%m%d12.nc -│ ├── . -│ ├── . -│ └── %Y123112.nc -. -. -. -└── 2018 - ├── 2018010112.nc - ├── . - ├── . - ├── . - └── 2018123112.nc -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `ERA5` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -The NetCDF files of the dataset contain 28 variables. You may see a list of variables by using the `ncdump -h` command on one of the files: -```console -foo@bar:~$ module load cdo/2.0.4 -foo@bar:~$ module load nco/5.0.6 -foo@bar:~$ ncdump -h /project/rpp-kshook/Model_Output/RDRSv2.1/1980/1980010112.nc -``` - -## Spatial Extent -The spatial extent of the `RDRS` v2.1 is on latitutes from `+5.75` to `+64.75` and longitudes from `-179.9925` to `179.9728` covering North America. The resolution is 0.09 degrees (~10km). - -## Temporal Extent -The time-steps are hourly covering from `January 1980` to `December 2018`. - -## Short Description on `RDRS` v2.1 Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 1.5 (or 2) meters, 2) surface pressure, 3) air temperature at 1.5 (or 2) meters, 4) wind speed at 10 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `RDRS` v2.1 dataset and their details are described in the table below: -|Variable Name |RDRSv2.1 Variable |Unit |IPCC abbreviation|Comments | -|----------------------|-------------------|-----|-----------------|----------------------| -|surface pressure |RDRS_v2.1_P_P0_SFC |mb |ps | | -|specific humidity@1.5m|RDRS_v2.1_P_HU_1.5m|1 |huss | | -|air tempreature @1.5m |RDRS_v2.1_P_TT_1.5m|C |tas | | -|wind speed @10m |RDRS_v2.1_P_UVC_10m|kts |wspd |WIND=SQRT(U102+V102)| -|precipitation |RDRS_v2.1_A_PR0_SFC|m/hr | |CaPA outputs | -|short wave radiation |RDRS_v2.1_P_FB_SFC |W m-2|rsds |Downward solar flux | -|long wave radiation |RDRS_v2.1_P_FI_SFC |W m-2|rlds |Downward infrared flux| diff --git a/scripts/rdrs/rdrs.sh b/scripts/rdrs/rdrs.sh deleted file mode 100755 index a8ec96b..0000000 --- a/scripts/rdrs/rdrs.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n extract-dataset -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variable:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "ERROR $(basename $0): arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variable) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "ERROR $(basename $0): invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# check if $ensemble is provided -if [[ -n "$ensemble" ]]; then - echo "ERROR $(basename $0): redundant argument (ensemble) provided"; - exit 1; -fi - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -# the structure of file names is as follows: "YYYYMMDD12.nc" -rdrsFormat="%Y%m%d" # rdrs file date format -exportFormat="%Y%m%d" # exported file date format -fileStruct="" # source dataset files' prefix constant - -latVar="rlat" -lonVar="rlon" - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing ECCC RDRSv2.1..." - -# make the output directory -echo "$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" -echo "$(basename $0): creating cache directory under $cache" -mkdir -p "$cache" - -# define necessary dates -startYear=$(date --date="$startDate" +"%Y") # start year (first folder) -endYear=$(date --date="$endDate" +"%Y") # end year (last folder) -yearsRange=$(seq $startYear $endYear) - -toDate="$startDate" -toDateUnix="$(unix_epoch "$toDate")" -endDateUnix="$(unix_epoch "$endDate")" - -for yr in $yearsRange; do - # creating yearly directory - mkdir -p "$outputDir/$yr" # making the output directory - mkdir -p "$cache/$yr" # making the cache directory - - # setting the end point, either the end of current year, or the $endDate - endOfCurrentYearUnix=$(date --date="$yr-01-01 +1year -1day" "+%s") # last time-step of the current year - if [[ $endOfCurrentYearUnix -le $endDateUnix ]]; then - endPointUnix=$endOfCurrentYearUnix - else - endPointUnix=$endDateUnix - fi - - # extract variables from the forcing data files - while [[ "$toDateUnix" -le "$endPointUnix" ]]; do - # date manipulations - toDateFormatted=$(date --date "$toDate" +"$rdrsFormat") # current timestamp formatted to conform to RDRS naming convention - - # creating file name - file="${toDateFormatted}12.nc" # current file name - - # change lon values so the extents are from ~-180 to 0 - ncap2 -s 'where(lon>0) lon=lon-360' "${datasetDir}/${yr}/${file}" "${cache}/${yr}/${file}" - - # extracting variables from the files and spatial subsetting - cdo -z zip -s -L -sellonlatbox,"$lonLims","$latLims" \ - -selvar,"$variables" \ - "${cache}/${yr}/${file}" "${outputDir}/${yr}/${prefix}${file}" - - [ $( jobs | wc -l ) -ge $( nproc ) ] && wait # forking shell processes - - # increment time-step by one unit - toDate="$(date --date "$toDate 1day")" # current time-step - toDateUnix="$(unix_epoch "$toDate")" # current timestamp in unix EPOCH time - done - - # wait to make sure the while loop is finished - wait - - # go to the next year if necessary - if [[ "$toDateUnix" == "$endOfCurrentYearUnix" ]]; then - toDate=$(date --date "$toDate 1day") - fi - -done - -mkdir "$HOME/empty_dir" -echo "$(basename $0): deleting temporary files from $cache" -rsync -aP --delete "$HOME/empty_dir/" "$cache" -rm -r "$cache" -echo "$(basename $0): temporary files from $cache are removed" -echo "$(basename $0): results are produced under $outputDir" - diff --git a/scripts/wfdei_gem_capa/README.md b/scripts/wfdei_gem_capa/README.md deleted file mode 100644 index 1546a06..0000000 --- a/scripts/wfdei_gem_capa/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# CCRN `WFDEI-GEM-CaPA` -In this file, the details of the dataset is explained. - -## Location of Dataset Files -The `WFDEI-GEM-CaPA` dataset is located under the following directory accessible from Digital Alliance of Canada (formerly Compute Canada) Graham cluster: - -``` -/project/rpp-kshook/Model_Output/181_WFDEI-GEM-CaPA_1979-2016 -``` -and the structure of the dataset hourly files is as following: - -```console -/project/rpp-kshook/Model_Output/181_WFDEI-GEM-CaPA_1979-2016 -├── hus_WFDEI_GEM_1979_2016.Feb29.nc -├── pr_WFDEI_GEM_1979_2016.Feb29.nc -├── ps_WFDEI_GEM_1979_2016.Feb29.nc -├── rlds_WFDEI_GEM_1979_2016.Feb29.nc -├── rsds_WFDEI_GEM_1979_2016.Feb29.nc -├── ta_WFDEI_GEM_1979_2016.Feb29.nc -└── wind_WFDEI_GEM_1979_2016.Feb29.nc -``` - -## Coordinate Variables and Time-stamps - -### Coordinate Variables -The coordinate variables of the `WFDEI-GEM-CaPA` simulations are `lon` and `lat` representing the longitude and latitude points, respectively. - -### Time-stamps -The time-stamps are included in the original files. - -## Dataset Variables -The list of variables included in the dataset is descriped in [Short Description on Dataset Variables](##short-description-on-dataset-variables) - -## Spatial Extent -The spatial extent of the dataset is on latitutes from `31.0625` to `71.9375` and longitudes from `-149.9375` to `-50.0625` covering North America. The resolution is 0.125 degrees. - -## Temporal Extent -The time-steps are 3-hourly covering from `January 1951` to `December 2100`. - -## Short Description on Dataset Variables -In most hydrological modelling applications, usually 7 variables are needed detailed as following: 1) specific humidity at 40 meters, 2) surface pressure, 3) air temperature at 40 meters, 4) wind speed at 40 meters, 5) precipitation, 6) downward short wave radiation, and 7) downward long wave radiation. These variables are available through `WFDEI-GEM-CaPA` dataset and their details are described in the table below: -|Variable Name |Dataset Variable |Unit |IPCC abbreviation|Comments | -|----------------------|-------------------|-----|-----------------|----------------------| -|surface pressure |ps |Pa |ps |surface pressure at time stamp| -|specific humidity@40m |hus |kg/kg|huss |specific humidity elevated to 40m at time stamp| -|air tempreature @40m |ta |K |tas |air temperature elevated to 40m at time stamp| -|wind speed @40m |wind |m/s |wspd |wind speed elevated to 40m at time stamp| -|precipitation |pr |kg m-2 s-1| |Mean rainfall rate over the previous 3 hours| -|short wave radiation |rsds |W m-2|rsds |Mean surface incident shortwave radiation over the previous 3 hours| -|long wave radiation |lsds |W m-2|rlds |Mean surface incident shortwave radiation over the previous 3 hour| - diff --git a/scripts/wfdei_gem_capa/wfdei_gem_capa.sh b/scripts/wfdei_gem_capa/wfdei_gem_capa.sh deleted file mode 100755 index 642397e..0000000 --- a/scripts/wfdei_gem_capa/wfdei_gem_capa.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash -# Meteorological Data Processing Workflow -# Copyright (C) 2022, University of Saskatchewan -# -# This file is part of Meteorological Data Processing Workflow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# ========================= -# Credits and contributions -# ========================= -# 1. Parts of the code are taken from https://www.shellscript.sh/tips/getopt/index.html - - -# ================ -# General comments -# ================ -# * All variables are camelCased for distinguishing from function names; -# * function names are all in lower_case with words seperated by underscore for legibility; -# * shell style is based on Google Open Source Projects' -# Style Guide: https://google.github.io/styleguide/shellguide.html - - -# =============== -# Usage Functions -# =============== -short_usage() { - echo "usage: $(basename $0) [-cio DIR] [-v VARS] [-se DATE] [-t CHAR] [-ln REAL,REAL] [-p STR]" -} - - -# argument parsing using getopt - WORKS ONLY ON LINUX BY DEFAULT -parsedArguments=$(getopt -a -n wfdei_gem_capa -o i:v:o:s:e:t:l:n:p:c:m: --long dataset-dir:,variables:,output-dir:,start-date:,end-date:,time-scale:,lat-lims:,lon-lims:,prefix:,cache:,ensemble: -- "$@") -validArguments=$? -if [ "$validArguments" != "0" ]; then - short_usage; - exit 1; -fi - -# check if no options were passed -if [ $# -eq 0 ]; then - echo "$(basename $0): ERROR! arguments missing"; - exit 1; -fi - -# check long and short options passed -eval set -- "$parsedArguments" -while : -do - case "$1" in - -i | --dataset-dir) datasetDir="$2" ; shift 2 ;; # required - -v | --variables) variables="$2" ; shift 2 ;; # required - -o | --output-dir) outputDir="$2" ; shift 2 ;; # required - -s | --start-date) startDate="$2" ; shift 2 ;; # required - -e | --end-date) endDate="$2" ; shift 2 ;; # required - -t | --time-scale) timeScale="$2" ; shift 2 ;; # redundant - added for compatibility - -l | --lat-lims) latLims="$2" ; shift 2 ;; # required - -n | --lon-lims) lonLims="$2" ; shift 2 ;; # required - -p | --prefix) prefix="$2" ; shift 2 ;; # optional - -c | --cache) cache="$2" ; shift 2 ;; # required - -m | --ensemble) ensemble="$2" ; shift 2 ;; # redundant - added for compatibility - - # -- means the end of the arguments; drop this, and break out of the while loop - --) shift; break ;; - - # in case of invalid option - *) - echo "$(basename $0): ERROR! invalid option '$1'"; - short_usage; exit 1 ;; - esac -done - -# raise error in case --ensemble argument are provided -if [[ -n "$ensemble" ]]; then - echo "$(basename $0): ERROR! invalid option '--ensemble'" - exit 1 -fi - -# make array of variable names -IFS=',' read -ra variablesArr <<< "$(echo "$variables")" - -# check the prefix of not set -if [[ -z $prefix ]]; then - prefix="data" -fi - - -# ===================== -# Necessary Assumptions -# ===================== -# TZ to be set to UTC to avoid invalid dates due to Daylight Saving -alias date='TZ=UTC date' - -# expand aliases for the one stated above -shopt -s expand_aliases - - -# ========================== -# Necessary Global Variables -# ========================== -# the structure of file names is as follows: "%var__WFDEI_GEM_1979_2016.Feb29.nc" -format="%Y-%m-%dT%H:%M:%S" # date format -fileStruct="_WFDEI_GEM_1979_2016.Feb29.nc" # source dataset files' suffix constant - -latVar="lat" -lonVar="lon" -timeVar="time" - - -# =================== -# Necessary Functions -# =================== -# Modules below available on Compute Canada (CC) Graham Cluster Server -load_core_modules () { - module -q load cdo/2.0.4 - module -q load nco/5.0.6 -} -load_core_modules - - -####################################### -# useful one-liners -####################################### -#calcualte Unix EPOCH time in seconds from 1970-01-01 00:00:00 -unix_epoch () { date --date="$@" +"%s"; } - -#check whether the input is float or real -check_real () { if [[ "$1" == *'.'* ]]; then echo 'float'; else echo 'int'; fi; } - -#convert to float if the number is 'int' -to_float () { if [[ $(check_real $1) == 'int' ]]; then printf "%.1f" "$1"; echo; else printf "%.5f" "$1"; echo; fi; } - -#join array element by the specified delimiter -join_by () { local IFS="$1"; shift; echo "$*"; } - -#to_float the latLims and lonLims, real numbers delimited by ',' -lims_to_float () { IFS=',' read -ra l <<< $@; f_arr=(); for i in "${l[@]}"; do f_arr+=($(to_float $i)); done; echo $(join_by , "${f_arr[@]}"); } - - -# =============== -# Data Processing -# =============== -# display info -echo "$(basename $0): processing CCRN WFDEI-GEM_CaPA..." - -# make the output directory -echo "$(basename $0): creating output directory under $outputDir" -mkdir -p "$outputDir" - -# reformat $startDate and $endDate -startDateFormated="$(date --date="$startDate" +"$format")" # startDate -endDateFormated="$(date --date="$endDate" +"$format")" # endDate - -# extract $startYear and $endYear -startYear="$(date --date="$startDate" +"%Y")" -endYear="$(date --date="$endDate" +"%Y")" - -# making the output directory -mkdir -p "$outputDir" - -# loop over variables -for var in "${variablesArr[@]}"; do - ncks -O -d "$latVar",$(lims_to_float "$latLims") \ - -d "$lonVar",$(lims_to_float "$lonLims") \ - -d "$timeVar","$startDateFormated","$endDateFormated" \ - "$datasetDir/${var}${fileStruct}" "$outputDir/${prefix}${var}_WFDEI_GEM_${startYear}_${endYear}.Feb29.nc" - -done - -# wait to assure the loop is over -wait - -echo "$(basename $0): results are produced under $outputDir." - From 5ce933f243577c6e99e0244195027fb633e8ff17 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:14:20 -0500 Subject: [PATCH 29/34] Updates to the documentations The help message has been trimmed to provide more information to the users. This include values provided to the `--lon-lims` that must be within the [-180, +180] limits. This has not been mentioned before to the users and could have provided confusion, as there are multiple methods to describe longitudes. Furthermore, the list of datasets on the main page of the repository has been updated to reflect the most up-to-date list. Signed-off-by: Kasra Keshavarz --- README.md | 118 +++++++++++++++++++++++++-------------------- extract-dataset.sh | 2 + 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 8b9ad3c..d4e25de 100644 --- a/README.md +++ b/README.md @@ -4,51 +4,56 @@ This repository contains scripts to process meteorological datasets in NetCDF fi ```console Usage: extract-dataset [options...] + Script options: - -d, --dataset Meteorological forcing dataset of interest - -i, --dataset-dir=DIR The source path of the dataset file(s) - -v, --variable=var1[,var2[...]] Variables to process - -o, --output-dir=DIR Writes processed files to DIR - -s, --start-date=DATE The start date of the data - -e, --end-date=DATE The end date of the data - -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds - -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds - -a, --shape-file=PATH Path to the ESRI shapefile; optional - -m, --ensemble=ens1,[ens2[...]] Ensemble members to process; optional - Leave empty to extract all ensemble members - -j, --submit-job Submit the data extraction process as a job - on the SLURM system; optional - -k, --no-chunk No parallelization, recommended for small domains - -p, --prefix=STR Prefix prepended to the output files - -b, --parsable Parsable SLURM message mainly used - for chained job submissions - -c, --cache=DIR Path of the cache directory; optional - -E, --email=user@example.com E-mail user when job starts, ends, or - fails; optional - -u, --account Digital Research Alliance of Canada's sponsor's - account name; optional, defaults to 'rpp-kshook' - -V, --version Show version - -h, --help Show this screen and exit + -d, --dataset Meteorological forcing dataset of interest + -i, --dataset-dir=DIR The source path of the dataset file(s) + -v, --variable=var1[,var2[...]] Variables to process + -o, --output-dir=DIR Writes processed files to DIR + -s, --start-date=DATE The start date of the data + -e, --end-date=DATE The end date of the data + -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds; + optional; within the [-90, +90] limits + -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds; + optional; within the [-180, +180] limits + -a, --shape-file=PATH Path to the ESRI shapefile; optional + -m, --ensemble=ens1,[ens2,[...]] Ensemble members to process; optional + Leave empty to extract all ensemble members + -M, --model=model1,[model2,[...]] Models that are part of a dataset, + only applicable to climate datasets, optional + -S, --scenario=scn1,[scn2,[...]] Climate scenarios to process, only applicable + to climate datasets, optional + -j, --submit-job Submit the data extraction process as a job + on the SLURM system; optional + -k, --no-chunk No parallelization, recommended for small domains + -p, --prefix=STR Prefix prepended to the output files + -b, --parsable Parsable SLURM message mainly used + for chained job submissions + -c, --cache=DIR Path of the cache directory; optional + -E, --email=user@example.com E-mail user when job starts, ends, or + fails; optional + -u, --account Digital Research Alliance of Canada's sponsor's + account name; optional, defaults to 'rpp-kshook' + -L, --list-datasets List all the available datasets and the + corresponding keywords for '--dataset' option + -V, --version Show version + -h, --help Show this screen and exit ``` # Available Datasets -|# |Dataset |Time Scale |DOI |Description | -|--|--------------------------|--------------------------------|-------------------------|--------------------------------------| -|1 |WRF-CONUS I (control) |Hourly (Oct 2000 - Dec 2013) |10.1007/s00382-016-3327-9|[link](./scripts/conus_i) | -|2 |WRF-CONUS II (control)[^1]|Hourly (Jan 1995 - Dec 2015) |10.5065/49SN-8E08 |[link](./scripts/conus_ii) | -|3 |ERA5[^2] |Hourly (Jan 1950 - Dec 2020) |10.24381/cds.adbb2d47 and [link](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview)|[link](./scripts/era5)| -|4 |RDRS v2.1 |Hourly (Jan 1980 - Dec 2018) |10.5194/hess-25-4917-2021|[link](./scripts/rdrs) | -|5 |CanRCM4-WFDEI-GEM-CaPA |3-Hourly (Jan 1951 - Dec 2100) |10.5194/essd-12-629-2020 |[link](./scripts/canrcm4_wfdei_gem_capa)| -|6 |WFDEI-GEM-CaPA |3-Hoursly (Jan 1979 - Dec 2016) |10.20383/101.0111 |[link](./scripts/wfdei_gem_capa) | -|7 |Daymet |Daily (Jan 1980 - Dec 2022)[^3] |10.3334/ORNLDAAC/2129 |[link](./scripts/daymet) | -|8 |BCC-CSM2-MR |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/bcc-csm2-mr) | -|9 |CNRM-CM6-1 |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/cnrm-cm6-1) | -|10|EC-Earth3-Veg |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/ec-earth3-veg) | -|11|GDFL-CM4 |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/gdfl-cm4) | -|12|GDFL-ESM4 |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/gdfl-esm4) | -|13|IPSL-CM6A-LR |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/ipsl-cm6a-lr) | -|14|MRI-ESM2-0 |Daily (Jan 1950 - Dec 2100)[^4] |*TBD* |[link](./scripts/mri-esm2-0) | -|15|Hybrid Observation(AB Gov)|Daily (Jan 1950 - Dec 2019)[^4] |10.5194/hess-23-5151-2019|[link](./scripts/hybrid_obs) | +|# |Dataset |Time Period |DOI |Description | +|--|---------------------------|--------------------------------|--------------------------|-------------------------------------| +|1 |GWF-NCAR WRF-CONUS I |Hourly (Oct 2000 - Dec 2013) |10.1007/s00382-016-3327-9 |[link](./scripts/conus_i) | +|2 |GWF-NCAR WRF-CONUS II[^1] |Hourly (Jan 1995 - Dec 2015) |10.5065/49SN-8E08 |[link](./scripts/conus_ii) | +|3 |ECMWF ERA5[^2] |Hourly (Jan 1950 - Dec 2020) |10.24381/cds.adbb2d47 and [link](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview)|[link](./scripts/era5)| +|4 |ECCC RDRSv2.1 |Hourly (Jan 1980 - Dec 2018) |10.5194/hess-25-4917-2021 |[link](./scripts/eccc-rdrs) | +|5 |CCRN CanRCM4-WFDEI-GEM-CaPA|3-Hourly (Jan 1951 - Dec 2100) |10.5194/essd-12-629-2020 |[link](./scripts/ccrn-canrcm4_wfdei_gem_capa)| +|6 |CCRN WFDEI-GEM-CaPA |3-Hourly (Jan 1979 - Dec 2016) |10.20383/101.0111 |[link](./scripts/ccrn-wfdei_gem_capa)| +|7 |ORNL Daymet |Daily (Jan 1980 - Dec 2022)[^3] |10.3334/ORNLDAAC/2129 |[link](./scripts/ornl-daymet) | +|8 |Alberta Gov Climate Dataset|Daily (Jan 1950 - Dec 2100) |10.5194/hess-23-5151-201 |[link](./scripts/ab-gov) | +|9 |Ouranos ESPO-G6-R2 |Daily (Jan 1950 - Dec 2100) |10.1038/s41597-023-02855-z|[link](./scripts/ouranos-espo-g6-r2) | +|10|Ouranos MRCC5-CMIP6 |hourly (Jan 1950 - Dec 2100) |TBD |[link](./scripts/ouranos-mrcc5-cmip6)| +|11|NASA NEX-GDDP-CMIP6 |Daily (Jan 1950 - Dec 2100) |10.1038/s41597-022-01393-4|[link](./scripts/nasa-nex-gddp-cmip6)| [^1]: For access to the files on Graham cluster, please contact [Stephen O'Hearn](mailto:sdo124@mail.usask.ca). [^2]: ERA5 data from 1950-1979 are based on [ERA5 preliminary extenion](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview) and 1979 onwards are based on [ERA5 1979-present](https://doi.org/10.24381/cds.adbb2d47). @@ -56,21 +61,22 @@ Script options: [^4]: Data is not publicly available yet. DOI is to be determined once the relevant paper is published. # General Example -As an example, follow the code block below. Please remember that you MUST have access to Graham cluster with Digital Research Alliance of Canada (DRA) and have access to `CONUS I` model outputs. Also, remember to generate a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with GitHub in advance. Enter the following codes in your Graham shell as a test case: +As an example, follow the code block below. Please remember that you MUST have access to Digital Research Alliance of Canada (DRA) clusters (specifically `Graham`) and have access to `RDRSv2.1` model outputs. Also, remember to generate a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with GitHub in advance. Enter the following codes in your Graham shell as a test case: ```console foo@bar:~$ git clone https://github.com/kasra-keshavarz/datatool # clone the repository foo@bar:~$ cd ./datatool/ # move to the repository's directory foo@bar:~$ ./extract-dataset.sh -h # view the usage message -foo@bar:~$ ./extract-dataset.sh --dataset=CONUS1 \ - --dataset-dir="/project/rpp-kshook/Model_Output/WRF/CONUS/CTRL" \ - --output-dir="$HOME/scratch/conus_i_output/" \ - --start-date="2001-01-01 00:00:00" \ - --end-date="2001-12-31 23:00:00" \ - --lat-lims=49,51 \ - --lon-lims=-117,-115 \ - --variable=T2,PREC_ACC_NC,Q2,ACSWDNB,ACLWDNB,U10,V10,PSFC \ - --prefix="conus_i"; +foo@bar:~$ ./extract-dataset.sh \ + --dataset="rdrs" \ + --dataset-dir="/project/rpp-kshook/Climate_Forcing_Data/meteorological-data/rdrsv2.1" \ + --output-dir="$HOME/scratch/rdrs_outputs/" \ + --start-date="2001-01-01 00:00:00" \ + --end-date="2001-12-31 23:00:00" \ + --lat-lims=49,51 \ + --lon-lims=-117,-115 \ + --variable="RDRS_v2.1_A_PR0_SFC,RDRS_v2.1_P_HU_09944" \ + --prefix="testing_"; ``` See the [examples](./examples) directory for real-world scripts for each meteorological dataset included in this repository. @@ -80,10 +86,16 @@ only in cases where jobs are submitted to clusters' schedulers. If processing is not submitted as a job, then the logs are printed on screen. # New Datasets -If you are considering any new dataset to be added to the data repository, and subsequently the associated scripts added here, you can open a new ticket on the **Issues** tab of the current repository. Or, you can make a [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) on this repository with your own script. +If you are considering any new dataset to be added to the data +repository, and subsequently the associated scripts added here, +you can open a new ticket on the **Issues** tab of the current +repository. Or, you can make a +[Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) +on this repository with your own script. # Support -Please open a new ticket on the **Issues** tab of the current repository in case of any issues. +Please open a new ticket on the **Issues** tab of this repository for +support. # License Meteorological Data Processing Workflow - datatool
diff --git a/extract-dataset.sh b/extract-dataset.sh index 946b73e..b3b39e7 100755 --- a/extract-dataset.sh +++ b/extract-dataset.sh @@ -60,7 +60,9 @@ Script options: -s, --start-date=DATE The start date of the data -e, --end-date=DATE The end date of the data -l, --lat-lims=REAL,REAL Latitude's upper and lower bounds + optional; within the [-90, +90] limits -n, --lon-lims=REAL,REAL Longitude's upper and lower bounds + optional; within the [-180, +180] limits -a, --shape-file=PATH Path to the ESRI shapefile; optional -m, --ensemble=ens1,[ens2,[...]] Ensemble members to process; optional Leave empty to extract all ensemble members From 877b24aa56adb53092bc099a0f091d0f38d93a7a Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:24:24 -0500 Subject: [PATCH 30/34] Upgrading style of warning message --- scripts/ornl-daymet/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/ornl-daymet/README.md b/scripts/ornl-daymet/README.md index 0f8edee..c487232 100644 --- a/scripts/ornl-daymet/README.md +++ b/scripts/ornl-daymet/README.md @@ -62,7 +62,8 @@ The spatial resolutaion of `Daymet` gridded data is 1 $km$. The model files are |2 |`pr` | `+16.85`° to `+19.93`° | `-67.97`° to `-64.13`° | |3 |`hi` | `+17.96`° to `+23.51`° | `-160.30`° to `-154.78`° | -:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. +> [!WARNING] +> As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. ## Temporal Extent The time-steps are daily and the temporal extent for each domain is listed in the following table: @@ -74,7 +75,8 @@ The time-steps are daily and the temporal extent for each domain is listed in th Also, "[t]he Daymet calendar is based on a standard calendar year. All Daymet years have 1 - 365 days, including leap years. For leap years, the Daymet database includes leap day. Values for December 31 are discarded from leap years to maintain a 365-day year." -:warning: As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. +> [!WARNING] +> As mentioned previously, only the `na` domain is considered in the subsetting process of `datatool`. ## Short Description on `Daymet` Variables The variables currently available through the `Daymet` dataset and their details are described in the table below, taken from the [source](https://daymet.ornl.gov/overview): From 69a80a3ea7742f43961e213fd747376acd7dc33e Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:25:11 -0500 Subject: [PATCH 31/34] Upgrading style of warning message --- scripts/ornl-daymet/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ornl-daymet/README.md b/scripts/ornl-daymet/README.md index c487232..1f4daa7 100644 --- a/scripts/ornl-daymet/README.md +++ b/scripts/ornl-daymet/README.md @@ -1,7 +1,8 @@ # `Daymet` dataset In this file, the details of the dataset is explained. -:warning: the dataset files are divided between three different spatial domains: 1) North America (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). For the moment, only the `na` domain is considered in `datatool`. +> [!WARNING] +> the dataset files are divided between three different spatial domains: 1) North America (na), 2) Peurto Rico (pr), and 3) Hawaii (hi). For the moment, only the `na` domain is considered in `datatool`. ## Location of Dataset Files The global `Daymet` dataset is located under the following directory accessible from Digital Research Alliance of Canada (DRA) Graham cluster: From b61717f8937bd58debd9b3d5b5ad1a3554139f0b Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:25:41 -0500 Subject: [PATCH 32/34] Updating link addresses for CONUS I & II --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d4e25de..4fcd1fa 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,8 @@ Script options: # Available Datasets |# |Dataset |Time Period |DOI |Description | |--|---------------------------|--------------------------------|--------------------------|-------------------------------------| -|1 |GWF-NCAR WRF-CONUS I |Hourly (Oct 2000 - Dec 2013) |10.1007/s00382-016-3327-9 |[link](./scripts/conus_i) | -|2 |GWF-NCAR WRF-CONUS II[^1] |Hourly (Jan 1995 - Dec 2015) |10.5065/49SN-8E08 |[link](./scripts/conus_ii) | +|1 |GWF-NCAR WRF-CONUS I |Hourly (Oct 2000 - Dec 2013) |10.1007/s00382-016-3327-9 |[link](./scripts/gwf-ncar-conus_i) | +|2 |GWF-NCAR WRF-CONUS II[^1] |Hourly (Jan 1995 - Dec 2015) |10.5065/49SN-8E08 |[link](./scripts/gwf-ncar-conus_ii) | |3 |ECMWF ERA5[^2] |Hourly (Jan 1950 - Dec 2020) |10.24381/cds.adbb2d47 and [link](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview)|[link](./scripts/era5)| |4 |ECCC RDRSv2.1 |Hourly (Jan 1980 - Dec 2018) |10.5194/hess-25-4917-2021 |[link](./scripts/eccc-rdrs) | |5 |CCRN CanRCM4-WFDEI-GEM-CaPA|3-Hourly (Jan 1951 - Dec 2100) |10.5194/essd-12-629-2020 |[link](./scripts/ccrn-canrcm4_wfdei_gem_capa)| From 618cc7dafcd36cb95ca89b1cbcb1f0c2efec2b28 Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:27:10 -0500 Subject: [PATCH 33/34] Updating link address to ERA5 dataset --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4fcd1fa..46441ee 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Script options: |--|---------------------------|--------------------------------|--------------------------|-------------------------------------| |1 |GWF-NCAR WRF-CONUS I |Hourly (Oct 2000 - Dec 2013) |10.1007/s00382-016-3327-9 |[link](./scripts/gwf-ncar-conus_i) | |2 |GWF-NCAR WRF-CONUS II[^1] |Hourly (Jan 1995 - Dec 2015) |10.5065/49SN-8E08 |[link](./scripts/gwf-ncar-conus_ii) | -|3 |ECMWF ERA5[^2] |Hourly (Jan 1950 - Dec 2020) |10.24381/cds.adbb2d47 and [link](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview)|[link](./scripts/era5)| +|3 |ECMWF ERA5[^2] |Hourly (Jan 1950 - Dec 2020) |10.24381/cds.adbb2d47 and [link](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-preliminary-back-extension?tab=overview)|[link](./scripts/ecmwf-era5)| |4 |ECCC RDRSv2.1 |Hourly (Jan 1980 - Dec 2018) |10.5194/hess-25-4917-2021 |[link](./scripts/eccc-rdrs) | |5 |CCRN CanRCM4-WFDEI-GEM-CaPA|3-Hourly (Jan 1951 - Dec 2100) |10.5194/essd-12-629-2020 |[link](./scripts/ccrn-canrcm4_wfdei_gem_capa)| |6 |CCRN WFDEI-GEM-CaPA |3-Hourly (Jan 1979 - Dec 2016) |10.20383/101.0111 |[link](./scripts/ccrn-wfdei_gem_capa)| From 0d254fcf9cae93b09b81898b23689cb3c3c78e4e Mon Sep 17 00:00:00 2001 From: Kasra Farmer Date: Tue, 5 Mar 2024 18:32:57 -0500 Subject: [PATCH 34/34] Removing dead link for the Ouranos MRCC5 dataset for now --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 46441ee..55c148c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Script options: |7 |ORNL Daymet |Daily (Jan 1980 - Dec 2022)[^3] |10.3334/ORNLDAAC/2129 |[link](./scripts/ornl-daymet) | |8 |Alberta Gov Climate Dataset|Daily (Jan 1950 - Dec 2100) |10.5194/hess-23-5151-201 |[link](./scripts/ab-gov) | |9 |Ouranos ESPO-G6-R2 |Daily (Jan 1950 - Dec 2100) |10.1038/s41597-023-02855-z|[link](./scripts/ouranos-espo-g6-r2) | -|10|Ouranos MRCC5-CMIP6 |hourly (Jan 1950 - Dec 2100) |TBD |[link](./scripts/ouranos-mrcc5-cmip6)| +|10|Ouranos MRCC5-CMIP6 |hourly (Jan 1950 - Dec 2100) |TBD |link | |11|NASA NEX-GDDP-CMIP6 |Daily (Jan 1950 - Dec 2100) |10.1038/s41597-022-01393-4|[link](./scripts/nasa-nex-gddp-cmip6)| [^1]: For access to the files on Graham cluster, please contact [Stephen O'Hearn](mailto:sdo124@mail.usask.ca).