Merge branch 'master' into master

nf-core · Jan 7, 2025 · 74133e1 · 74133e1
2 parents 72abf00 + ea8ed7b
commit 74133e1
Show file tree

Hide file tree

Showing 13 changed files with 278 additions and 159 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -153,6 +153,7 @@ jobs:
           - "ucd_sonic"
           - "uge"
           - "unibe_ibu"
+          - "unity"
           - "unc_lccc"
           - "unc_longleaf"
           - "uod_hpc"
@@ -172,7 +173,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Install Nextflow
+      - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+      - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "latest-everything"

diff --git a/README.md b/README.md
@@ -205,6 +205,7 @@ Currently documentation is available for the following systems:
 - [UNC_LONGLEAF](docs/unc_longleaf.md)
 - [UGE](docs/uge.md)
 - [UNIBE_IBU](docs/unibe_ibu.md)
+- [Unity](docs/unity.md)
 - [UOD_HPC](docs/uod_hpc.md)
 - [UPPMAX](docs/uppmax.md)
 - [UTD_GANYMEDE](docs/utd_ganymede.md)

diff --git a/conf/daisybio.config b/conf/daisybio.config
@@ -11,7 +11,17 @@ params {
 // define workDir in /nfs/scratch/nf-core_work/ named after the launch dir
 def work_dir = "/nfs/scratch/nf-core_work/"
 if(new File(work_dir).exists() && System.getenv("PWD")) {
-        workDir = work_dir+System.getenv("PWD").tokenize('/').join('.')
+    work_dir = work_dir+System.getenv("PWD").tokenize('/').join('.')
+    workDir = work_dir
+
+    // if directory does not exist, create it and set the group to the group launch dir
+    if(!new File(work_dir).exists()) {
+        "mkdir -p ${work_dir}".execute()
+        def pwd = System.getenv("PWD")
+        def group = "stat -c %g ${pwd}".execute().text.trim()
+        "chgrp -R ${group} ${work_dir}".execute()
+        "chmod -R g+s ${work_dir}".execute()
+    }
 }
 
 process {

diff --git a/conf/eva.config b/conf/eva.config
@@ -25,14 +25,14 @@ profiles {
         params {
             igenomes_base              = "/mnt/archgen/public_data/igenomes"
             config_profile_description = 'MPI-EVA archgen profile, provided by nf-core/configs.'
-            max_memory                 = 256.GB
+            max_memory                 = 370.GB
             max_cpus                   = 32
             max_time                   = 365.d
         }
 
         process {
             resourceLimits = [
-                memory: 256.GB,
+                memory: 370.GB,
                 cpus: 32,
                 time: 365.d
             ]

diff --git a/conf/pdc_kth.config b/conf/pdc_kth.config
@@ -40,7 +40,7 @@ def clusterOptionsCreator = { mem, time, cpus ->
         case "dardel":
             String extra = ''
 
-            if (time < 7.d && mem <= 111.GB && cpus < 256) {
+            if (time <= 7.d && mem <= 111.GB && cpus <= 256) {
                 extra += ' -p shared '
             }
             else if (time < 1.d) {

diff --git a/conf/psmn.config b/conf/psmn.config
@@ -18,47 +18,47 @@ process {
         cpus: 32,
         time: 96.h
     ]
-    executor       = 'slurm'
-    clusterOptions = "--partition=E5,Lake,Lake-flix"
+    executor = 'slurm'
+    queue = "E5,Lake"
 
-    cpus           = 1
-    memory         = 32.GB
-    time           = 24.h
+    cpus     = 1
+    memory   = 32.GB
+    time     = 24.h
 
     withLabel: 'process_single|process_single_thread|sc_tiny|sc_small|sc_medium' {
-        clusterOptions = "--partition=E5,Lake,Lake-flix"
-        cpus           = 1
-        memory         = 114.GB
-        time           = 24.h
+        queue = "Lake"
+        cpus   = 1
+        memory = 96.GB
+        time   = 24.h
     }
     withLabel: 'process_low|mc_small|process_very_low' {
-        clusterOptions = "--partition=E5,Lake,Lake-flix"
-        cpus           = 16
-        memory         = 90.GB
-        time           = 24.h
+        queue = "Lake"
+        cpus   = 16
+        memory = 80.GB
+        time   = 24.h
     }
     withLabel: 'process_medium|mc_medium' {
-        clusterOptions = "--partition=Lake,Lake-flix"
-        cpus           = 32
-        memory         = 180.GB
-        time           = 48.h
+        queue = "Lake"
+        cpus   = 32
+        memory = 180.GB
+        time   = 48.h
     }
     withLabel: 'process_high|mc_large|mc_huge|process_high_cpus|cpus_max' {
-        clusterOptions = "--partition=Lake,Lake-flix"
-        cpus           = 32
-        memory         = 370.GB
-        time           = 48.h
+        queue = "Lake"
+        cpus   = 32
+        memory = 370.GB
+        time   = 48.h
     }
     withLabel: 'process_long|process_maximum_time|process_long_parallelized' {
-        clusterOptions = "--partition=Lake"
-        time           = 96.h
+        queue = "Lake"
+        time   = 96.h
     }
     withLabel: 'process_high_memory|memory_max' {
-        clusterOptions = "--partition=Lake,Lake-flix"
-        memory         = 370.GB
+        queue = "Epyc"
+        memory = 500.GB
     }
     withLabel: gpu {
-        clusterOptions = "--partition=E5-GPU"
+        queue = "E5-GPU"
     }
 }
 

diff --git a/conf/unity.config b/conf/unity.config
@@ -0,0 +1,51 @@
+/*
+    * -------------------------------------------------
+    *  Unity HPC cluster config file for nf-core
+    * -------------------------------------------------
+    * https://unity.rc.umass.edu/
+    */
+
+params {
+    config_profile_description = 'Unity HPC cluster profile provided by nf-core/configs.'
+    config_profile_contact = 'Berent Aldikacti (@baldikacti)'
+    config_profile_url = 'https://unity.rc.umass.edu/'
+
+    igenomes_base = '/datasets/bio/igenomes'
+
+    max_memory = 2.TB
+    max_cpus = 192
+    max_time = 14.d
+}
+
+process {
+    resourceLimits = [
+        cpus: 192,
+        memory: 2.TB,
+        time: 14.d
+    ]
+    executor = 'slurm'
+
+    // Selects partition based on process time
+    queue = { task.time <= 2.h ? 'cpu-preempt' : 'cpu' }
+    clusterOptions = { "${task.time >= 48.h ? '-q long' : ''}" }
+
+    maxRetries = 2
+
+}
+
+// Limits job submission to 1000 consecutive run and 20 submissions per second
+executor {
+    queueSize = 1000
+    submitRateLimit = '20sec'
+    pollInterval = '30sec'
+    queueStatInterval = '1min'
+}
+
+apptainer {
+    enabled = true
+    autoMounts = true
+    pullTimeout = '120m'
+}
+
+// clean the generated files in the working directory
+cleanup = true
diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config
@@ -19,15 +19,16 @@ singularity {
 }
 
 def membership = "groups".execute().text
+def genomics_queue_memory = 28.GB
 
 def select_queue = { memory, cpu ->
-    if (memory <= 28.GB && cpu <= 16 && membership.contains('genomics')) {
+    if (memory <= genomics_queue_memory && cpu <= 16 && membership.contains('genomics')) {
         return 'genomics,normal'
     }
-    if (memory > 28.GB && memory <= 125.GB && cpu <= 12 && membership.contains('kim')) {
+    if (memory > genomics_queue_memory && memory <= 125.GB && cpu <= 12 && membership.contains('kim')) {
         return 'Kim,128s'
     }
-    if (memory > 28.GB && memory <= 125.GB && cpu <= 16) {
+    if (memory > genomics_queue_memory && memory <= 125.GB && cpu <= 16) {
         return '128s'
     }
     if (memory <= 250.GB && cpu <= 28) {
@@ -56,13 +57,12 @@ process {
     queue = { select_queue(task.memory, task.cpu) }
 
     withLabel:process_medium {
-        cpus   = { 16 * task.attempt }
-        memory = { 30.GB * task.attempt }
+        cpus   = 16
+        memory = { 27.GB * task.attempt }
     }
-}
 
-params {
-    max_memory = 250.GB
-    max_cpus = 28
-    max_time = 96.h
+    withName: TRIMGALORE {
+        cpus   = 16
+        memory = { 27.GB * task.attempt }
+    }
 }