Merge pull request #15 from RIVM-bioinformatics/dev

Memory requirements per job & auto-updater

ViroConstrictor/ViroConstrictor.py

@@ -1,10 +1,13 @@
-# pylint: disable=C0103
+"""
+Starting point of the ViroConstrictor pipeline and wrapper
 
+Copyright © 2021 RIVM
 
-"""
-#placeholder block
+https://github.com/RIVM-bioinformatics/ViroConstrictor
 """
 
+# pylint: disable=C0103
+
 import argparse
 import multiprocessing
 import os
@@ -17,6 +20,7 @@
 from .functions import MyHelpFormatter, color
 from .runconfigs import WriteConfigs
 from .samplesheet import WriteSampleSheet
+from .update import update
 from .userprofile import ReadConfig
 from .validatefasta import IsValidFasta
 from .version import __version__
@@ -192,6 +196,12 @@ def main():
     --> Change working directories and make necessary local files for snakemake
     --> Run snakemake with appropriate settings
     """
+
+    ##> Check the default userprofile, make it if it doesn't exist
+    conf = ReadConfig(os.path.expanduser("~/.ViroConstrictor_defaultprofile.ini"))
+
+    update(sys.argv, conf)
+
     flags = get_args(sys.argv[1:])
 
     inpath = os.path.abspath(flags.input)
@@ -213,9 +223,6 @@ def main():
 
     Snakefile = os.path.join(here, "workflow", "workflow.smk")
 
-    ##> Check the default userprofile, make it if it doesn't exist
-    conf = ReadConfig(os.path.expanduser("~/.ViroConstrictor_defaultprofile.ini"))
-
     ##@ check if the input directory contains valid files
     if CheckInputFiles(inpath) is False:
         print(

ViroConstrictor/runconfigs.py

@@ -9,6 +9,11 @@
 import yaml
 
 
+def get_max_local_mem():
+    avl_mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    return int(round(avl_mem_bytes / (1024.0 ** 2) - 2000, -3))
+
+
 def SnakemakeConfig(conf, cores, dryrun):
     cores = cores - 2
     compmode = conf["COMPUTING"]["compmode"]
@@ -25,13 +30,14 @@ def SnakemakeConfig(conf, cores, dryrun):
     if compmode == "grid":
         queuename = conf["COMPUTING"]["queuename"]
         threads = "{threads}"
+        mem = "{resources.mem_mb}"
         config = {
             "cores": 300,
             "latency-wait": 60,
             "use-conda": True,
             "dryrun": dryrun,
             "jobname": "ViroConstrictor_{name}.jobid{jobid}",
-            "drmaa": f' -q {queuename} -n {threads} -R "span[hosts=1]"',
+            "drmaa": f' -q {queuename} -n {threads} -R "span[hosts=1]" -M {mem}',
             "drmaa-log-dir": "logs/drmaa",
         }
 
@@ -50,6 +56,8 @@ def SnakemakeParams(conf, cores, ref, prim, feats, platform, samplesheet, amplic
 
     params = {
         "sample_sheet": samplesheet,
+        "computing_execution": conf["COMPUTING"]["compmode"],
+        "max_local_mem": get_max_local_mem(),
         "reference_file": ref,
         "primer_file": prim,
         "features_file": feats,
@@ -60,7 +68,7 @@ def SnakemakeParams(conf, cores, ref, prim, feats, platform, samplesheet, amplic
             "QC": threads_midcpu,
             "AdapterRemoval": threads_lowcpu,
             "PrimerRemoval": threads_highcpu,
-            "Consensus": threads_lowcpu,
+            "Consensus": threads_midcpu,
             "Index": threads_lowcpu,
             "Typing": threads_lowcpu,
         },

ViroConstrictor/update.py

@@ -0,0 +1,123 @@
+import json
+import subprocess
+import sys
+from distutils.version import LooseVersion
+from urllib import request
+
+from .functions import color
+from .userprofile import AskPrompts
+from .version import __version__
+
+
+def update(args, conf):
+
+    autocontinue = False
+    ask_prompt = False
+
+    if conf["GENERAL"]["auto_update"] == "yes":
+        autocontinue = True
+
+    if autocontinue is False:
+        if conf["GENERAL"]["ask_for_update"] == "yes":
+            ask_prompt = True
+
+    if autocontinue is True:
+        try:
+            latest_release = request.urlopen(
+                "https://api.github.com/repos/RIVM-bioinformatics/Viroconstrictor/releases"
+            )
+        except Exception as e:
+            sys.stderr.write("Unable to connect to GitHub API\n" f"{e}")
+            return
+
+        latest_release = json.loads(latest_release.read().decode("utf-8"))[0]
+
+        latest_release_tag = latest_release["tag_name"]
+        latest_release_tag_tidied = LooseVersion(
+            latest_release["tag_name"].lstrip("v").strip()
+        )
+
+        localversion = LooseVersion(__version__)
+
+        if localversion < latest_release_tag_tidied:
+            subprocess.run(
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "install",
+                    "--upgrade",
+                    f"git+https://github.com/RIVM-bioinformatics/ViroConstrictor@{latest_release_tag}",
+                ],
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+            print(
+                f"ViroConstrictor updated to {color.YELLOW + color.BOLD}{latest_release_tag}{color.END}"
+            )
+
+            subprocess.run(args)
+            sys.exit(0)
+        return
+
+    if autocontinue is False and ask_prompt is False:
+        return
+
+    if autocontinue is False and ask_prompt is True:
+        try:
+            latest_release = request.urlopen(
+                "https://api.github.com/repos/RIVM-bioinformatics/Viroconstrictor/releases"
+            )
+        except Exception as e:
+            sys.stderr.write("Unable to connect to GitHub API\n" f"{e}")
+            return
+
+        latest_release = json.loads(latest_release.read().decode("utf-8"))[0]
+
+        latest_release_tag = latest_release["tag_name"]
+        latest_release_tag_tidied = LooseVersion(
+            latest_release["tag_name"].lstrip("v").strip()
+        )
+
+        localversion = LooseVersion(__version__)
+
+        if localversion < latest_release_tag_tidied:
+            if (
+                AskPrompts(
+                    f"""
+There's a new version of ViroConstrictor available.
+
+Current version: {color.RED + color.BOLD}{'v' + __version__}{color.END}
+Latest version: {color.GREEN + color.BOLD}{latest_release_tag}{color.END}\n""",
+                    f"""Do you want to update? [yes/no] """,
+                    ["yes", "no"],
+                    fixedchoices=True,
+                )
+                == "yes"
+            ):
+                subprocess.run(
+                    [
+                        sys.executable,
+                        "-m",
+                        "pip",
+                        "install",
+                        "--upgrade",
+                        f"git+https://github.com/RIVM-bioinformatics/ViroConstrictor@{latest_release_tag}",
+                    ],
+                    check=True,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                )
+
+                print(
+                    f"ViroConstrictor updated to {color.YELLOW + color.BOLD}{latest_release_tag}{color.END}"
+                )
+
+                subprocess.run(args)
+                sys.exit(0)
+            print(f"Skipping update to version {latest_release_tag}")
+            print(f"Continuing...")
+            return
+        return

ViroConstrictor/userprofile.py

@@ -77,16 +77,65 @@ def BuildConfig(file):
 
     if conf_object["COMPUTING"]["compmode"] == "grid":
         conf_object["COMPUTING"]["queuename"] = AskPrompts(
-            f"""Grid mode has been chosen. Please enter the name of computing-queue that you wish to use on your grid/HPC cluster.\nThis is necessary so ViroConstrictor will send all the various tasks to the correct (remote) computers.\n\n{color.BOLD + color.UNDERLINE + color.YELLOW}Please note that this is case-sensitive{color.END}\n""",
+            f"""
+Grid mode has been chosen. Please enter the name of computing-queue that you wish to use on your grid/HPC cluster.\nThis is necessary so ViroConstrictor will send all the various tasks to the correct (remote) computers.\n\n{color.BOLD + color.UNDERLINE + color.YELLOW}Please note that this is case-sensitive{color.END}\n""",
             "Please specify the name of the Queue on your grid/HPC cluster that you wish to use. [free text] ",
             [],
             fixedchoices=False,
         )
 
+    conf_object["GENERAL"] = {
+        "auto_update": AskPrompts(
+            f"""
+ViroConstrictor can check and update itself everytime you run it.
+Please specify whether you wish to enable the auto-update feature.
+            """,
+            f"""Do you wish to enable the auto-update feature? [yes/no] """,
+            ["yes", "no"],
+            fixedchoices=True,
+        )
+    }
+
+    if conf_object["GENERAL"]["auto_update"] == "no":
+        conf_object["GENERAL"]["ask_for_update"] = AskPrompts(
+            f"""
+ViroConstrictor will not automatically update itself, but ViroConstrictor can still check for updates and ask you if you wish to update.
+            """,
+            f"""Do you want ViroConstrictor to {color.YELLOW}ask you{color.END} to update everytime a new update is available? [yes/no] """,
+            ["yes", "no"],
+            fixedchoices=True,
+        )
+
     with open(file, "w") as conffile:
         conf_object.write(conffile)
 
 
+def AllOptionsGiven(config):
+    all_present = True
+
+    if config.has_section("COMPUTING") is True:
+        if config.has_option("COMPUTING", "compmode") is True:
+            if config["COMPUTING"]["compmode"] == "grid":
+                if config.has_option("COMPUTING", "queuename") is False:
+                    all_present = False
+        else:
+            all_present = False
+    else:
+        all_present = False
+
+    if config.has_section("GENERAL") is True:
+        if config.has_option("GENERAL", "auto_update") is True:
+            if config["GENERAL"]["auto_update"] == "no":
+                if config.has_option("GENERAL", "ask_for_update") is False:
+                    all_present = False
+        else:
+            all_present = False
+    else:
+        all_present = False
+
+    return all_present
+
+
 def ReadConfig(file):
     if FileExists(file) is False:
         BuildConfig(file)
@@ -95,4 +144,9 @@ def ReadConfig(file):
 
     config = configparser.ConfigParser()
     config.read(file)
+
+    while AllOptionsGiven(config) is False:
+        BuildConfig(file)
+        config = configparser.ConfigParser()
+        config.read(file)
     return config