Merge pull request #288 from smoors/export

implement exportvariable filter

README.md

@@ -433,6 +433,15 @@ allow_update_submit_opts = false
 options via custom module `det_submit_opts` provided by the pull request being
 processed.
 
+```
+allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"]
+```
+`allowed_exportvars` defines a list of name-value pairs (environment
+variables) that are allowed to be specified in a PR command with the
+`exportvariable` filter. To specify multiple environment variables, multiple
+`exportvariable` filters must be used (one per variable). These variables will
+be exported into the build environment before running the bot/build.sh script.
+
 
 #### `[bot_control]` section
 

app.cfg.example

@@ -132,6 +132,10 @@ no_build_permission_comment = Label `bot:build` has been set by user `{build_lab
 # whether or not to allow updating the submit options via custom module det_submit_opts
 allow_update_submit_opts = false
 
+# defines which name-value pairs (environment variables) are allowed to be
+# exported into the build environment via `exportvariable` filters
+allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"]
+
 
 [deploycfg]
 # script for uploading built software packages

scripts/bot-build.slurm

@@ -10,6 +10,7 @@
 #
 # author: Kenneth Hoste (@boegel)
 # author: Thomas Roeblitz (@trz42)
+# author: Sam Moors (@smoors)
 #
 # license: GPLv2
 #
@@ -23,6 +24,14 @@
 #    for example, repos.cfg and configuration file bundles for repositories
 
 echo "Starting bot-build.slurm"
+EXPORT_VARS_SCRIPT=cfg/export_vars.sh
+if [ -f ${EXPORT_VARS_SCRIPT} ]; then
+    echo "${EXPORT_VARS_SCRIPT} script found in '${PWD}', so sourcing it!"
+    source ${EXPORT_VARS_SCRIPT}
+else
+    echo "could not find ${EXPORT_VARS_SCRIPT} script in '${PWD}', skipping" >&2
+fi
+echo "$EXPORT_VARS_SCRIPT finished"
 BOT_BUILD_SCRIPT=bot/build.sh
 if [ -f ${BOT_BUILD_SCRIPT} ]; then
     echo "${BOT_BUILD_SCRIPT} script found in '${PWD}', so running it!"

tasks/build.py

@@ -46,6 +46,8 @@
 _ERROR_GIT_CLONE = "curl"
 _ERROR_NONE = "none"
 
+# other constants
+EXPORT_VARS_FILE = 'export_vars.sh'
 
 Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id', 'accelerator'))
 
@@ -121,7 +123,7 @@ def get_build_env_cfg(cfg):
     cvmfs_customizations = {}
     try:
         cvmfs_customizations_str = buildenv.get(config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS)
-        log("{fn}(): cvmfs_customizations '{cvmfs_customizations_str}'")
+        log(f"{fn}(): cvmfs_customizations '{cvmfs_customizations_str}'")
 
         if cvmfs_customizations_str is not None:
             cvmfs_customizations = json.loads(cvmfs_customizations_str)
@@ -170,6 +172,34 @@ def get_architecture_targets(cfg):
     return arch_target_map
 
 
+def get_allowed_exportvars(cfg):
+    """
+    Obtain list of allowed export variables
+
+    Args:
+        cfg (ConfigParser): ConfigParser instance holding full configuration
+            (typically read from 'app.cfg')
+
+    Returns:
+        (list): list of allowed export variable-value pairs of the format VARIABLE=VALUE
+    """
+    fn = sys._getframe().f_code.co_name
+
+    buildenv = cfg[config.SECTION_BUILDENV]
+    allowed_str = buildenv.get(config.BUILDENV_SETTING_ALLOWED_EXPORTVARS)
+    allowed = []
+
+    if allowed_str:
+        try:
+            allowed = json.loads(allowed_str)
+        except json.JSONDecodeError as err:
+            print(err)
+            error(f"{fn}(): Value for allowed_exportvars ({allowed_str}) could not be decoded.")
+
+    log(f"{fn}(): allowed_exportvars '{json.dumps(allowed)}'")
+    return allowed
+
+
 def get_repo_cfg(cfg):
     """
     Obtain mappings of architecture targets to repository identifiers and
@@ -463,6 +493,29 @@ def apply_cvmfs_customizations(cvmfs_customizations, arch_job_dir):
             #      for now, only existing mappings may be customized
 
 
+def prepare_export_vars_file(job_dir, exportvars):
+    """
+    Set up EXPORT_VARS_FILE in directory <job_dir>/cfg. This file will be
+    sourced before running the bot/build.sh script.
+
+    Args:
+        job_dir (string): working directory of the job
+        exportvars (list): strings of the form VAR=VALUE to be exported
+
+    Returns:
+        None (implicitly)
+    """
+    fn = sys._getframe().f_code.co_name
+
+    content = '\n'.join(f'export {x}' for x in exportvars)
+    export_vars_path = os.path.join(job_dir, 'cfg', EXPORT_VARS_FILE)
+
+    with open(export_vars_path, 'w') as file:
+        file.write(content)
+
+    log(f"{fn}(): created exported variables file {export_vars_path}")
+
+
 def prepare_jobs(pr, cfg, event_info, action_filter):
     """
     Prepare all jobs whose context matches the given filter. Preparation includes
@@ -484,6 +537,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
     build_env_cfg = get_build_env_cfg(cfg)
     arch_map = get_architecture_targets(cfg)
     repocfg = get_repo_cfg(cfg)
+    allowed_exportvars = get_allowed_exportvars(cfg)
 
     base_repo_name = pr.base.repo.full_name
     log(f"{fn}(): pr.base.repo.full_name '{base_repo_name}'")
@@ -510,6 +564,16 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
         log(f"{fn}(): found no accelerator requirement")
         accelerator = None
 
+    # determine exportvars from action_filter argument
+    exportvars = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_EXPORT)
+
+    # all exportvar filters must be allowed in order to run any jobs
+    if exportvars:
+        not_allowed = [x for x in exportvars if x not in allowed_exportvars]
+        if not_allowed:
+            log(f"{fn}(): exportvariable(s) {not_allowed} not allowed")
+            return []
+
     jobs = []
     for arch, slurm_opt in arch_map.items():
         arch_dir = arch.replace('/', '_')
@@ -564,6 +628,9 @@ def prepare_jobs(pr, cfg, event_info, action_filter):
 
             prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type, accelerator)
 
+            if exportvars:
+                prepare_export_vars_file(job_dir, exportvars)
+
             # enlist jobs to proceed
             job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id, accelerator)
             jobs.append(job)

tools/config.py

@@ -25,7 +25,7 @@
 # Local application imports (anything from EESSI/eessi-bot-software-layer)
 from .logging import error
 
-# define configration constants
+# define configuration constants
 #   SECTION_sectionname for any section name in app.cfg
 #   sectionname_SETTING_settingname for any setting with name settingname in
 #     section sectionname
@@ -37,6 +37,7 @@
 BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT = 'command_response_fmt'
 
 SECTION_BUILDENV = 'buildenv'
+BUILDENV_SETTING_ALLOWED_EXPORTVARS = 'allowed_exportvars'
 BUILDENV_SETTING_ALLOW_UPDATE_SUBMIT_OPTS = 'allow_update_submit_opts'
 BUILDENV_SETTING_BUILD_JOB_SCRIPT = 'build_job_script'
 BUILDENV_SETTING_BUILD_LOGS_DIR = 'build_logs_dir'

tools/filter.py

@@ -20,16 +20,18 @@
 # (none yet)
 
 
-# NOTE because one can use any prefix of one of the four components below to
+# NOTE because one can use any prefix of one of the components below to
 # define a filter, we need to make sure that no two filters share the same
 # prefix OR we have to change the handling of filters.
 FILTER_COMPONENT_ACCEL = 'accelerator'
 FILTER_COMPONENT_ARCH = 'architecture'
+FILTER_COMPONENT_EXPORT = 'exportvariable'
 FILTER_COMPONENT_INST = 'instance'
 FILTER_COMPONENT_JOB = 'job'
 FILTER_COMPONENT_REPO = 'repository'
 FILTER_COMPONENTS = [FILTER_COMPONENT_ACCEL,
                      FILTER_COMPONENT_ARCH,
+                     FILTER_COMPONENT_EXPORT,
                      FILTER_COMPONENT_INST,
                      FILTER_COMPONENT_JOB,
                      FILTER_COMPONENT_REPO