Merge branch 'dev' into documentation/notify_new_possible_errors

.ansible-lint

@@ -35,6 +35,7 @@ exclude_paths:
   - tests/sanity/ignore-2.11.txt
   - tests/sanity/ignore-2.12.txt
   - tests/sanity/ignore-2.13.txt
+  - tests/sanity/ignore-2.14.txt
   - venv*
 parseable: true
 quiet: false

.github/ISSUE_TEMPLATE/bug_issue.yml

@@ -1,7 +1,7 @@
 name: Report a bug
 description: Request that a bug be reviewed. Complete all required fields.
 title: "[Bug] Enter description"
-labels: [Bug]
+labels: ["Bug", "Needs Triage" ]
 assignees:
   - IBMAnsibleHelper
 body:

.github/workflows/ac-ansible-test-sanity.yml

@@ -0,0 +1,71 @@
+name: AC Ansible sanity
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - staging*
+    paths-ignore:
+      - '**.tar.gz'
+      - 'pycache/**'
+      - '.ansible-lint'
+      - 'cache/**'
+      - '.DS_Store'
+      - '.git/**'
+      - '.github/**'
+      - '.gitignore'
+      - '.python-version'
+      - '.pytest_cache/**'
+      - '.vscode/**'
+      - 'Jenkinsfile'
+      - 'ac'
+      - 'ansible.cfg'
+      - 'changelogs/**'
+      - 'collections/**'
+      - 'docs/**'
+      - 'scripts/**'
+      - 'test_config.yml'
+      - 'tests/*.ini'
+      - 'tests/*.py'
+      - 'tests/.pytest_cache'
+      - 'tests/pycache'
+      - 'tests/functional'
+      - 'tests/helpers'
+      - 'tests/requirements.txt'
+      - 'tests/unit'
+      - 'tests/sanity/ignore-*'
+      - 'venv*'
+
+jobs:
+  ansible-sanity:
+    runs-on: ubuntu-latest
+    env:
+      branch: ${{ github.event.pull_request.head.ref }}
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - name: Set up venv
+      run: |
+        python -m pip install --upgrade pip
+        pip install virtualenv
+        mkdir venv
+        virtualenv venv/venv-2.16
+
+    - name: Install dependencies
+      run: |
+        source venv/venv-2.16/bin/activate
+        python -m pip install --upgrade pip
+        pip install ansible
+
+    - name: Run ac-sanity
+      run: |
+        source venv/venv-2.16/bin/activate
+        ./ac --ac-build
+        ./ac --ac-sanity

.github/workflows/ac-bandit.yml

@@ -0,0 +1,38 @@
+name: AC Bandit
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - staging*
+
+jobs:
+  bandit:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - name: Set up venv
+      run: |
+        python -m pip install --upgrade pip
+        pip install virtualenv
+        mkdir venv
+        virtualenv venv/venv-2.16
+
+    - name: Install dependencies
+      run: |
+        source venv/venv-2.16/bin/activate
+        python -m pip install --upgrade pip
+        pip install bandit
+
+    - name: Run ac-bandit
+      run: |
+        source venv/venv-2.16/bin/activate
+        ./ac --ac-bandit --level l

.github/workflows/ac-galaxy-importer.yml

@@ -0,0 +1,40 @@
+name: AC Galaxy Importer
+
+on:
+  pull_request:
+    branches:
+      - dev
+      - staging*
+
+jobs:
+  galaxy-importer:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - name: Set up venv
+      run: |
+        python -m pip install --upgrade pip
+        pip install virtualenv
+        mkdir venv
+        virtualenv venv/venv-2.16
+
+    - name: Install dependencies
+      run: |
+        source venv/venv-2.16/bin/activate
+        python -m pip install --upgrade pip
+        pip install ansible
+        pip install ansible-importer
+        pip install galaxy-importer
+
+    - name: Run ac-galaxy-importer
+      run: |
+        source venv/venv-2.16/bin/activate
+        ./ac --ac-galaxy-importer

CHANGELOG.rst

@@ -5,18 +5,23 @@ ibm.ibm_zos_core Release Notes
 .. contents:: Topics
 
 
-v1.9.0-beta.1
-=============
+v1.9.0
+======
 
 Release Summary
 ---------------
 
-Release Date: '2024-01-31'
+Release Date: '2024-03-11'
 This changelog describes all changes made to the modules and plugins included
 in this collection. The release date is the date the changelog is created.
 For additional details such as required dependencies and availability review
 the collections `release notes <https://ibm.github.io/z_ansible_collections_doc/ibm_zos_core/docs/source/release_notes.html>`__
 
+Major Changes
+-------------
+
+- zos_job_submit - when job statuses were read, were limited to AC (active), CC (completed normally), ABEND (ended abnormally) and ? (error unknown), SEC (security error), JCLERROR (job had a jcl error). Now the additional statuses are supported, CANCELLED (job was cancelled), CAB (converter abend), CNV (converter error), SYS (system failure) and FLU (job was flushed). (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+
 Minor Changes
 -------------
 
@@ -32,11 +37,22 @@ Minor Changes
 Bugfixes
 --------
 
+- module_utils/job.py - job output containing non-printable characters would crash modules. Fix now handles the error gracefully and returns a message to the user inside `content` of the `ddname` that failed. (https://github.com/ansible-collections/ibm_zos_core/pull/1288).
+- zos_apf - When operation=list was selected and more than one data set entry was fetched, the module only returned one data set. Fix now returns the complete list. (https://github.com/ansible-collections/ibm_zos_core/pull/1236).
 - zos_copy - When copying an executable data set with aliases and destination did not exist, destination data set was created with wrong attributes. Fix now creates destination data set with the same attributes as the source. (https://github.com/ansible-collections/ibm_zos_core/pull/1066).
 - zos_copy - When performing a copy operation to an existing file, the copied file resulted in having corrupted contents. Fix now implements a workaround to not use the specific copy routine that corrupts the file contents. (https://github.com/ansible-collections/ibm_zos_core/pull/1064).
+- zos_data_set - Fixes a small parsing bug in module_utils/data_set function which extracts volume serial(s) from a LISTCAT command output. Previously a leading '-' was left behind for volser strings under 6 chars. (https://github.com/ansible-collections/ibm_zos_core/pull/1247).
 - zos_job_output - When passing a job ID or name less than 8 characters long, the module sent the full stack trace as the module's message. Change now allows the use of a shorter job ID or name, as well as wildcards. (https://github.com/ansible-collections/ibm_zos_core/pull/1078).
 - zos_job_query - The module handling ZOAU import errors obscured the original traceback when an import error ocurred. Fix now passes correctly the context to the user. (https://github.com/ansible-collections/ibm_zos_core/pull/1042).
 - zos_job_query - When passing a job ID or name less than 8 characters long, the module sent the full stack trace as the module's message. Change now allows the use of a shorter job ID or name, as well as wildcards. (https://github.com/ansible-collections/ibm_zos_core/pull/1078).
+- zos_job_submit - Was ignoring the default value for location=DATA_SET, now when location is not specified it will default to DATA_SET. (https://github.com/ansible-collections/ibm_zos_core/pull/1120).
+- zos_job_submit - when a JCL error occurred, the ret_code[msg_code] contained JCLERROR followed by an integer where the integer appeared to be a reason code when actually it is a multi line marker used to coordinate errors spanning more than one line. Now when a JCLERROR occurs, only the JCLERROR is returned for property ret_code[msg_code]. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when a response was returned, it contained an undocumented property; ret_code[msg_text]. Now when a response is returned, it correctly returns property ret_code[msg_txt]. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when typrun=copy was used in JCL it would fail the module with an improper message and error condition. While this case continues to be considered a failure, the message has been corrected and it fails under the condition that not enough time has been added to the modules execution. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when typrun=hold was used in JCL it would fail the module with an improper message and error condition. While this case continues to be considered a failure, the message has been corrected and it fails under the condition that not enough time has been added to the modules execution. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when typrun=jchhold was used in JCL it would fail the module with an improper message and error condition. While this case continues to be considered a failure, the message has been corrected and it fails under the condition that not enough time has been added to the modules execution. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when typrun=scan was used in JCL, it would fail the module. Now typrun=scan no longer fails the module and an appropriate message is returned with appropriate return code values. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+- zos_job_submit - when wait_time_s was used, the duration would run approximately 5 second longer than reported in the duration. Now the when duration is returned, it is the actual accounting from when the job is submitted to when the module reads the job output. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
 - zos_operator - The module handling ZOAU import errors obscured the original traceback when an import error ocurred. Fix now passes correctly the context to the user. (https://github.com/ansible-collections/ibm_zos_core/pull/1042).
 - zos_unarchive - Using a local file with a USS format option failed when sending to remote because dest_data_set option had an empty dictionary. Fix now leaves dest_data_set as None when using a USS format option. (https://github.com/ansible-collections/ibm_zos_core/pull/1045).
 - zos_unarchive - When unarchiving USS files, the module left temporary files on the remote. Change now removes temporary files. (https://github.com/ansible-collections/ibm_zos_core/pull/1073).

README.md

@@ -51,7 +51,7 @@ and ansible-doc to automate tasks on z/OS.
 
 Ansible version compatibility
 =============================
-This collection has been tested against **Ansible Core** versions >=2.14.
+This collection has been tested against **Ansible Core** versions >=2.15.
 The Ansible Core versions supported for this collection align to the
 [ansible-core support matrix](https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix). Review the
 [Ansible community changelogs](https://docs.ansible.com/ansible/latest/reference_appendices/release_and_maintenance.html#ansible-community-changelogs) for corresponding **Ansible community packages**
@@ -64,11 +64,12 @@ for more more information on supported versions of Ansible.
 
 Other Dependencies
 ==================
-This release of the **IBM z/OS core collection** requires the z/OS managed node have:
-- [z/OS](https://www.ibm.com/docs/en/zos) V2R4 or later.
+This release of the **IBM z/OS core collection** requires the z/OS managed node have the following:
+- [z/OS](https://www.ibm.com/docs/en/zos)
 - [z/OS shell](https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.4.0/com.ibm.zos.v2r4.bpxa400/part1.htm).
-- [IBM Open Enterprise SDK for Python](https://www.ibm.com/products/open-enterprise-python-zos) 3.9 - 3.11.
-- [IBM Z Open Automation Utilities](https://www.ibm.com/docs/en/zoau/1.2.x) 1.2.5 (or later) but prior to version 1.3.
+- [IBM Open Enterprise SDK for Python](https://www.ibm.com/products/open-enterprise-python-zos)
+- [IBM Z Open Automation Utilities](https://www.ibm.com/docs/en/zoau/1.2.x)
+For specific dependency versions, please review the [release notes](https://ibm.github.io/z_ansible_collections_doc/ibm_zos_core/docs/source/release_notes.html) for the version of the IBM Ansible z/OS core installed.
 
 Copyright
 =========

ac

@@ -242,6 +242,18 @@ ac_build(){
 }
 
 # ------------------------------------------------------------------------------
+# Run galaxy importer on collection.
+# ------------------------------------------------------------------------------
+#->ac-galaxy-importer:
+## Build current branch and run galaxy importer on collection.
+## Usage: ac [--ac-galaxy-importer]
+## Example:
+##  $ ac --ac-galaxy-importer
+ac_galaxy_importer(){
+    message "Running Galaxy Importer"
+    . $VENV_BIN/activate && collection_name=$($VENV_BIN/ansible-galaxy collection build --force | awk -F/ '{print $NF}') && python -m galaxy_importer.main $collection_name
+}
+
 # Run a changelog lint locally
 # ------------------------------------------------------------------------------
 #->ac-changelog:
@@ -679,6 +691,10 @@ while true; do
             ensure_managed_venv_exists $1
             option_submitted="--ac-build"
             ;;
+      --ac-galaxy-importer)                              # Command
+            ensure_managed_venv_exists $1
+            option_submitted="--ac-galaxy-importer"
+            ;;
       --ac-changelog)                              # Command
             ensure_managed_venv_exists $1
             option_submitted="--ac-changelog"
@@ -835,6 +851,8 @@ if [ "$option_submitted" ] && [ "$option_submitted" = "--ac-bandit" ] ; then
     ac_bandit $level
 elif [ "$option_submitted" ] && [ "$option_submitted" = "--ac-build" ] ; then
     ac_build
+elif [ "$option_submitted" ] && [ "$option_submitted" = "--ac-galaxy-importer" ] ; then
+    ac_galaxy_importer
 elif [ "$option_submitted" ] && [ "$option_submitted" = "--ac-changelog" ] ; then
     ac_changelog $command
 elif [ "$option_submitted" ] && [ "$option_submitted" = "--ac-install" ] ; then

changelogs/.plugin-cache.yaml

@@ -6,7 +6,11 @@ plugins:
   callback: {}
   cliconf: {}
   connection: {}
-  filter: {}
+  filter:
+    filter_wtor_messages:
+      description: Filter a list of WTOR messages
+      name: filter_wtor_messages
+      version_added: 1.2.0
   httpapi: {}
   inventory: {}
   lookup: {}
@@ -131,4 +135,4 @@ plugins:
   strategy: {}
   test: {}
   vars: {}
-version: 1.9.0-beta.1
+version: 1.9.0

changelogs/changelog.yaml

@@ -1176,6 +1176,76 @@ releases:
       name: zos_script
       namespace: ''
     release_date: '2023-10-24'
+  1.9.0:
+    changes:
+      bugfixes:
+      - module_utils/job.py - job output containing non-printable characters would
+        crash modules. Fix now handles the error gracefully and returns a message
+        to the user inside `content` of the `ddname` that failed. (https://github.com/ansible-collections/ibm_zos_core/pull/1288).
+      - zos_apf - When operation=list was selected and more than one data set entry
+        was fetched, the module only returned one data set. Fix now returns the complete
+        list. (https://github.com/ansible-collections/ibm_zos_core/pull/1236).
+      - zos_data_set - Fixes a small parsing bug in module_utils/data_set function
+        which extracts volume serial(s) from a LISTCAT command output. Previously
+        a leading '-' was left behind for volser strings under 6 chars. (https://github.com/ansible-collections/ibm_zos_core/pull/1247).
+      - zos_job_submit - Was ignoring the default value for location=DATA_SET, now
+        when location is not specified it will default to DATA_SET. (https://github.com/ansible-collections/ibm_zos_core/pull/1120).
+      - zos_job_submit - when a JCL error occurred, the ret_code[msg_code] contained
+        JCLERROR followed by an integer where the integer appeared to be a reason
+        code when actually it is a multi line marker used to coordinate errors spanning
+        more than one line. Now when a JCLERROR occurs, only the JCLERROR is returned
+        for property ret_code[msg_code]. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when a response was returned, it contained an undocumented
+        property; ret_code[msg_text]. Now when a response is returned, it correctly
+        returns property ret_code[msg_txt]. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when typrun=copy was used in JCL it would fail the module
+        with an improper message and error condition. While this case continues to
+        be considered a failure, the message has been corrected and it fails under
+        the condition that not enough time has been added to the modules execution.
+        (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when typrun=hold was used in JCL it would fail the module
+        with an improper message and error condition. While this case continues to
+        be considered a failure, the message has been corrected and it fails under
+        the condition that not enough time has been added to the modules execution.
+        (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when typrun=jchhold was used in JCL it would fail the module
+        with an improper message and error condition. While this case continues to
+        be considered a failure, the message has been corrected and it fails under
+        the condition that not enough time has been added to the modules execution.
+        (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when typrun=scan was used in JCL, it would fail the module.
+        Now typrun=scan no longer fails the module and an appropriate message is returned
+        with appropriate return code values. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      - zos_job_submit - when wait_time_s was used, the duration would run approximately
+        5 second longer than reported in the duration. Now the when duration is returned,
+        it is the actual accounting from when the job is submitted to when the module
+        reads the job output. (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      major_changes:
+      - zos_job_submit - when job statuses were read, were limited to AC (active),
+        CC (completed normally), ABEND (ended abnormally) and ? (error unknown), SEC
+        (security error), JCLERROR (job had a jcl error). Now the additional statuses
+        are supported, CANCELLED (job was cancelled), CAB (converter abend), CNV (converter
+        error), SYS (system failure) and FLU (job was flushed). (https://github.com/ansible-collections/ibm_zos_core/pull/1283).
+      release_summary: 'Release Date: ''2024-03-11''
+
+        This changelog describes all changes made to the modules and plugins included
+
+        in this collection. The release date is the date the changelog is created.
+
+        For additional details such as required dependencies and availability review
+
+        the collections `release notes <https://ibm.github.io/z_ansible_collections_doc/ibm_zos_core/docs/source/release_notes.html>`__'
+    fragments:
+    - 1120-bugfix-zos_job_submit-default_value.yml
+    - 1236-bugfix-zos_apf-return-list.yml
+    - 1246-bugfix-zos_job_submit-typrun.yml
+    - 1247-volser-parsing-leading-dash-bugfix.yml
+    - 1288-job-submit-non-utf8-chars.yml
+    - 1292-doc-zos_tso_command-example.yml
+    - 1294-doc-zos_ping-scp.yml
+    - 1296-doc-sftp-collection-requirements.yml
+    - v1.9.0_summary.yml
+    release_date: '2024-03-16'
   1.9.0-beta.1:
     changes:
       bugfixes:

changelogs/fragments/1032-clean-job_submit-test.yml

@@ -0,0 +1,3 @@
+trivial:
+  - test_zos_job_submit_func.py - Removed test setting that was covering a missing duration value.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1364).

changelogs/fragments/1307-update-sanity-zos_copy.yml

@@ -0,0 +1,10 @@
+minor_changes:
+  - zos_copy - Documented `group` and `owner` options.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1307).
+
+trivial:
+  - zos_copy - Removed many of the variables that were passed from the
+    action plugin to the module, reimplementing the logic inside the
+    module instead. Removed the use of temp_path variable inside zos_copy
+    in favor of using remote_src to deal with files copied to remote.
+    (https://github.com/ansible-collections/ibm_zos_core/pull/1307).