Merge branch 'release-5.2.x'

opencb · Jul 17, 2024 · b0812f6 · b0812f6
2 parents 717f469 + 67acad1
commit b0812f6
Show file tree

Hide file tree

Showing 16 changed files with 235 additions and 18 deletions.
diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml
@@ -6,6 +6,10 @@ on:
       maven_opts:
         type: string
         required: false
+      build_folder:
+        type: string
+        required: false
+        default: "build-folder"
     outputs:
       version:
         description: "Project version"
@@ -34,10 +38,10 @@ jobs:
             ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
           fi
       - name: Maven Build (skip tests)
-        run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }}
+        run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress
       - uses: actions/upload-artifact@v4
         with:
-          name: build-folder
+          name: ${{ inputs.build_folder }}
           path: build
       - id: get_project_version
         name: Get project version

diff --git a/.github/workflows/deploy-docker-hub-workflow.yml b/.github/workflows/deploy-docker-hub-workflow.yml
@@ -12,6 +12,10 @@ on:
       branch:
         type: string
         required: false
+      build_folder:
+        type: string
+        required: false
+        default: "build-folder"
     secrets:
       DOCKER_HUB_USER:
         required: true
@@ -31,7 +35,7 @@ jobs:
           ref: ${{ inputs.branch }}
       - uses: actions/download-artifact@v4
         with:
-          name: build-folder
+          name: ${{ inputs.build_folder }}
           path: build
       - name: Chmod
         run: "find build -regex '.*sh' | while read file ; do chmod u+x $file ; done"

diff --git a/.github/workflows/deploy-maven-repository-workflow.yml b/.github/workflows/deploy-maven-repository-workflow.yml
@@ -37,17 +37,17 @@ jobs:
           gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}   # Value of the GPG private key to import
           gpg-passphrase: MAVEN_GPG_PASSPHRASE                    # env variable for GPG private key passphrase
       - name: Deploy to Maven Central repository
-        run: mvn clean deploy -DskipTests -P deploy-maven ${{ inputs.maven_opts }}
+        run: mvn clean deploy -DskipTests -P deploy-maven ${{ inputs.maven_opts }} --no-transfer-progress
         env:
-          MAVEN_NEXUS_USER: ${{ secrets.MAVEN_NEXUS_USER }}
-          MAVEN_NEXUS_PASSWORD: ${{ secrets.MAVEN_NEXUS_PASSWORD }}
+          MAVEN_NEXUS_USER: ${{ secrets.MAVEN_USER_TOKEN }}
+          MAVEN_NEXUS_PASSWORD: ${{ secrets.MAVEN_PASSWORD_TOKEN }}
           MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
       - name: Set up Java for publishing to GitHub Packages
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
       - name: Deploy to GitHub Packages repository
-        run: mvn clean deploy -DskipTests -P deploy-github ${{ inputs.maven_opts }}
+        run: mvn clean deploy -DskipTests -P deploy-github ${{ inputs.maven_opts }} --no-transfer-progress
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/manual-deploy-maven.yml b/.github/workflows/manual-deploy-maven.yml
@@ -0,0 +1,23 @@
+name: Manual maven deploy workflow
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch, tag or SHA of the source code to build docker."
+        type: string
+        required: true
+jobs:
+  checkout:
+    name: Build Java app
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ref: "${{ inputs.branch }}"
+
+  deploy-maven:
+    uses: ./.github/workflows/deploy-maven-repository-workflow.yml
+    needs: checkout
+    secrets: inherit
diff --git a/.github/workflows/publish-test-report-workflow.yml b/.github/workflows/publish-test-report-workflow.yml
@@ -0,0 +1,39 @@
+name: Reusable workflow to push in DockerHub
+
+on:
+  workflow_call:
+    inputs:
+      target_path:
+        type: string
+        required: true
+      content:
+        type: string
+        required: false
+    secrets:
+      SSH_TESTING_SERVER_HOST:
+        required: true
+      SSH_TESTING_SERVER_PASSWORD:
+        required: true
+      SSH_TESTING_SERVER_PORT:
+        required: true
+      SSH_TESTING_SERVER_USER:
+        required: true
+
+jobs:
+  publish-test-report:
+    name: Publish test report
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.content }}
+          path: report
+      - name: Deploy tests web recursively to remote
+        run: |
+          set -e
+          set -x
+          export SSHPASS=${{ secrets.SSH_TESTING_SERVER_PASSWORD }}
+          SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=60"
+          sshpass -e ssh ${SSH_OPTS} ${{ secrets.SSH_TESTING_SERVER_USER }}@${{ secrets.SSH_TESTING_SERVER_HOST }} -p ${{ secrets.SSH_TESTING_SERVER_PORT }} "mkdir -p ${{ inputs.target_path }}"
+          echo "Copying report tests to remote path ${{ inputs.target_path }}"
+          sshpass -e scp ${SSH_OPTS} -r -P ${{ secrets.SSH_TESTING_SERVER_PORT }} report ${{ secrets.SSH_TESTING_SERVER_USER }}@${{ secrets.SSH_TESTING_SERVER_HOST }}:${{ inputs.target_path }}
diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml
@@ -0,0 +1,16 @@
+name: Pull request approve workflow
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
+
+on:
+  pull_request_review:
+    types: [ submitted ]
+
+jobs:
+  build:
+    uses: ./.github/workflows/build-java-app-workflow.yml
+
+  test:
+    name: "Run all tests before merging"
+    uses: ./.github/workflows/test-analysis.yml
+    needs: build
+    secrets: inherit
diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -29,7 +29,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: mvn -B verify surefire-report:report --fail-never org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_java-common-libs
+        run: mvn -B verify surefire-report:report --fail-never org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_java-common-libs --no-transfer-progress
       - name: Publish Test Report
         uses: scacap/action-surefire-report@v1
         ## Skip cancelled()

diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml
@@ -0,0 +1,108 @@
+name: TEST Xetabase and publish report workflow
+
+on:
+  workflow_call:
+    inputs:
+      task:
+        type: string
+        required: true
+      branch:
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      task:
+        type: string
+        description: 'Task ID to be tested.'
+        required: true
+      branch:
+        type: string
+        description: 'Branch of opencga-enterprise to be tested and built.'
+        required: true
+env:
+  AZCOPY_SPA_CLIENT_SECRET: ${{ secrets.AZCOPY_SPA_CLIENT_SECRET }}
+  AZCOPY_AUTO_LOGIN_TYPE: "SPN"
+  AZCOPY_SPA_APPLICATION_ID: ${{ secrets.AZCOPY_SPA_APPLICATION_ID }}
+  AZCOPY_TENANT_ID: ${{ secrets.AZCOPY_TENANT_ID }}
+
+jobs:
+  test:
+    name: Execute JUnit and Jacoco tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}'
+        uses: actions/checkout@v4
+        with:
+          repository: zetta-genomics/opencga-enterprise
+          ref: ${{ github.event.inputs.branch }}
+          token: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
+          path: opencga-enterprise
+          fetch-depth: "10"
+      - id: get_opencga_branch
+        name: Get OpenCGA branch from 'pom.xml' property
+        run: |
+          pwd
+          chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh
+          opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ github.event.inputs.task }})
+          echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+        with:
+          repository: opencb/opencga
+          ref: ${{ steps.get_opencga_branch.outputs.opencga_branch }}
+          path: opencga
+          fetch-depth: '10'
+      - name: Set up JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Install Samtools
+        run: sudo apt-get install samtools python3-deeptools
+      - name: Start MongoDB v6.0
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: 6.0
+          mongodb-replica-set: rs-test
+      - name: K8s Tunnel MongoDB
+        run: |
+          wget https://dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl
+          chmod +x ./kubectl
+          echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf
+          ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf &
+      - name: Install Azure AZCOPY
+        uses: kheiakiyama/install-azcopy-action@v1
+        with:
+          version: 'v10'
+      - name: DockerHub login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise
+        run: |
+          ln -s opencga opencga-enterprise/opencga-home
+          cd opencga-enterprise
+          ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ github.event.inputs.task }} -c localhost:27018 -H hdp3.1
+      - name: Upload reports results to Github
+        uses: actions/upload-artifact@v4
+        with:
+          name: report-test
+          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/reports/test
+      - name: Upload log
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-log
+          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/build.log
+      - name: Upload junit reports to a remote scp server
+        uses: garygrossgarten/github-action-scp@release
+        with:
+          local: opencga-enterprise/reports/test
+          remote: /var/www/html/reports/xetabase/${{ github.event.inputs.task }}/
+          host: ${{ secrets.SSH_TESTING_SERVER_HOST}}
+          port: ${{ secrets.SSH_TESTING_SERVER_PORT}}
+          username: ${{ secrets.SSH_TESTING_SERVER_USER }}
+          password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }}
+          concurrency: 2
+
+
diff --git a/.github/workflows/testkeeper.yml b/.github/workflows/testkeeper.yml
@@ -0,0 +1,21 @@
+name: Test Keeper Secrets integration with Github Actions
+
+on:
+  workflow_dispatch:
+
+jobs:
+  buildexecutable:
+    runs-on: ubuntu-22.04
+    name: Build with Keeper secrets
+    steps:
+      - name: Retrieve secrets from Keeper
+        id: ksecrets
+        uses: Keeper-Security/ksm-action@master
+        with:
+          keeper-secret-config: ${{ secrets.KEEPER_SM_GH_OPENCB }}
+          secrets: |
+            AZURE_KUBE_CONFIG/field/Secret Value > env:KEEPERTOKEN
+
+      - name: Print Login & Password & URL
+        run: |
+          echo "Login is ${{ env.KEEPERTOKEN }}"
diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.1.0</version>
+        <version>5.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.1.0</version>
+        <version>5.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.1.0</version>
+        <version>5.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.1.0</version>
+        <version>5.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.1.0</version>
+        <version>5.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-lib/src/main/java/org/opencb/commons/exec/Command.java b/commons-lib/src/main/java/org/opencb/commons/exec/Command.java
@@ -76,8 +76,6 @@ public Command(String[] cmdArray, Map<String, String> environment) {
     @Override
     public void run() {
         try {
-            setStatus(Status.RUNNING);
-
             startTime();
             logger.debug(Commandline.describeCommand(cmdArray));
             if (environment != null && environment.size() > 0) {
@@ -92,6 +90,7 @@ public void run() {
             } else {
                 proc = Runtime.getRuntime().exec(cmdArray);
             }
+            setStatus(Status.RUNNING);
 
             InputStream is = proc.getInputStream();
             // Thread out = readOutputStream(is);
@@ -134,7 +133,10 @@ public void run() {
 
     @Override
     public void destroy() {
-        proc.destroy();
+        if (proc != null) {
+            proc.destroy();
+            setStatus(Status.KILLED);
+        }
     }
 
     private Thread readOutputStream(InputStream ins) throws IOException {
@@ -182,7 +184,7 @@ private Thread readStream(String outputName, OutputStream outputStream, StringBu
                     }
                     logger.debug("Read {} - Exit while", outputName);
                 } catch (Exception ex) {
-                    ex.printStackTrace();
+                    logger.error("Error reading " + outputName, ex);
                     exception = ex.toString();
                 }
         }, outputName + "_reader");

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>org.opencb.commons</groupId>
     <artifactId>commons</artifactId>
-    <version>5.1.0</version>
+    <version>5.2.0</version>
     <packaging>pom</packaging>
 
     <name>OpenCB commons project</name>