Run SI tests on OSS DC/OS cluster with latest Marathon.

Summary:
Create a cluster for open DC/OS based on the PR that has the latest
Marathon version. So far this is hard coded for open until we enable a
job for enterprise as well.

Test Plan:
See
https://jenkins.mesosphere.com/service/jenkins/view/Marathon/job/System%20Integration%20Tests/job/marathon-si-karsten/

Reviewers: unterstein, zen-dog, ichernetsky, timcharper, kensipe, jenkins

Reviewed By: ichernetsky, kensipe, jenkins

Subscribers: marathon-team, marathon-dev, jenkins

JIRA Issues: MARATHON-7581

Differential Revision: https://phabricator.mesosphere.com/D978

Jenkinsfile.si

@@ -4,50 +4,20 @@ timeout(90) {
 node('shakedown') {
   wrap([$class: 'MesosSingleUseSlave']) {
   wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
-    def timeout=360000
-
-    properties([
-      parameters([
-        string(name: 'dcos_url',
-          defaultValue: '',
-          description: 'URL to DC/OS Enterprise Cluster'
-        )]
-      )
-    ])
-
-    stage('Checkout Repo') {
-      checkout scm
-    }
-
-    stage('Run System Integration Tests'){
-      withCredentials(
-        [ [$class: 'FileBinding',
-          credentialsId: '11fcc957-5156-4470-ae34-d433da88248a',
-          variable: 'DOT_SHAKEDOWN'],
-          [$class: 'FileBinding',
-          credentialsId: '23743034-1ac4-49f7-b2e6-a661aee2d11b',
-          variable: 'CLI_TEST_SSH_KEY'],
-          [$class: 'StringBinding',
-          credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d',
-          variable: 'DOCKER_HUB_USERNAME'],
-          [$class: 'StringBinding',
-          credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825',
-          variable: 'DOCKER_HUB_PASSWORD']
-        ]) {
-          try {
-            sh "cp -f \"${DOT_SHAKEDOWN}\" ~/.shakedown"
-            sh """
-              TERM=velocity shakedown \
-                --stdout all \
-                --stdout-inline \
-                --timeout ${timeout} \
-                --pytest-option "--junitxml=shakedown.xml" \
-                --ssh-key-file \"${CLI_TEST_SSH_KEY}\" \
-                --dcos-url $params.dcos_url tests/system/test_marathon_root.py tests/system/test_marathon_universe.py
-               """
-          } finally {
-            junit allowEmptyResults: true, testResults: 'shakedown.xml'
+    stage("Run System Integration Test Pipeline") {
+      try {
+        checkout scm
+        withCredentials(
+          [ [$class: 'FileBinding', credentialsId: '23743034-1ac4-49f7-b2e6-a661aee2d11b', variable: 'CLI_TEST_SSH_KEY'],
+            [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'AWS_ACCESS_KEY_ID', credentialsId: '7155bd15-767d-4ae3-a375-e0d74c90a2c4', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
+            [$class: 'FileBinding', credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'],
+            [$class: 'StringBinding', credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'],
+            [$class: 'StringBinding', credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD']
+          ]) {
+            sh """ci/system_integration.sh"""
           }
+        } finally {
+            junit allowEmptyResults: true, testResults: 'shakedown.xml'
         }
     }
   }

ci/launch_cluster.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+# Hardcode configuration to open DC/OS.
+VARIANT="open"
+CHANNEL="testing/pull/1739"
+
+JOB_NAME_SANITIZED=$(echo "$JOB_NAME" | tr -c '[:alnum:]-' '-')
+DEPLOYMENT_NAME="$JOB_NAME_SANITIZED-$(date +%s)"
+
+if [ "$VARIANT" == "open" ]; then
+  TEMPLATE="https://s3.amazonaws.com/downloads.dcos.io/dcos/${CHANNEL}/cloudformation/single-master.cloudformation.json"
+else
+  TEMPLATE="https://s3.amazonaws.com/downloads.mesosphere.io/dcos-enterprise-aws-advanced/${CHANNEL}/${VARIANT}/cloudformation/ee.single-master.cloudformation.json"
+fi
+
+echo "Workspace: ${WORKSPACE}"
+echo "Using: ${TEMPLATE}"
+
+apk --upgrade add gettext wget
+wget 'https://downloads.dcos.io/dcos-test-utils/bin/linux/dcos-launch' && chmod +x dcos-launch
+
+
+envsubst <<EOF > config.yaml
+---
+launch_config_version: 1
+template_url: $TEMPLATE
+deployment_name: $DEPLOYMENT_NAME
+provider: aws
+aws_region: eu-central-1
+template_parameters:
+    KeyName: default
+    AdminLocation: 0.0.0.0/0
+    PublicSlaveInstanceCount: 1
+    SlaveInstanceCount: 5
+EOF
+./dcos-launch create
+./dcos-launch wait

ci/system_integration.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+./ci/launch_cluster.sh
+
+DCOS_URL="http://$(./dcos-launch describe | jq -r ".masters[0].public_ip")/"
+cp -f "$DOT_SHAKEDOWN" "$HOME/.shakedown"
+
+TERM=velocity shakedown \
+  --stdout all \
+  --stdout-inline \
+  --timeout 360000 \
+  --pytest-option "--junitxml=shakedown.xml" \
+  --ssh-key-file "$CLI_TEST_SSH_KEY" \
+  --dcos-url "$DCOS_URL" tests/system/test_marathon_root.py tests/system/test_marathon_universe.py
+