.pipeline.yml

pipeline:
  create_cluster:
    image: banzaicloud/plugin-pipeline-client:dev-b5e2e54
    cluster_name: "lp-org-03"
    cluster_provider: "google"
    google_project: "puski-test-194413"
    google_node_count: "1"
    google_gke_version: "1.9.4-gke.1"
    log_level: debug
    cluster_state: deleted

    secrets: [plugin_endpoint, plugin_token, plugin_secret_id]

  install_monitoring:
    image: banzaicloud/plugin-pipeline-client:dev-b5e2e54
    deployment_name: "banzaicloud-stable/pipeline-cluster-monitor"
    deployment_release_name: "monitor"

    secrets: [plugin_endpoint, plugin_token, plugin_secret_id]

  install_spark_resources:
    image: banzaicloud/plugin-pipeline-client:dev-b5e2e54

    deployment_name: "banzaicloud-stable/spark"
    deployment_release_name: "release-1"
    deployment_values:
     historyServer:
        enabled: true
     spark-hs:
       app:
         logDirectory: "gs://puski-testing/pdidata"

    secrets: [plugin_endpoint, plugin_token, plugin_secret_id]

  remote_checkout:
    image: banzaicloud/plugin-k8s-proxy:0.3.0
    original_image: plugins/git

  remote_build:
    image: banzaicloud/plugin-k8s-proxy:0.3.0
    original_image: maven:3.5-jdk-8
    original_commands:
      - mvn clean package

  run:
    image: banzaicloud/plugin-k8s-proxy:0.3.0
    original_image: banzaicloud/plugin-spark-submit-k8s:0.3.0
    proxy_service_account: spark

    spark_submit_options:
      class: banzaicloud.SparkPi
      kubernetes-namespace: default
    spark_submit_configs:
      spark.app.name: sparkpi
      spark.local.dir: /tmp/spark-locals
      spark.kubernetes.driver.docker.image: banzaicloud/spark-driver:v2.2.1-k8s-1.0.11
      spark.kubernetes.executor.docker.image: banzaicloud/spark-executor:v2.2.1-k8s-1.0.11
      spark.kubernetes.initcontainer.docker.image: banzaicloud/spark-init:v2.2.1-k8s-1.0.11
      spark.dynamicAllocation.enabled: "true"
      spark.kubernetes.resourceStagingServer.uri: http://spark-rss:10000
      spark.kubernetes.resourceStagingServer.internal.uri: http://spark-rss:10000
      spark.shuffle.service.enabled: "true"
      spark.kubernetes.shuffle.namespace: default
      spark.kubernetes.shuffle.labels: app=spark-shuffle-service,spark-version=2.2.0
      spark.kubernetes.authenticate.driver.serviceAccountName: spark
      spark.metrics.conf: /opt/spark/conf/metrics.properties
      spark.eventLog.enabled: "true"
      spark.eventLog.dir: "gs://puski-testing/pdidata"
    spark_submit_app_args:
      - target/spark-pi-1.0-SNAPSHOT.jar
      - 1000