From f5af31aaeab6368a65790a1d133e6726ed4c5750 Mon Sep 17 00:00:00 2001 From: Chen Jing Date: Thu, 1 Sep 2022 13:58:10 +0800 Subject: [PATCH] Change the spark version to 3.1.3 (#743) Signed-off-by: Chen Jing Signed-off-by: Chen Jing --- .../training_template/docker-compose-spark-slim.yml | 2 +- docker-deploy/training_template/docker-compose-spark.yml | 2 +- .../public/fate_flow/conf/service_conf.yaml | 2 +- docs/FATE_On_Spark.md | 2 +- helm-charts/FATE/templates/core/fateflow/configmap.yaml | 2 +- helm-charts/FATE/templates/core/python-spark.yaml | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-deploy/training_template/docker-compose-spark-slim.yml b/docker-deploy/training_template/docker-compose-spark-slim.yml index 228e3d321..2fadb725d 100644 --- a/docker-deploy/training_template/docker-compose-spark-slim.yml +++ b/docker-deploy/training_template/docker-compose-spark-slim.yml @@ -62,7 +62,7 @@ services: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml diff --git a/docker-deploy/training_template/docker-compose-spark.yml b/docker-deploy/training_template/docker-compose-spark.yml index 2f9f0c8b5..cef47153c 100644 --- a/docker-deploy/training_template/docker-compose-spark.yml +++ b/docker-deploy/training_template/docker-compose-spark.yml @@ -61,7 +61,7 @@ services: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml diff --git a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml index 7ec13f019..e6893425d 100644 --- a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml +++ b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml @@ -76,7 +76,7 @@ fate_on_eggroll: fate_on_spark: spark: # default use SPARK_HOME environment variable - home: /data/projects/spark-3.1.2-bin-hadoop3.2/ + home: /data/projects/spark-3.1.3-bin-hadoop3.2/ cores_per_node: 20 nodes: 2 linkis_spark: diff --git a/docs/FATE_On_Spark.md b/docs/FATE_On_Spark.md index d83c59621..7c7c7c56d 100644 --- a/docs/FATE_On_Spark.md +++ b/docs/FATE_On_Spark.md @@ -11,7 +11,7 @@ As the above figure show, the EggRoll provide both computing and storage resourc Since FATE v1.5.0 a user can select Spark as the underlying computing engine, however, spark itself is an in-memory computing engine without the data persistence. Thus, HDFS is also needed to be deployed to help on data persistence. For example, a user need to upload their data to HDFS through FATE before doing any training job, and the output data of each component will also be stored in the HDFS module. -**Currently the verifed Spark version is [3.1.2](https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz) and the Hadoop is [3.2.1](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz)** +**Currently the verifed Spark version is [3.1.3](https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz) and the Hadoop is [3.2.1](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz)** The following picture shows the architecture of FATE on Spark:
diff --git a/helm-charts/FATE/templates/core/fateflow/configmap.yaml b/helm-charts/FATE/templates/core/fateflow/configmap.yaml index e516e7680..298218dc4 100644 --- a/helm-charts/FATE/templates/core/fateflow/configmap.yaml +++ b/helm-charts/FATE/templates/core/fateflow/configmap.yaml @@ -128,7 +128,7 @@ data: fate_on_spark: spark: # default use SPARK_HOME environment variable - home: /data/projects/spark-3.1.2-bin-hadoop3.2/ + home: /data/projects/spark-3.1.3-bin-hadoop3.2/ cores_per_node: {{ .Values.modules.python.spark.cores_per_node }} nodes: {{ .Values.modules.python.spark.nodes }} linkis_spark: diff --git a/helm-charts/FATE/templates/core/python-spark.yaml b/helm-charts/FATE/templates/core/python-spark.yaml index 2664b9a30..ca14a14be 100644 --- a/helm-charts/FATE/templates/core/python-spark.yaml +++ b/helm-charts/FATE/templates/core/python-spark.yaml @@ -125,8 +125,8 @@ spec: # fix fateflow conf must use IP sed -i "s/host: fateflow/host: ${POD_IP}/g" /data/projects/fate/conf/service_conf.yaml - cp /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults-template.conf /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf - sed -i "s/fateflow/${POD_IP}/g" /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + cp /data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults-template.conf /data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf + sed -i "s/fateflow/${POD_IP}/g" /data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf sleep 5 && python fateflow/python/fate_flow/fate_flow_server.py livenessProbe: @@ -161,7 +161,7 @@ spec: subPath: logs - mountPath: /data/projects/fate/conf-tmp/ name: python-confs - - mountPath: /data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults-template.conf + - mountPath: /data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults-template.conf name: python-confs subPath: spark-defaults.conf {{- if eq .Values.federation "RabbitMQ" }}