diff --git a/docker-deploy/training_template/docker-compose-spark-slim.yml b/docker-deploy/training_template/docker-compose-spark-slim.yml index 228e3d321..2fadb725d 100644 --- a/docker-deploy/training_template/docker-compose-spark-slim.yml +++ b/docker-deploy/training_template/docker-compose-spark-slim.yml @@ -62,7 +62,7 @@ services: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml diff --git a/docker-deploy/training_template/docker-compose-spark.yml b/docker-deploy/training_template/docker-compose-spark.yml index 2f9f0c8b5..cef47153c 100644 --- a/docker-deploy/training_template/docker-compose-spark.yml +++ b/docker-deploy/training_template/docker-compose-spark.yml @@ -61,7 +61,7 @@ services: - 9380:9380 - 9360:9360 volumes: - - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.2-bin-hadoop3.2/conf/spark-defaults.conf + - ./confs/spark/spark-defaults.conf:/data/projects/spark-3.1.3-bin-hadoop3.2/conf/spark-defaults.conf - ./confs/fate_flow/conf/service_conf.yaml:/data/projects/fate/conf/service_conf.yaml - ./confs/fate_flow/conf/pulsar_route_table.yaml:/data/projects/fate/conf/pulsar_route_table.yaml - ./confs/fate_flow/conf/rabbitmq_route_table.yaml:/data/projects/fate/conf/rabbitmq_route_table.yaml diff --git a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml index 7ec13f019..e6893425d 100644 --- a/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml +++ b/docker-deploy/training_template/public/fate_flow/conf/service_conf.yaml @@ -76,7 +76,7 @@ fate_on_eggroll: fate_on_spark: spark: # default use SPARK_HOME environment variable - home: /data/projects/spark-3.1.2-bin-hadoop3.2/ + home: /data/projects/spark-3.1.3-bin-hadoop3.2/ cores_per_node: 20 nodes: 2 linkis_spark: diff --git a/docs/FATE_On_Spark.md b/docs/FATE_On_Spark.md index d83c59621..7c7c7c56d 100644 --- a/docs/FATE_On_Spark.md +++ b/docs/FATE_On_Spark.md @@ -11,7 +11,7 @@ As the above figure show, the EggRoll provide both computing and storage resourc Since FATE v1.5.0 a user can select Spark as the underlying computing engine, however, spark itself is an in-memory computing engine without the data persistence. Thus, HDFS is also needed to be deployed to help on data persistence. For example, a user need to upload their data to HDFS through FATE before doing any training job, and the output data of each component will also be stored in the HDFS module. -**Currently the verifed Spark version is [3.1.2](https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz) and the Hadoop is [3.2.1](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz)** +**Currently the verifed Spark version is [3.1.3](https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop3.2.tgz) and the Hadoop is [3.2.1](https://archive.apache.org/dist/hadoop/common/hadoop-3.2.1/hadoop-3.2.1.tar.gz)** The following picture shows the architecture of FATE on Spark: