Skip to content
This repository has been archived by the owner on Nov 23, 2017. It is now read-only.

add java 8 to image creation script; remove duplication #12

Open
wants to merge 1 commit into
base: branch-1.5
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions create_image.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
#!/bin/bash
# Creates an AMI for the Spark EC2 scripts starting with a stock Amazon
# Linux AMI.
# This has only been tested with Amazon Linux AMI 2014.03.2
# This has only been tested with Amazon Linux AMI 2014.03.2

JAVA_VERSION=1.8.0
HADOOP_VERSION=2.6.0
MAVEN_VERSION=3.3.3

set -e

Expand All @@ -11,19 +15,19 @@ if [ "$(id -u)" != "0" ]; then
fi

# Dev tools
sudo yum install -y java-1.7.0-openjdk-devel gcc gcc-c++ ant git
sudo yum install -y java-$JAVA_VERSION-openjdk-devel gcc gcc-c++ ant git
# Perf tools
sudo yum install -y dstat iotop strace sysstat htop perf
sudo debuginfo-install -q -y glibc
sudo debuginfo-install -q -y kernel
sudo yum --enablerepo='*-debug*' install -q -y java-1.7.0-openjdk-debuginfo.x86_64
sudo yum --enablerepo='*-debug*' install -q -y java-$JAVA_VERSION-openjdk-debuginfo.x86_64

# PySpark and MLlib deps
sudo yum install -y python-matplotlib python-tornado scipy libgfortran
# SparkR deps
sudo yum install -y R
# Other handy tools
sudo yum install -y pssh
sudo yum install -y pssh tmux tree
# Ganglia
sudo yum install -y ganglia ganglia-web ganglia-gmond ganglia-gmetad

Expand All @@ -48,14 +52,14 @@ done

# Install Maven (for Hadoop)
cd /tmp
wget "http://archive.apache.org/dist/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.tar.gz"
tar xvzf apache-maven-3.2.3-bin.tar.gz
mv apache-maven-3.2.3 /opt/
wget "http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz"
tar xvzf apache-maven-$MAVEN_VERSION-bin.tar.gz
mv apache-maven-$MAVEN_VERSION /opt/

# Edit bash profile
echo "export PS1=\"\\u@\\h \\W]\\$ \"" >> ~/.bash_profile
echo "export JAVA_HOME=/usr/lib/jvm/java-1.7.0" >> ~/.bash_profile
echo "export M2_HOME=/opt/apache-maven-3.2.3" >> ~/.bash_profile
echo "export JAVA_HOME=/usr/lib/jvm/java-$JAVA_VERSION" >> ~/.bash_profile
echo "export M2_HOME=/opt/apache-maven-$MAVEN_VERSION" >> ~/.bash_profile
echo "export PATH=\$PATH:\$M2_HOME/bin" >> ~/.bash_profile

source ~/.bash_profile
Expand All @@ -64,11 +68,12 @@ source ~/.bash_profile
sudo mkdir /root/hadoop-native
cd /tmp
sudo yum install -y protobuf-compiler cmake openssl-devel
wget "http://archive.apache.org/dist/hadoop/common/hadoop-2.4.1/hadoop-2.4.1-src.tar.gz"
tar xvzf hadoop-2.4.1-src.tar.gz
cd hadoop-2.4.1-src
mvn package -Pdist,native -DskipTests -Dtar
sudo mv hadoop-dist/target/hadoop-2.4.1/lib/native/* /root/hadoop-native
wget "http://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION-src.tar.gz"
tar xvzf hadoop-$HADOOP_VERSION-src.tar.gz
cd hadoop-$HADOOP_VERSION-src
mvn package -Pdist,native -DskipTests -Dmaven.javadoc.skip=true -Dtar
sudo mv hadoop-dist/target/hadoop-$HADOOP_VERSION/lib/native/* /root/hadoop-native
rm -rf ~/.m2

# Install Snappy lib (for Hadoop)
yum install -y snappy
Expand Down