Hadoop和spark集群搭建详解

[SeatDefaults]

greeter-session=unity-greeter

user-session=Ubuntu

greeter-show-manual-login=true

127.0.0.1 localhost

192.168.32.131 SparkMaster

192.168.32.132 SparkWorker1

192.168.32.133 SparkWorker2

# The following lines are desirable for IPv6 capable hosts

::1 ip6-localhost ip6-loopback

fe00::0 ip6-localnet

ff00::0 ip6-mcastprefix

ff02::1 ip6-allnodes

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

scp ~/.ssh/id_rsa.pub root@SparkMaster:~/.ssh/id_rsa.pub.SparkWorker1

cat id_rsa.pub.SparkWorker1 >> authorized_keys

cat id_rsa.pub.SparkWorker2 >> authorized_keys

scp authorized_keys root@SparkWorker1:~/.ssh/authorized_keys

ssh SparkMaster

ssh SparkWorker1

mkdir /urs/lib/java

cd /urs/lib/java

tar -zxvf jdk-8u25-linux-i586.tar.gz

gedit ~/.bashrc

#JAVA

export JAVA_HOME=/usr/lib/java/jdk1.8.0_25

export JRE_HOME=${JA VA_HOME}/jre

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.6.0

export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_INSTALL/lib/native

export HADOOP_OPTS="-Djava.library.path=$HADOOP_INSTALL/lib"

PATH=${IDEA_HOME}/bin:${SPARK_HOME}/bin:${SCALA_HOME}/bin:${HADOOP_HO

ME}/bin:${JA VA_HOME}/bin:$PA TH

scp -r /usr/lib/java/jdk1.8.0_25 root@SparkWorker2:~/usr/lib/java/

scp ~/.bashrc root@SparkWorker1:~/.bashrc

mkdir /urs/lib/hadoop

cd /urs/lib/hadoop

tar -zxvf hadoop-2.6.0.tar.gz

mkdir dfs

mkdir name

mkdir data

cd etc/hadoop

# The java implementation to use.

export JAVA_HOME=/usr/lib/java/jdk1.8.0_25

# some Java parameters

export JAVA_HOME=/usr/lib/java/jdk1.8.0_25

#echo "run java in $JA VA_HOME"

JAVA_HOME=$JA VA_HOME

fi

export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000

export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA

SparkWorker2

Licensed under the Apache License, Version 2.0 (the "License");

you may not use this file except in compliance with the License.

You may obtain a copy of the License at

distributed under the License is distributed on an "AS IS" BASIS,

WITHOUT W ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

<!-- Put site-specific property overrides in this file. -->

<configuration>

</configuration>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!--

Licensed under the Apache License, Version 2.0 (the "License");

You may obtain a copy of the License at

Unless required by applicable law or agreed to in writing, software