CentOS 6 部署 CDH 5.7

1
setenforce 0
2

3
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
4
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux

1
chkconfig iptables off
2
service iptables stop
3

4
chkconfig ip6tables off
5
service ip6tables stop

1
echo 1 > /proc/sys/net/ipv6/conf/all/disable_ipv6
2
echo 1 > /proc/sys/net/ipv6/conf/default/disable_ipv6
3

4
cat >> /etc/sysctl.conf <<EOF
5
net.ipv6.conf.all.disable_ipv6 = 1
6
net.ipv6.conf.default.disable_ipv6 = 1
7
EOF

1
echo 1 > /proc/sys/vm/swappiness
2

3
cat >> /etc/sysctl.conf <<EOF
4
vm.swappiness = 1
5
EOF

1
echo never > /sys/kernel/mm/transparent_hugepage/defrag
2
echo never > /sys/kernel/mm/transparent_hugepage/enabled
3

4
cat >> /etc/rc.local <<EOF
5
echo never > /sys/kernel/mm/transparent_hugepage/defrag
6
echo never > /sys/kernel/mm/transparent_hugepage/enabled
7
EOF

1
cat >> /etc/pam.d/login <<EOF
2
session required pam_limits.so
3
EOF
4

5
cat >> /etc/security/limits.conf <<EOF
6
* soft nofile 65535
7
* hard nofile 65535
8
EOF
9

10
sed -i 's/#UsePAM no/UsePAM yes/g' /etc/ssh/sshd_config

1
cat > /etc/fstab <<EOF
2
/dev/sda1   swap        swap        defaults            0 0
3
/dev/sda2   /           ext4        defaults            1 1
4
/dev/sga1   /sga        ext4        defaults,noatime    0 0
5
/dev/sgb1   /sgb        ext4        defaults,noatime    0 0
6
/dev/sgc1   /sgc        ext4        defaults,noatime    0 0
7
/dev/sgd1   /sgd        ext4        defaults,noatime    0 0
8
/dev/sge1   /sge        ext4        defaults,noatime    0 0
9
/dev/sgf1   /sgf        ext4        defaults,noatime    0 0
10
tmpfs       /dev/shm    tmpfs       defaults            0 0
11
devpts      /dev/pts    devpts      gid=5,mode=620      0 0
12
sysfs       /sys        sysfs       defaults            0 0
13
proc        /proc       proc        defaults            0 0
14
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-m-001.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-m-002.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-m-003.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-001.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-002.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-003.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-004.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-005.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-006.bigdata.com
4
EOF

1
cat > /etc/sysconfig/network <<EOF
2
NETWORKING=yes
3
HOSTNAME=bigdata-w-007.bigdata.com
4
EOF

1
cat > /etc/hosts <<EOF
2
127.0.0.1       localhost
3
192.168.86.201  bigdata-m-001.bigdata.com bigdata-m-001
4
192.168.86.202  bigdata-m-002.bigdata.com bigdata-m-002
5
192.168.86.203  bigdata-m-003.bigdata.com bigdata-m-003
6
192.168.86.1    bigdata-w-001.bigdata.com bigdata-w-001
7
192.168.86.2    bigdata-w-002.bigdata.com bigdata-w-002
8
192.168.86.3    bigdata-w-003.bigdata.com bigdata-w-003
9
192.168.86.4    bigdata-w-004.bigdata.com bigdata-w-004
10
192.168.86.5    bigdata-w-005.bigdata.com bigdata-w-005
11
192.168.86.6    bigdata-w-006.bigdata.com bigdata-w-006
12
192.168.86.7    bigdata-w-007.bigdata.com bigdata-w-007
13
EOF

1
rm -f /etc/yum.repos.d/*
2

3
cat > /etc/yum.repos.d/centos.repo <<EOF
4
[CentOS]
5
name = CentOS
6
baseurl = http://bigdata-m-003.bigdata.com/CentOS-6.8-x86_64/
7
gpgkey = http://bigdata-m-003.bigdata.com/CentOS-6.8-x86_64/RPM-GPG-KEY-CentOS-6
8
gpgcheck = 1
9
enabled = 1
10
EOF
11

12
cat > /etc/yum.repos.d/cloudera-manager.repo <<EOF
13
[Cloudera-Manager]
14
name = Cloudera-Manager
15
baseurl = http://bigdata-m-003.bigdata.com/Cloudera/CM/5.7/
16
gpgkey = http://bigdata-m-003.bigdata.com/Cloudera/CM/5.7/RPM-GPG-KEY-cloudera
17
gpgcheck = 1
18
enabled = 1
19
EOF

1
cd /data && nohup python -m SimpleHTTPServer 80 > /dev/null 2>&1 &
2

3
yum -y install httpd
4

5
kill -9 `netstat -antp | grep "0 0.0.0.0:80" | awk -F" " '{print $7}' | cut -d "/" -f 1`
6

7
chkconfig httpd on
8
service httpd start
9

10
ln -s /data/CentOS-6.8-x86_64 /var/www/html/CentOS-6.8-x86_64
11
ln -s /data/Cloudera /var/www/html/Cloudera
12
ln -s /data/JDBC /var/www/html/JDBC
13
ln -s /data/JDK /var/www/html/JDK

1
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

1
yum -y install ntp ntpdate
2

3
cat > /etc/ntp.conf <<EOF
4
driftfile /var/lib/ntp/drift
5
restrict default kod nomodify notrap nopeer noquery
6
restrict -6 default kod nomodify notrap nopeer noquery
7
restrict 127.0.0.1
8
restrict -6 ::1
9
server 192.168.86.253
10
includefile /etc/ntp/crypto/pw
11
keys /etc/ntp/keys
12
EOF
13

14
chkconfig ntpd on
15
service ntpd start
16

17
ntpq -p

1
yum -y install wget unzip
2

3
wget http://bigdata-m-003.bigdata.com/JDK/jdk-8u74-linux-x64.rpm
4
wget http://bigdata-m-003.bigdata.com/JDK/jce_policy-8.zip
5

6
rpm -ivh jdk-8u74-linux-x64.rpm
7

8
unzip jce_policy-8.zip
9
cp UnlimitedJCEPolicyJDK8/local_policy.jar /usr/java/jdk1.8.0_74/jre/lib/security
10
cp UnlimitedJCEPolicyJDK8/US_export_policy.jar /usr/java/jdk1.8.0_74/jre/lib/security
11

12
alternatives --install /usr/bin/java java /usr/java/jdk1.8.0_74/jre/bin/java 9999
13

14
cat > /etc/profile.d/java_home.sh <<EOF
15
export JAVA_HOME="/usr/java/jdk1.8.0_74"
16
EOF

1
ssh-keygen -t rsa -b 8192 -N ''
2
ssh-copy-id -i root@bigdata-m-001.bigdata.com
3
ssh-copy-id -i root@bigdata-m-002.bigdata.com
4
ssh-copy-id -i root@bigdata-m-003.bigdata.com
5
ssh-copy-id -i root@bigdata-w-001.bigdata.com
6
ssh-copy-id -i root@bigdata-w-002.bigdata.com
7
ssh-copy-id -i root@bigdata-w-003.bigdata.com
8
ssh-copy-id -i root@bigdata-w-004.bigdata.com
9
ssh-copy-id -i root@bigdata-w-005.bigdata.com
10
ssh-copy-id -i root@bigdata-w-006.bigdata.com
11
ssh-copy-id -i root@bigdata-w-007.bigdata.com

1
yum -y install mysql mysql-server
2

3
chkconfig mysqld on
4
service mysqld start
5

6
mysql_secure_installation
7

8
mysql -uroot -p <<EOF
9
create database scm default charset utf8 collate utf8_general_ci;
10
grant all privileges on scm.* to 'scm_db_usr'@'%' identified by '__scm_db_pwd__';
11
create database hive default charset utf8 collate utf8_general_ci;
12
grant all privileges on hive.* to 'hive_db_usr'@'%' identified by '__hive_db_pwd__';
13
create database hue default charset utf8 collate utf8_general_ci;
14
grant all privileges on hue.* to 'hue_db_usr'@'%' identified by '__hue_db_pwd__';
15
create database oozie default charset utf8 collate utf8_general_ci;
16
grant all privileges on oozie.* to 'oozie_db_usr'@'%' identified by '__oozie_db_pwd__';
17
flush privileges;
18
EOF

1
mkdir -p /usr/share/java
2
wget -O /usr/share/java/mysql-connector-java.jar http://bigdata-m-003.bigdata.com/JDBC/MySQL/mysql-connector-java-5.1.40-bin.jar
3

4
yum -y install cloudera-manager-server
5

6
/usr/share/cmf/schema/scm_prepare_database.sh --host bigdata-m-003.bigdata.com --port 3306 mysql scm scm_db_usr __scm_db_pwd__
7

8
chkconfig cloudera-scm-server on
9
service cloudera-scm-server start