关于docker:使用-Docker-搭建-Hadoop-集群

创立容器

# 创立自定义网络
docker network create --subnet=172.18.0.0/16 mynetwork

# 启动容器
# cluster-master
docker run -itd --privileged  --name cluster-master -h cluster-master -p 18088:18088 -p 9870:9870 --net mynetwork --ip 172.18.0.2 centos:latest /usr/sbin/init
# cluster-slaves
docker run -itd --privileged --name cluster-slave1 -h cluster-slave1 --net mynetwork --ip 172.18.0.3 centos:latest /usr/sbin/init
docker run -itd --privileged --name cluster-slave2 -h cluster-slave2 --net mynetwork --ip 172.18.0.4 centos:latest /usr/sbin/init
docker run -itd --privileged --name cluster-slave3 -h cluster-slave3 --net mynetwork --ip 172.18.0.5 centos:latest /usr/sbin/init
# docker run -itd --privileged -v /System/Volumes/Data/data/personal/docker/hadoop/sys/fs/cgroup:/sys/fs/cgroup --name cluster-slave4 -h cluster-slave4 --net mynetwork --ip 172.18.0.6 centos:latest /usr/sbin/init

装置 OpenSSH 免密登录

master 装置 openSSH

# 进入 master 容器中
docker exec -it cluster-master /bin/bash
# 在容器外部执行
yum -y install openssh openssh-server openssh-clients vim

# 去掉文件 Port 22  PermitRootLogin(开启近程登录) yes  PasswordAuthentication(应用明码) yes
vim /etc/ssh/sshd_config

# 启动 openSSH
systemctl start sshd

# 生成 ssh key
ssh-keygen -t rsa # 一路回车就好

cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys

slave 装置 openSSH（slave1 slave2 slave3 同理装置）

# 进入 master 容器中
docker exec -it cluster-slave1 /bin/bash
# 在容器外部执行
yum -y install openssh openssh-server openssh-clients vim

# 去掉文件 Port 22  PermitRootLogin(开启近程登录) yes  PasswordAuthentication(应用明码) yes
vim /etc/ssh/sshd_config

# 启动 openSSH
systemctl start sshd

# 应用 master 的 ~/.ssh/* 替换 slave1 的~/.ssh/*

systemctl start sshd

Ansible 装置

yum -y install epel-release
yum -y install ansible

# 批改
vi /etc/ansible/hosts
`
[cluster]
cluster-master
cluster-slave1
cluster-slave2
cluster-slave3

[master]
cluster-master

[slaves]
cluster-slave1
cluster-slave2
cluster-slave3
`

批改 hosts

:>/etc/hosts
cat >>/etc/hosts<<EOF
127.0.0.1   localhost
172.18.0.2  cluster-master
172.18.0.3  cluster-slave1
172.18.0.4  cluster-slave2
172.18.0.5  cluster-slave3
EOF

source ~/.bashrc

# 散发至 slave
ansible cluster -m copy -a "src=~/.bashrc dest=~/"

装置 jdk 和 hodoop（四台机器都要装置）

# 将宿主机 jdk 文件和 hadoop 文件挪动至 docker 容器
docker cp ~/Download/hadoop-3.3.0.tar.gz  容器 ID 或容器名称:/opt
docker cp ~/Download/jdk-8u261-linux-x64.tar.gz 容器 ID 或容器名称:/opt

# 将文件挪动至 slave
scp ./* root@cluster-slave1:/opt/
scp ./* root@cluster-slave2:/opt/
scp ./* root@cluster-slave3:/opt/

# 解压
tar -zxvf hadoop-3.3.0.tar.gz
tar -zxvf jdk-8u261-linux-x64.tar.gz

# 配置环境变量 
vim ~/.bashrc
`
# hadoop
export HADOOP_HOME=/opt/hadoop-3.3.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

#java
export JAVA_HOME=/opt/jdk1.8.0_261
export PATH=$JAVA_HOME/bin:$PATH
`
source ~/.bashrc

配置 hadoop 运行所需配置文件