自作サーバーでSparkクラスタをKubernetes上に構築してみた Running Apache Spark on Kubernetes in Homebrew junk Server
4クラスタを構築する最低ラインだと
CPU：i7 （8Thread）
MEM：16GB
で
CPU：2Thread
MEM：2GB
のような3台のクラスタが作れます。是非お試しあれ。
実行コマンドは下記に載せました。
※＜＞は全角になっているので半角に変換して下さい。
全ノード共通
sudo vi /etc/hosts
192.168.1.14 me01 me01.me
192.168.1.15 me02 me02.me
192.168.1.16 me03 me03.me
sudo vi /etc/ssh/sshd_config
PubkeyAuthentication yes
ssh-keygen
cat ~/.ssh/id_rsa.pub ＞＞ ~/.ssh/authorized_keys
chmod 700 ~/.ssh
chmod 600 ~/.ssh/authorized_keys
vi ~/.ssh/config
host *
StrictHostKeyChecking no
UserKnownHostsFile=/dev/null
chmod 600 ~/.ssh/config
eval $(ssh-agent)
ssh-add ~/.ssh/id_rsa
ssh -A me01.me
exit
sudo vi /etc/selinux/config
SELINUX=Disabled
sudo systemctl disable firewalld
sudo reboot
sudo su -
yum -y update
yum install -y docker
systemctl enable docker && systemctl start docker
swapoff -a
vi /etc/fstab
#/dev/mapper/centos-swap swap swap defaults 0 0
cat ＜＜EOF ＞ /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system
cat ＜＜EOF ＞ /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF
yum install -y kubelet kubeadm kubectl
systemctl enable kubelet.service && systemctl start kubelet.service
shutdown -h now
マスターのみ
sudo kubeadm init --pod-network-cidr=10.244.0.0/16
kubeadm join 192.168.1.14:6443 --token t1mc27.bw9ry9bnejuglaev --discovery-token-ca-cert-hash sha256:e11c0d0132ba4fa6a9628022039cb85f01cb74bd0d47d31901c049b04afa1d7e
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl get nodes
kubectl get pods --all-namespaces
sudo su -
docker info | grep -i cgroup
docker run -d -p 5000:5000 -v /var/lib/registry:/var/lib/registry --restart=always --name registry registry:2
docker ps
vi /etc/sysconfig/docker
#OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false'
OPTIONS='--insecure-registry me01.me:5000 --selinux-enabled --log-driver=journald --signature-verification=false'
ワーカー
sudo nmtui
sudo reboot
ssh -A me02.me
ssh -A me03.me
sudo su -
kubeadm join 192.168.1.14:6443 --token t1mc27.bw9ry9bnejuglaev --discovery-token-ca-cert-hash sha256:e11c0d0132ba4fa6a9628022039cb85f01cb74bd0d47d31901c049b04afa1d7e
vi /etc/sysconfig/docker
#OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false'
OPTIONS='--insecure-registry me01.me:5000 --selinux-enabled --log-driver=journald --signature-verification=false'
マスター
kubectl get nodes
sudo yum -y install java-1.8.0-openjdk
sudo yum -y install wget
wget http://ftp.yz.yamagata-u.ac.jp/pub/network/apache/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
gzip -cd spark-2.3.1-bin-hadoop2.7.tgz | tar xvf -
cd ~/spark-2.3.1-bin-hadoop2.7
sudo ./bin/docker-image-tool.sh -r me01.me:5000/myspark build
sudo ./bin/docker-image-tool.sh -r me01.me:5000/myspark push
sudo docker push me01.me:5000/myspark/spark:latest
kubectl create serviceaccount default
kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:default --namespace=default
bin/spark-submit \
--master k8s://https://me01.me:6443 \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
--conf spark.executor.instances=3 \
--conf spark.kubernetes.container.image=me01.me:5000/myspark/spark \
local:///opt/spark/examples/jars/spark-examples_2.11-2.3.1.jar
kubectl get pod
kubectl logs spark-pi-01b9a2b9a0c83d558bdc394ead9dce6c-driver | less
■参考にさせて頂いたHP
https://www.server-world.info/
https://qiita.com/ysakashita/items/1f87646ae804ba509d07
■公式HP
https://spark.apache.org/docs/2.3.0/running-on-kubernetes.html
https://kubernetes.io/