通过kubeadm在AWS上创建高可用k8s集群

#!/bin/bash

LOG=/etc/keepalived/keepalived.log

IP=$1

INSTANCE_ID=$2

VIP=10.3.3.88

ENI=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].NetworkInterfaceId')

echo $(date) >> ${LOG}

echo "Adding IP $IP to ENI $ENI success" >> ${LOG}

aws ec2 assign-private-ip-addresses --network-interface-id $ENI --private-ip-addresses $IP --allow-reassignment

echo "Adding IP $IP success1" >> ${LOG}

sudo ip address add $VIP/24 dev eth0 >> ${LOG}

echo "Adding IP $IP success2" >> ${LOG}

#!/bin/bash

IP=$1

INSTANCE_ID=$2

VIP=10.3.3.88

LOG=/etc/keepalived/keepalived.log

ENI=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].NetworkInterfaceId')

echo $(date) >> ${LOG}

echo "Removing IP $IP from ENI $ENI" >> ${LOG}

sudo aws ec2 unassign-private-ip-addresses --network-interface-id $ENI --private-ip-addresses $IP

sudo p address del $VIP/24 dev eth0 >> ${LOG}

echo "remove ip success" >> ${LOG}

在三台master节点都执行

  # 启动keepalived

$sudo systemctl start keepalived.service

设置开机启动

$sudo  systemctl enable keepalived.service

# 查看启动状态

 $sudo  systemctl status keepalived.service 

启动后查看master1的网卡信息

ip a s eth0  

sudo yum install -y haproxy

三台master节点的配置均相同，配置中声明了后端代理的两个master节点服务器，指定了haproxy运行的端口为16443等，因此16443端口为集群的入口

 cat > /etc/haproxy/haproxy.cfg << EOF

#---------------------------------------------------------------------

# Global settings

#---------------------------------------------------------------------

global

    # to have these messages end up in /var/log/haproxy.log you will

    # need to:

     # 1) configure syslog to accept network log events.  This is done

     #    by adding the '-r' option to the SYSLOGD_OPTIONS in

    #    /etc/sysconfig/syslog

    # 2) configure local2 events to go to the /var/log/haproxy.log

    #   file. A line like the following can be added to

    #   /etc/sysconfig/syslog

    #

    #    local2.*                       /var/log/haproxy.log

    #

    log /dev/log local0

    log /dev/log local1 notice

    daemon

    # turn on stats unix socket

    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------  

defaults

log global

mode http

option httplog

    retries                 3

    timeout http-request    10s

      timeout queue           1m

    timeout connect         10s

     timeout client          1m

    timeout server          1m

    timeout http-keep-alive 10s

     timeout check           10s

maxconn                 3000    

frontend monitor-in

  bind *:33305 

  mode http

  option httplog

  monitor-uri /monitor 

listen stats

  bind    *:8006

  mode    http

  stats   enable

  stats   hide-version

  stats   uri       /stats

  stats   refresh   30s 

  stats   realm     Haproxy Sta tistics

  stats   auth      admin:admin

#---------------------------------------------------------------------

# apiserver frontend which proxys to the masters

#---------------------------------------------------------------------

frontend apiserver

    bind *:16443

    mode tcp

    option tcplog

    default_backend apiserver

#---------------------------------------------------------------------

# round robin balancing for apiserver

#---------------------------------------------------------------------

 backend apiserver

    option httpchk GET /healthz

option ssl-hello-chk

    http-check expect status 200

    mode tcp

    balance     roundrobin

   server master1 10.3.3.41:6443 check

     server master2 10.3.3.48:6443 check

    server master3 10.3.3.115:6443 check   

EOF

或者sudo vim /etc/haproxy/haproxy.cfg

http://69.230.210.85:8006/stats 

两台master都启动

 # 设置开机启动

 $ sudo systemctl enable haproxy

# 开启haproxy

$ sudo  systemctl start haproxy 

 # 查看启动状态

$ sudo  systemctl status haproxy

检查端口

sudo netstat -lntup|grep haproxy  

Kubernetes默认CRI（容器运行时）为Docker，因此先安装Docker。

 $ wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

$ yum -y install docker-ce-18.06.1.ce-3.el7

 $ systemctl enable docker && systemctl start docker

$ docker --version

Docker version 18.06.1-ce, build e68fc7a

$ cat > /etc/docker/daemon.json << EOF

{

"exec-opts": ["native.cgroupdriver=systemd"],

 "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"],

"log-driver": "json-file",
"log-opts": {

"max-size":"500m",

"max-file":"3"
},

"storage-driver": "overlay2",

"storage-opts": [ "overlay2.override_kernel_check=true" ]

} 

EOF 

或者sudo vim /etc/docker/daemon.json

sudo systemctl daemon-reload

sudo systemctl restart docker

sudo systemctl enable docker.service

 $ cat > /etc/yum.repos.d/kubernetes.repo << EOF

[kubernetes]

 name=Kubernetes

baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64

 enabled=1

 gpgcheck=0

repo_gpgcheck=0

gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg  

EOF

或者sudo vim /etc/yum.repos.d/kubernetes.repo  

由于版本更新频繁，这里指定版本号部署：

 #分别查询1.22.5包是否在yum源里

sudo yum list kubelet --showduplicates | sort -r

sudo yum list kubectl --showduplicates | sort -r

sudo yum list kubeadm --showduplicates | sort -r

$ sudo yum install -y kubelet-1.22.5 kubeadm-1.22.5 kubectl-1.22.5

$ sudo systemctl enable kubelet 

部署完高可用软件完再部署k8s容器

kubeadm config print init-defaults > kubeadm-init.yaml

在具有vip的master上 *** 作，这里为master1

$ sudo vim kubeadm-init.yaml

 apiVersion: kubeadm.k8s.io/v1beta3

bootstrapTokens:

- groups:

  - system:bootstrappers:kubeadm:default-node-token

  token: abcdef.0123456789abcdef

  ttl: 24h0m0s

  usages:

  - signing

  - authentication

kind: InitConfiguration

localAPIEndpoint:

  advertiseAddress: 10.3.3.41  <<---当前主机IP

  bindPort: 6443

nodeRegistration:

  criSocket: /var/run/dockershim.sock

  imagePullPolicy: IfNotPresent

  name: master1              <<--当前主机名还有下面三行

  taints:

  - effect: NoSchedule

    key: node-role.kubernetes.io/master

---

 apiServer:

  certSANs:

   - 10.3.3.88        <<----VIP

  timeoutForControlPlane: 4m0s

apiVersion: kubeadm.k8s.io/v1beta3

certificatesDir: /etc/kubernetes/pki

clusterName: kubernetes

controlPlaneEndpoint: 10.3.3.88:16443   <<<---VIP 

controllerManager: {}

dns: 

  type: CoreDNS   <<----手动添加

etcd:

  local:

    dataDir: /var/lib/etcd

imageRepository: registry.aliyuncs.com/google_containers

kind: ClusterConfiguration

kubernetesVersion: 1.22.5

networking:

  dnsDomain: cluster.local

  podSubnet: 10.244.0.0/16 <<----可以默认 注意与之后部署容器网络(CNI)里面定义Pod网络一致

  serviceSubnet: 10.96.0.0/16  <<----可以默认

scheduler: {} 

手动拉取镜像【加速】

sudo kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers

$ sudo kubeadm init --config kubeadm-init.yaml --upload-certs --ignore-preflight-errors=all --v=5

或者sudo kubeadm init --apiserver-advertise-address=10.3.3.4 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.22.5 --service-cidr=10.96.0.0/12 --pod-network-cidr=10.244.0.0/16 --control-plane-endpoint 10.3.3.4:6443 --ignore-preflight-errors=all(需要copy 证书到其他的master)

注意 ：如果出现 Error writing Crisocket information for the control-plane node: timed out waiting for the condition

重启 kubeadm 然后再执行init

sudo kubeadm reset

按照提示配置环境变量，使用kubectl工具：

 $ mkdir -p $HOME/.kube

$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

$ kubectl get nodes

$ kubectl get pods -n kube-system 

sudo yum -y install bash-completion && cd ~;echo "source <(kubectl completion bash)" >> .bashrc

按照提示保存以下内容，一会要使用：

#You can now join any number of the control-plane node running the following command on each as root:

kubeadm join 10.3.3.54:6443 --token abcdef.0123456789abcdef

--discovery-token-ca-cert-hash sha256:30cd7726c990de6288d144178095122c1149ef237c3ab85f6e68ce46d96578e8

--control-plane --certificate-key 77dfd4ee1a71f1a531d1ee2128469a728466a9add052acd14f2a9fda04c595e8

#Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.3.3.54:6443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:30cd7726c990de6288d144178095122c1149ef237c3ab85f6e68ce46d96578e8

查看集群状态

 kubectl get cs

kubectl get pods -n kube-system 

从官方地址获取到calico的yaml，在master1上执行

wget https://docs.projectcalico.org/manifests/calico.yaml

修改CALICO_IPV4POOL_CIDR为自定义地址

该地址与kubeadm init指定pod-network-cidr一致

安装calico网络

kubectl apply -f calico.yaml

检查

kubectl get pods -n kube-system

安装flannel

跟pod一样

 kubectl apply -f flannel.yaml 

集群部署后scheduler状态为Unhealthy

在安装网络插件之后再修改此处，所有的master都修改

sudo vim /etc/kubernetes/manifests/kube-scheduler.yaml

sudo vim /etc/kubernetes/manifests/kube-controller-manager.yaml

- --port=0 注释

重启

sudo systemctl restart kubelet.service

执行在master1上init后输出的join命令,需要带上参数--control-plane表示把master控制节点加入集群

sudo kubeadm join 10.3.3.88:16443 --token abcdef.0123456789abcdef

--discovery-token-ca-cert-hash sha256:30cd7726c990de6288d144178095122c1149ef237c3ab85f6e68ce46d96578e8

--control-plane --certificate-key  --ignore-preflight-errors=all --v=5

检查状态

 kubectl get node

kubectl get pods --all-namespaces 

在node1上执行

向集群添加新节点，执行在kubeadm init输出的kubeadm join命令：

sudo kubeadm join 10.3.3.88:16443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:30cd7726c990de6288d144178095122c1149ef237c3ab85f6e68ce46d96578e8  --ignore-preflight-errors=all --v=5

检查状态

kubectl get node

kubectl get pods --all-namespaces

在Kubernetes集群中创建一个pod，验证是否正常运行：

$ kubectl create deployment nginx --image=nginx

$ kubectl expose deployment nginx --port=80 --type=NodePort

$ kubectl get pod,svc

访问：curl http://10.3.3.41:31552