注意docker的安装，可直接参考清华源（https://mirrors.tuna.tsinghua.edu.cn/help/docker-ce/）、阿里云源（https://developer.aliyun.com/mirror/docker-ce?spm=a2c6h.13651102.0.0.3e221b11pq7nqt）等。

此处以清华源为例

• 安装

  # 如果过去安装过 docker，先删掉:
  root@master-01:~# apt-get remove docker docker-engine docker.io
  
  # 安装依赖:
  root@master-01:~# apt-get -y install apt-transport-https ca-certificates curl gnupg2 software-properties-common
  
  # 信任 Docker 的 GPG 公钥:
  root@master-01:~# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
  
  # 对于amd64 架构的计算机，添加软件仓库:
  root@master-01:~# add-apt-repository \
     "deb [arch=amd64] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu \ $(lsb_release -cs) \ stable"
  
  # 安装
  root@master-01:~# apt-get update
  # apt-get -y install docker-ce 
  
  # 设置镜像加速和驱动
  root@master-01:~# cat <<EOF > /etc/docker/daemon.json
  { 
              
      "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"],
      "exec-opts": ["native.cgroupdriver=systemd"]
  }
  EOF
  
  root@master-01:~# systemctl daemon-reload
  root@master-01:~# systemctl restart docker.service
  

注意：k8s软件的安装也可以参考阿里云源（https://developer.aliyun.com/mirror/kubernetes?spm=a2c6h.13651102.0.0.3e221b11pq7nqt）和清华源（https://mirrors.tuna.tsinghua.edu.cn/help/kubernetes/）等

• 配置k8s国内源 以阿里云源为例：

    root@master-01:~# apt-get update && apt-get install -y apt-transport-https
    root@master-01:~# curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - 
    root@master-01:~# cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
    deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
    EOF
    root@master-01:~# apt-get update
    root@master-01:~# apt-get install -y kubelet kubeadm kubectl
  

  注意：在安装kubelet、kubeadm、kubectl等软件时可指定版本，如kubelet=1.20.0-00 kubeadm=1.20.0-00 kubectl=1.20.0-00

    ## 查看可安装的k8s版本
    root@master-01:~# apt list kubeadm -a
    Listing... Done
    kubeadm/kubernetes-xenial 1.24.0-00 amd64
    kubeadm/kubernetes-xenial 1.23.6-00 amd64
    kubeadm/kubernetes-xenial 1.23.5-00 amd64
    kubeadm/kubernetes-xenial 1.23.4-00 amd64
    ···
    kubeadm/kubernetes-xenial 1.22.4-00 amd64
    kubeadm/kubernetes-xenial 1.22.3-00 amd64
    kubeadm/kubernetes-xenial 1.22.2-00 amd64
    kubeadm/kubernetes-xenial 1.22.1-00 amd64
    kubeadm/kubernetes-xenial 1.22.0-00 amd64
    kubeadm/kubernetes-xenial 1.21.12-00 amd64
    ···
    kubeadm/kubernetes-xenial 1.21.0-00 amd64
    kubeadm/kubernetes-xenial 1.20.15-00 amd64
    ···
    kubeadm/kubernetes-xenial 1.6.3-00 amd64
    kubeadm/kubernetes-xenial 1.6.2-00 amd64
    kubeadm/kubernetes-xenial 1.6.1-00 amd64
    kubeadm/kubernetes-xenial 1.5.7-00 amd64
    
    ## 安装指定版本的k8s软件
    root@master-01:~# apt-get install -y kubelet=1.20.0-00 kubeadm=1.20.0-00 kubectl=1.20.0-00
    Reading package lists... Done
    Building dependency tree
    Reading state information... Done
    The following additional packages will be installed:
      conntrack cri-tools kubernetes-cni socat
    The following NEW packages will be installed:
      conntrack cri-tools kubeadm kubectl kubelet kubernetes-cni socat
    0 upgraded, 7 newly installed, 0 to remove and 57 not upgraded.
    Need to get 75.1 MB of archives.
    After this operation, 311 MB of additional disk space will be used.
    Get:1 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 cri-tools amd64 1.23.0-00 [15.3 MB]
    Get:2 http://cn.archive.ubuntu.com/ubuntu bionic/main amd64 conntrack amd64 1:1.4.4+snapshot20161117-6ubuntu2 [30.6 kB]
    Get:3 http://cn.archive.ubuntu.com/ubuntu bionic/main amd64 socat amd64 1.7.3.2-2ubuntu2 [342 kB]
    Get:4 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 kubernetes-cni amd64 0.8.7-00 [25.0 MB]
    Get:5 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 kubelet amd64 1.20.0-00 [18.8 MB]
    Get:6 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 kubectl amd64 1.20.0-00 [7942 kB]
    Get:7 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 kubeadm amd64 1.20.0-00 [7707 kB]
    Fetched 75.1 MB in 24s (3171 kB/s)
    Selecting previously unselected package conntrack.
    (Reading database ... 67508 files and directories currently installed.)
    Preparing to unpack .../0-conntrack_1%3a1.4.4+snapshot20161117-6ubuntu2_amd64.deb ...
    Unpacking conntrack (1:1.4.4+snapshot20161117-6ubuntu2) ...
    Selecting previously unselected package cri-tools.
    Preparing to unpack .../1-cri-tools_1.23.0-00_amd64.deb ...
    Unpacking cri-tools (1.23.0-00) ...
    Selecting previously unselected package kubernetes-cni.
    Preparing to unpack .../2-kubernetes-cni_0.8.7-00_amd64.deb ...
    Unpacking kubernetes-cni (0.8.7-00) ...
    Selecting previously unselected package socat.
    Preparing to unpack .../3-socat_1.7.3.2-2ubuntu2_amd64.deb ...
    Unpacking socat (1.7.3.2-2ubuntu2) ...
    Selecting previously unselected package kubelet.
    Preparing to unpack .../4-kubelet_1.20.0-00_amd64.deb ...
    Unpacking kubelet (1.20.0-00) ...
    Selecting previously unselected package kubectl.
    Preparing to unpack .../5-kubectl_1.20.0-00_amd64.deb ...
    Unpacking kubectl (1.20.0-00) ...
    Selecting previously unselected package kubeadm.
    Preparing to unpack .../6-kubeadm_1.20.0-00_amd64.deb ...
    Unpacking kubeadm (1.20.0-00) ...
    Setting up conntrack (1:1.4.4+snapshot20161117-6ubuntu2) ...
    Setting up kubernetes-cni (0.8.7-00) ...
    Setting up cri-tools (1.23.0-00) ...
    Setting up socat (1.7.3.2-2ubuntu2) ...
    Setting up kubelet (1.20.0-00) ...
    Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service → /lib/systemd/system/kubelet.service.
    Setting up kubectl (1.20.0-00) ...
    Setting up kubeadm (1.20.0-00) ...
    Processing triggers for man-db (2.8.3-2ubuntu0.1) ...
  

root@master-01:~# kubeadm init \
--apiserver-advertise-address=192.168.43.100 \
--kubernetes-version=v1.20.0 \
--image-repository registry.aliyuncs.com/google_containers \
--pod-network-cidr=10.244.0.0/16 \
--service-cidr=10.96.0.0/12 --ignore-preflight-errors=Swap
[init] Using Kubernetes version: v1.20.0
[preflight] Running pre-flight checks
        [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.16. Latest validated version: 19.03
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local master-01] and IPs [10.96.0.1 192.168.43.100]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [localhost master-01] and IPs [192.168.43.100 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [localhost master-01] and IPs [192.168.43.100 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 22.005887 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.20" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node master-01 as control-plane by adding the labels "node-role.kubernetes.io/master=''" and "node-role.kubernetes.io/control-plane='' (deprecated)"
[mark-control-plane] Marking the node master-01 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: k3jj1c.okb3o8vzl90cb57a
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.43.100:6443 --token k3jj1c.okb3o8vzl90cb57a \
    --discovery-token-ca-cert-hash sha256:e0159d7cac73cd8201f7e1ce920161f1b990d44f22a8d39825c3cfc00b27bca3

# 如果在内网安装，可使用pull提前下载好镜像，公网则不必如此
# kubeadm config images pull

# 授权用户使用kubectl命令管理集群
ops@master-01:~# mkdir -p $HOME/.kube
root@master-01:~# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
root@master-01:~# chown $(id -u):$(id -g) $HOME/.kube/config

## 永久生效
ops@master-01:~# export KUBECONFIG=$HOME/.kube/config
ops@master-01:~# echo "export KUBECONFIG=$HOME/.kube/config" >> $HOME/.bashrc
ops@master-01:~# source $HOME/.bashrc

参数解释：

• -apiserver-advertise-address：设置Master节点API Server的监听地址
• -kubernetes-version ：指定Kubernetes版本
• -image-repository ：由于kubeadm默认是从官网k8s.grc.io下载所需镜像，国内无法访问，所以这里通过–image-repository指定为阿里云镜像仓库地址
• -pod-network-cidr ：指定pod网络段
• -service-cidr ：指定service网络段
• -ignore-preflight-errors=Swap ：忽略swap报错信息

检查集群的健康性：

# 集群健康检查，查看node节点和集群状态
ops@master-01:~$ kubectl get node
NAME       STATUS     ROLES                  AGE    VERSION
master01   NotReady   control-plane,master   8m5s   v1.20.0

## 查看集群状态
root@master-01:~# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                       ERROR
scheduler            Unhealthy   Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused 
controller-manager   Unhealthy   Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused 
etcd-0               Healthy     { 
          "health":"true"} 

## 出现这种情况是kube-controller-manager.yaml和kube-scheduler.yaml文件中均设置的默认端口是0，在文件中注释掉就可以了。(每台master节点都要执行操作)，将下面两个文件中的- --port=0注释掉即可
root@master-01:~# vim /etc/kubernetes/manifests/kube-scheduler.yaml
root@master-01:~# vim /etc/kubernetes/manifests/kube-controller-manager.yaml

root@master-01:~# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   { 
          "health":"true"}

## 查看kube-system命名空间下的Pod状态，其中coredns由于网络插件未部署，因此一直是pending状态
root@master-01:~# kubectl get pods -n kube-system -o wide
NAME    READY   STATUS    RESTARTS   AGE     IP    NODE    NOMINATED NODE   READINESS GATES
coredns-7f89b7bc75-9hxk4  0/1 Pending   0  7m     <none>        <none>    <none>  <none>
coredns-7f89b7bc75-p2rjn  0/1 Pending   0  7m     <none>        <none>    <none>  <none>
etcd-master-01            1/1 Running   0  7m  192.168.43.100   master-01  <none> <none>
kube-apiserver-master-01  1/1 Running   0  7m  192.168.43.100   master-01  <none>  <none>
kube-controller-manager-master-01 1/1 Running 0 7m 192.168.43.100 master-01 <none> <none>
kube-proxy-7m2ct   1/1  Running   0   7m   192.168.43.100   master-01   <none>     <none>
kube-scheduler-master-01   1/1 Running  0 7m 192.168.43.100   master-01   <none>   <none>

在每个node节点上执行

root@node-01:~# kubeadm join 192.168.43.100:6443 --token k3jj1c.okb3o8vzl90cb57a \
    --discovery-token-ca-cert-hash sha256:e0159d7cac73cd8201f7e1ce920161f1b990d44f22a8d39825c3cfc00b27bca3
[preflight] Running pre-flight checks
        [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.16. Latest validated version: 19.03
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

# 拷贝master上的配置文件到所有node节点上，在master上执行
root@master-01:~# scp -P 10022 /etc/kubernetes/admin.conf ops@192.168.43.101:/home/ops
root@master-01:~# scp -P 10022 /etc/kubernetes/admin.conf ops@192.168.43.102:/home/ops

# 在node上执行
root@node-01:~# mkdir -p $HOME/.kube
root@node-01:~# cp -i /home/ops/admin.conf $HOME/.kube/config
root@node-01:~# chown $(id -u):$(id -g) $HOME/.kube/config

## 永久生效
ops@master-01:~# export KUBECONFIG=$HOME/.kube/config
ops@master-01:~# echo "export KUBECONFIG=$HOME/.kube/config" >> $HOME/.bashrc
ops@master-01:~# source $HOME/.bashrc

# 可查看node节点信息，由于网络插件未部署，因此status均为NotReady
root@node-01:~# kubectl get nodes -o wide
NAME        STATUS     ROLES                  AGE     VERSION   INTERNAL-IP      EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION       CONTAINER-RUNTIME
master-01   NotReady   control-plane,master   21m     v1.20.0   192.168.43.100   <none>        Ubuntu 18.04.5 LTS   4.15.0-177-generic   docker://20.10.16
node-01     NotReady   <none>                 3m12s   v1.20.0   192.168.43.101   <none>        Ubuntu 18.04.5 LTS   4.15.0-177-generic   docker://20.10.16
node-02     NotReady   <none>                 2m58s   v1.20.0   192.168.43.102   <none>        Ubuntu 18.04.5 LTS   4.15.0-177-generic   docker://20.10.16

## 将节点都加入集群后，再次查看kube-system空间的pod
root@master-01:~# kubectl get pods -n kube-system -o wide
NAME    READY   STATUS    RESTARTS   AGE     IP    NODE    NOMINATED NODE   READINESS GATES
coredns-7f89b7bc75-9hxk4  0/1 Pending   0  27m     <none>        <none>    <none>  <none>
coredns-7f89b7bc75-p2rjn  0/1 Pending   0  27m     <none>        <none>    <none>  <none>
etcd-master-01            1/1 Running   0  27m  192.168.43.100   master-01  <none> <none>
kube-apiserver-master-01  1/1 Running   0  27m  192.168.43.100   master-01  <none>  <none>
kube-controller-manager-master-01 1/1 Running 0 17m 192.168.43.100 master-01 <none> <none>
kube-proxy-7m2ct   1/1  Running   0   27m   192.168.43.100   master-01   <none>     <none>
kube-proxy-94p7t   1/1  Running   0   8m56s 192.168.43.101   node-01     <none>     <none>
kube-proxy-v8c4s   1/1   Running   0  8m42s 192.168.43.102   node-02     <none>     <none>
kube-scheduler-master-01   1/1 Running  0 17m 192.168.43.100   master-01   <none>   <none>

默认token有效期为24小时，当过期之后，该token就不可用了。这时就需要重新创建token，可以直接使用命令快捷生成：

root@master-01:~# kubeadm token create --print-join-command

所有节点上执行

root@master-01:~# apt install -y bash-completion
root@master-01:~# source /usr/share/bash-completion/bash_completion

root@master-01:~# source <(kubectl completion bash)
root@master-01:~# echo "source <(kubectl completion bash)" >> ~/.bashrc

此处以calico网络插件 在未部署网络插件时，kubelet的报错信息如下：

## 此时查看kubelet的日志如下：
root@master-01:~# journalctl -xe
May 22 11:30:50 master-01 kubelet[10648]: E0522 11:30:50.623972   10648 kubelet.go:2160] Container runtime network not ready:
May 22 11:30:55 master-01 kubelet[10648]: W0522 11:30:55.049449   10648 cni.go:239] Unable to update cni config: no networks
May 22 11:30:55 master-01 kubelet[10648]: E0522 11:30:55.638376   10648 kubelet.go:2160] Container runtime network not ready:
May 22 11:31:00 master-01 kubelet[10648]: W0522 11:31:00.050922   10648 cni.go:239] Unable to update cni config: no networks
May 22 11:31:00 master-01 kubelet[10648]: E0522 11:31:00.659415   10648 kubelet.go:2160] Container runtime network not ready:
May 22 11:31:05 master-01 kubelet[10648]: W0522 11:31:05.052126   10648 cni.go:239] Unable to update cni config: no networks
May 22 11:31:05 master-01 kubelet[10648]: E0522 11:31:05.671484   10648 kubelet.go:2160] Container runtime network not ready:

root@master-01:~# tail -f /var/log/syslog
May 22 03:33:16 ops kubelet[10648]: E0522 11:33:16.274186   10648 kubelet.go:2160] Container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:docker: network plugin is not ready: cni config uninitialized
May 22 03:33:20 ops kubelet[10648]: W0522 11:33:20.074697   10648 cni.go:239] Unable to update cni config: no networks found in /etc/cni/net.d
···

• 上传插件

  calico.yaml默认（跟k8s版本保持一致）下载地址为：wget https://docs.projectcalico.org/manifests/calico.yaml，为避免安装插件时产生错误，我已将需要使用的calico.yaml和镜像上传至gitee仓库（上传不了镜像，镜像在docker仓库中）和docker仓库，也可以使用阿里云盘下载

  git仓库地址：https://gitee.com/qingdalf/kubernetes.git 阿里云盘地址：https://www.aliyundrive.com/s/i1LNTyyXTUV

  root@master-01:~# ls
  calico-image.tar.gz  calico.yaml  
  
  root@master-01:~# tar xzf calico-image.tar.gz
  
  root@master-01:~# ls
  calico-image.tar.gz  calico.yaml  cni.tar  controllers.tar  node.tar  pod2daemon.tar
  
  root@master-01:~# docker load -i cni.tar
  root@master-01:~# docker load -i node.tar
  root@master-01:~# docker load -i pod2daemon.tar
  root@master-01:~# docker load -i controllers.tar
  

• calico.yaml文件内容

---
# Source: calico/templates/calico-config.yaml
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
  name: calico-config
  namespace: kube-system
data:
  # Typha is disabled.
  typha_service_name: "none"
  # Configure the backend to use.
  calico_backend: "bird"
  # Configure the MTU to use for workload interfaces and tunnels.
  # - If Wireguard is enabled, set to your network MTU - 60
  # - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50
  # - Otherwise, if IPIP is enabled, set to your ne