四、部署Master

4.1 生成kube-apiserver证书

1. 自签证颁发机构（CA）

cd /web/TLS/k8s cat > ca-config.json << EOF { 
           "signing": { 
             "default": { 
               "expiry": "87600h"     },     "profiles": { 
               "kubernetes": { 
                  "expiry": "87600h",          "usages": [             "signing",             "key encipherment",             "server auth",             "client auth"         ]       }     }   } } EOF cat > ca-csr.json << EOF { 
             "CN": "kubernetes",     "key": { 
                 "algo": "rsa",         "size": 2048     },     "names": [         { 
                     "C": "CN",             "L": "Beijing",             "ST": "Beijing",             "O": "k8s",             "OU": "System"         }     ] } EOF 

生成证书：

cfssl gencert -initca ca-csr.json | cfssljson -bare ca -  会生成ca.pem和ca-key.pem文件。 

4.2 使用自签CA签发kube-apiserver HTTPS证书 创建证书申请文件：hosts–IP为访问授权IP或者域名列表，考虑到后期扩容可以多写几个

cat > server-csr.json << EOF { 
             "CN": "kubernetes",     "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "192.168.2.103",
      "192.168.2.100",
      "192.168.2.101",
      "192.168.2.103",
      "192.168.2.104",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": { 
        
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        { 
        
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

生成证书：

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes server-csr.json | cfssljson -bare server

会生成server.pem和server-key.pem文件。

下载地址： https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md#v1200 Server包里面包含了所有组件

4.3 解压二进制包

mkdir -p /web/kubernetes/{ 
        bin,cfg,ssl,logs} 
tar zxvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin
cp kube-apiserver kube-scheduler kube-controller-manager /web/kubernetes/bin
cp kubectl /usr/bin/

4.4 部署kube-apiserver 1. 创建配置文件

cat > /web/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\ --v=2 \\ --log-dir=/web/kubernetes/logs \\ --etcd-servers=https://192.168.2.103:2379,https://192.168.2.100:2379,https://192.168.2.101:2379 \\ --bind-address=192.168.2.103 \\ --secure-port=6443 \\ --advertise-address=192.168.2.103 \\ --allow-privileged=true \\ --service-cluster-ip-range=10.0.0.0/24 \\ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\ --authorization-mode=RBAC,Node \\ --enable-bootstrap-token-auth=true \\ --token-auth-file=/web/kubernetes/cfg/token.csv \\ --service-node-port-range=30000-33676 \\ --kubelet-client-certificate=/web/kubernetes/ssl/kube-apiserver/server.pem \\ --kubelet-client-key=/web/kubernetes/ssl/kube-apiserver/server-key.pem \\ --tls-cert-file=/web/kubernetes/ssl/kube-apiserver/server.pem \\ --tls-private-key-file=/web/kubernetes/ssl/kube-apiserver/server-key.pem \\ --client-ca-file=/web/kubernetes/ssl/kube-apiserver/ca.pem \\ --service-account-key-file=/web/kubernetes/ssl/kube-apiserver/ca-key.pem \\ --service-account-issuer=api \\ --service-account-signing-key-file=/web/kubernetes/ssl/kube-apiserver/server-key.pem \\ --etcd-cafile=/web/etcd/ssl/ca.pem \\ --etcd-certfile=/web/etcd/ssl/server.pem \\ --etcd-keyfile=/web/etcd/ssl/server-key.pem \\ --requestheader-client-ca-file=/web/kubernetes/ssl/kube-apiserver/ca.pem \\ --proxy-client-cert-file=/web/kubernetes/ssl/kube-apiserver/server.pem \\ --proxy-client-key-file=/web/kubernetes/ssl/kube-apiserver/server-key.pem \\ --requestheader-allowed-names=kubernetes \\ --requestheader-extra-headers-prefix=X-Remote-Extra- \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-username-headers=X-Remote-User \\ --enable-aggregator-routing=true \\ --audit-log-maxage=30 \\ --audit-log-maxbackup=3 \\ --audit-log-maxsize=100 \\ --audit-log-path=/web/kubernetes/logs/k8s-audit.log"
EOF

–logtostderr：启用日志 —v：日志等级 –log-dir：日志目录 –etcd-servers：etcd集群地址 –bind-address：监听地址 –secure-port：https安全端口 –advertise-address：集群通告地址 –allow-privileged：启用授权 –service-cluster-ip-range：Service虚拟IP地址段 –enable-admission-plugins：准入控制模块 –authorization-mode：认证授权，启用RBAC授权和节点自管理 –enable-bootstrap-token-auth：启用TLS bootstrap机制 –token-auth-file：bootstrap token文件 –service-node-port-range：Service nodeport类型默认分配端口范围 –kubelet-client-xxx：apiserver访问kubelet客户端证书 –tls-xxx-file：apiserver https证书 1.20版本必须加的参数：–service-account-issuer，–service-account-signing-key-file –etcd-xxxfile：连接Etcd集群证书 –audit-log-xxx：审计日志 启动聚合层相关配置：–requestheader-client-ca-file，–proxy-client-cert-file，–proxy-client-key-file，–requestheader-allowed-names，–requestheader-extra-headers-prefix，–requestheader-group-headers，–requestheader-username-headers，–enable-aggregator-routing

2. 拷贝刚才生成的证书 把刚才生成的证书拷贝到配置文件中的路径：

cp /web/TLS/k8s/kube-apiserver/ca*pem /web/TLS/k8s/kube-apiserver/server*pem \
/web/kubernetes/ssl/kube-apiserver

3. 启用 TLS Bootstrapping 机制 创建上述配置文件中token文件：

cat > /web/kubernetes/cfg/token.csv << EOF
c47ffb939f5ca36231d9e3121a252940,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF

格式：token，用户名，UID，用户组 token也可自行生成替换：

head -c 16 /dev/urandom | od -An -t x | tr -d ' '

4. systemd管理apiserver

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/web/kubernetes/cfg/kube-apiserver.conf
ExecStart=/web/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

5. 启动并设置开机启动

systemctl daemon-reload
systemctl start kube-apiserver 
systemctl enable kube-apiserver

4.5 部署kube-controller-manager 1. 创建配置文件

cat > /web/kubernetes/cfg/kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\ --v=2 \\ --log-dir=/web/kubernetes/logs \\ --leader-elect=true \\ --kubeconfig=/web/kubernetes/cfg/kube-controller-manager.kubeconfig \\ --bind-address=127.0.0.1 \\ --allocate-node-cidrs=true \\ --cluster-cidr=10.244.0.0/16 \\ --service-cluster-ip-range=10.0.0.0/24 \\ --cluster-signing-cert-file=/web/kubernetes/ssl/ca.pem \\ --cluster-signing-key-file=/web/kubernetes/ssl/ca-key.pem \\ --root-ca-file=/web/kubernetes/ssl/ca.pem \\ --service-account-private-key-file=/web/kubernetes/ssl/ca-key.pem \\ --cluster-signing-duration=87600h0m0s"
EOF

2. 生成kubeconfig文件 生成kube-controller-manager证书：

# 切换工作目录
cd /web/TLS/k8s

# 创建证书请求文件
cat > kube-controller-manager-csr.json << EOF
{ 
        
  "CN": "system:kube-controller-manager",
  "hosts": [],
  "key": { 
        
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    { 
        
      "C": "CN",
      "L": "BeiJing", 
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

# 生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json  \
-profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare \ 
kube-controller-manager

生成kubeconfig文件（以下是shell命令，直接在终端执行）：

KUBE_CONFIG="/web/kubernetes/cfg/kube-controller-manager.kubeconfig"
KUBE_APISERVER="https://192.168.2.103:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/web/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${ 
        KUBE_APISERVER} \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-credentials kube-controller-manager \
  --client-certificate=./kube-controller-manager.pem \
  --client-key=./kube-controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-controller-manager \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${ 
        KUBE_CONFIG}

3. systemd管理controller-manager

cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/web/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/web/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

4. 启动并设置开机启动

systemctl daemon-reload
systemctl start kube-controller-manager
systemctl enable kube-controller-manager

4.6 部署kube-scheduler 1. 创建配置文件

cat > /web/kubernetes/cfg/kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \\ --v=2 \\ --log-dir=/web/kubernetes/logs \\ --leader-elect \\ --kubeconfig=/web/kubernetes/cfg/kube-scheduler.kubeconfig \\ --bind-address=127.0.0.1"
EOF

2. 生成kubeconfig文件 生成kube-scheduler证书：

# 切换工作目录
cd /web/TLS/k8s

# 创建证书请求文件
cat > kube-scheduler-csr.json << EOF
{ 
        
  "CN": "system:kube-scheduler",
  "hosts": [],
  "key": { 
        
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    { 
        
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

# 生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \ 
-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

生成kubeconfig文件（以下是shell命令，直接在终端执行）：

KUBE_CONFIG="/web/kubernetes/cfg/kube-scheduler.kubeconfig"
KUBE_APISERVER="https://192.168.2.103:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/web/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${ 
        KUBE_APISERVER} \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-credentials kube-scheduler \
  --client-certificate=./kube-scheduler.pem \
  --client-key=./kube-scheduler-key.pem \
  --embed-certs=true \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-scheduler \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${ 
        KUBE_CONFIG}

3. systemd管理scheduler

cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/web/kubernetes/cfg/kube-scheduler.conf
ExecStart=/web/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

4. 启动并设置开机启动

systemctl daemon-reload
systemctl start kube-scheduler
systemctl enable kube-scheduler

5. 查看集群状态 生成kubectl连接集群的证书：

cat > admin-csr.json <<EOF
{ 
        
  "CN": "admin",
  "hosts": [],
  "key": { 
        
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    { 
        
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \ 
-profile=kubernetes admin-csr.json | cfssljson -bare admin

生成kubeconfig文件：

mkdir /root/.kube

KUBE_CONFIG="/root/.kube/config"
KUBE_APISERVER="https://192.168.2.103:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/web/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=${ 
        KUBE_APISERVER} \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-credentials cluster-admin \
  --client-certificate=./admin.pem \
  --client-key=./admin-key.pem \
  --embed-certs=true \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config set-context default \
  --cluster=kubernetes \
  --user=cluster-admin \
  --kubeconfig=${ 
        KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${ 
        KUBE_CONFIG}

通过kubectl工具查看当前集群组件状态：

kubectl get cs
NAME                STATUS    MESSAGE             ERROR
scheduler             Healthy   ok                  
controller-manager       Healthy   ok                  
etcd-2               Healthy   { 
        "health":"true"}   
etcd-1               Healthy   { 
        "health":"true"}   
etcd-0               Healthy   { 
        "health":"true"}  

如上输出说明Master节点组件运行正常。

6. 授权kubelet-bootstrap用户允许请求证书

kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap