Kubernetes高可用集群

0、服务器信息

服务器为腾讯云服务器（按需计费，按流量计费，并且将IP转换为弹性IP，使用结束可关机仅收取硬盘等固定资源费用）

1、环境准备

如未特别标注，均为所有机器都需执行

1.1、host配置

cat>>/etc/hosts<< EOF
10.19.0.5  Kuernetes1
10.19.0.9  Kuernetes2
10.19.0.11 Kuernetes3
10.19.0.2  Kuernetes4
10.19.0.4  Kuernetes5
10.19.0.12 Kuernetes6
EOF

1.2、ssh免密链接

# 生成ssh密钥
[root@Kuernetes1 ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:5T64EqJDkJMHYMXBs4OqQGWKjtPbF5yvTfh0V3Ynpso root@Kuernetes1
The key's randomart image is:
+---[RSA 2048]----+
|o.+o.            |
|o  *             |
|.+= o     .      |
|*+.o     o       |
|==  .. .S .   = o|
|=.o . =. o   = o.|
|oo + ..++ + o    |
|. + . o=.+ +     |
|   . ..o+ E      |
+----[SHA256]-----+
# 拷贝id_rsa.pub到目标机器，下次即可直接访问(第一次需要输入密码)
[root@Kuernetes1 ~]# for i in Kuernetes1 Kuernetes2 Kuernetes3 Kuernetes4 Kuernetes5 Kuernetes6;do ssh-copy-id -i .ssh/id_rsa.pub $i;done
······
# 测试免密连接
[root@Kuernetes1 ~]# ssh root@Kuernetes-2
Last login: Tue May  9 22:54:58 2023 from 183.195.73.137
[root@Kuernetes2 ~]# hostname
Kuernetes-2
[root@Kuernetes2 ~]# exit
logout
Connection to kuernetes2 closed.

1.3、升级内核

# 查看内核(3.10内核在大规模集群具有不稳定性，需将内核升级到4.19+)
[root@Kuernetes1 ~]# uname -sr
Linux 3.10.0-1160.88.1.el7.x86_64
# 升级软件包
[root@Kuernetes1 ~]# yum update -y --exclude=kernel*
······
# 导入ELRepo软件仓库的公共秘钥
[root@Kuernetes1 ~]# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
[root@Kuernetes1 ~]# rpm -Uvh https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
Retrieving https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
Preparing...                          ################################# [100%]
Updating / installing...
   1:elrepo-release-7.0-6.el7.elrepo  ################################# [100%]
# 镜像加速
[root@Kuernetes1 ~]# yum install -y yum-plugin-fastestmirror
······
# 查看当前可升级到的版本
[root@Kuernetes1 ~]# yum --disablerepo="*" --enablerepo="elrepo-kernel" list available 
······
# 升级内核(需要一定时间)
[root@Kuernetes1 ~]# yum --enablerepo=elrepo-kernel install -y kernel-lt
······
# 查看当前系统已安装内核
[root@Kuernetes1 ~]# awk -F' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux 7 Rescue 95a46ad0ee7f4772b6251edf4514b358 (5.4.242-1.el7.elrepo.x86_64)
1 : CentOS Linux (5.4.242-1.el7.elrepo.x86_64) 7 (Core)
2 : CentOS Linux (3.10.0-1160.88.1.el7.x86_64) 7 (Core)
3 : CentOS Linux (0-rescue-ba63ad6a0c7246dd8b30c727aae0e195) 7 (Core)
# 重构内核
[root@Kuernetes1 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.4.242-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-5.4.242-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1160.88.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1160.88.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-95a46ad0ee7f4772b6251edf4514b358
Found initrd image: /boot/initramfs-0-rescue-95a46ad0ee7f4772b6251edf4514b358.img
Found linux image: /boot/vmlinuz-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195
Found initrd image: /boot/initramfs-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195.img
done
# 再次查看当前系统已安装内核
[root@Kuernetes-6 ~]# awk -F' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux (5.4.242-1.el7.elrepo.x86_64) 7 (Core)
1 : CentOS Linux (3.10.0-1160.88.1.el7.x86_64) 7 (Core)
2 : CentOS Linux (0-rescue-92581b1e8bfc4373a10654f47c8911f3) 7 (Core)
3 : CentOS Linux (0-rescue-ba63ad6a0c7246dd8b30c727aae0e195) 7 (Core)
# 修改默认内核（GRUB_DEFAULT=0）
[root@Kuernetes1 ~]# vi /etc/default/grub
[root@Kuernetes1 ~]# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=0
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL="serial console"
GRUB_TERMINAL_OUTPUT="serial console"
GRUB_CMDLINE_LINUX="crashkernel=2G-8G:256M,8G-16G:512M,16G-:768M console=ttyS0,115200 console=tty0 panic=5 net.ifnames=0 biosdevname=0 intel_idle.max_cstate=1 intel_pstate=disable processor.max_cstate=1 amd_iommu=on iommu=pt"
GRUB_DISABLE_RECOVERY="true"
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
# 再次重构内核
[root@Kuernetes1 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.4.242-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-5.4.242-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1160.88.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1160.88.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-95a46ad0ee7f4772b6251edf4514b358
Found initrd image: /boot/initramfs-0-rescue-95a46ad0ee7f4772b6251edf4514b358.img
Found linux image: /boot/vmlinuz-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195
Found initrd image: /boot/initramfs-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195.img
done
# 重启
[root@Kuernetes1 ~]# reboot
# 检查重启后的内核
[root@Kuernetes1 ~]# uname -sr
Linux 5.4.242-1.el7.elrepo.x86_64

1.4、安装cfssl

仅主节点即可

https://github.com/cloudflare/cfssl/releases

[root@Kuernetes1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
······
[root@Kuernetes1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64
······
[root@Kuernetes1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl-certinfo_1.6.4_linux_amd64
······
[root@Kuernetes1 ~]# ls
cfssl_1.6.4_linux_amd64  cfssl-certinfo_1.6.4_linux_amd64  cfssljson_1.6.4_linux_amd64
[root@Kuernetes1 ~]# chmod +x cfssl*
[root@Kuernetes1 ~]# mv cfssl_1.6.4_linux_amd64 cfssl
[root@Kuernetes1 ~]# mv cfssl-certinfo_1.6.4_linux_amd64 cfssl-certinfo
[root@Kuernetes1 ~]# mv cfssljson_1.6.4_linux_amd64 cfssljson
[root@Kuernetes1 ~]# ls -l
total 28572
-rwxr-xr-x 1 root root 12054528 Apr 11 03:07 cfssl
-rwxr-xr-x 1 root root  9560064 Apr 11 03:08 cfssl-certinfo
-rwxr-xr-x 1 root root  7643136 Apr 11 03:07 cfssljson
[root@Kuernetes1 ~]# mv cfssl* /usr/bin/
[root@Kuernetes1 ~]# cfssl
No command is given.
Usage:
Available commands:
	revoke
	bundle
	certinfo
	sign
	genkey
	ocspserve
	info
	crl
	serve
	gencert
	gencsr
	scan
	gencrl
	ocspdump
	ocspsign
	print-defaults
	version
	ocsprefresh
	selfsign
Top-level flags:

1.5、系统设置

# set SELinux permissive(disable)
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

# close swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri 's/.*swap.*/#&/' /etc/fstab

ulimit -SHn 65535
# 在文件末尾追加内容
cat>>/etc/security/limits.conf<< EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF

1.6、ipvs组件

cat <<EOF | sudo tee /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

# 安装ipvs组件
[root@Kuernetes1 ~]# yum install ipvsadm ipset sysstat conntrack libseccomp -y
······
# 修改ipvs配置
[root@Kuernetes1 ~]# cat <<EOF | sudo tee /etc/modules-load.d/ipvs.conf
> ip_vs
> ip_vs_lc
> ip_vs_wlc
> ip_vs_rr
> ip_vs_wrr
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_dh
> ip_vs_sh
> ip_vs_fo
> ip_vs_nq
> ip_vs_sed
> ip_vs_ftp
> ip_vs_sh
> nf_conntrack
> ip_tables
> ip_set
> xt_set
> ipt_set
> ipt_rpfilter
> ipt_REJECT
> ipip
> EOF
# 重启
[root@Kuernetes1 ~]# systemctl enable --now systemd-modules-load.service
Job for systemd-modules-load.service failed because the control process exited with error code. See "systemctl status systemd-modules-load.service" and "journalctl -xe" for details.
# 查看ipvs模块是否加载
[root@Kuernetes1 ~]# lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_ftp              16384  0 
nf_nat                 45056  1 ip_vs_ftp
ip_vs_sed              16384  0 
ip_vs_nq               16384  0 
ip_vs_fo               16384  0 
ip_vs_sh               16384  0 
ip_vs_dh               16384  0 
ip_vs_lblcr            16384  0 
ip_vs_lblc             16384  0 
ip_vs_wrr              16384  0 
ip_vs_rr               16384  0 
ip_vs_wlc              16384  0 
ip_vs_lc               16384  0 
ip_vs                 155648  24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack          147456  2 nf_nat,ip_vs
nf_defrag_ipv6         24576  2 nf_conntrack,ip_vs
nf_defrag_ipv4         16384  1 nf_conntrack
libcrc32c              16384  3 nf_conntrack,nf_nat,ip_vs

1.7、docker

#!/bin/bash
# remove old docker
yum remove docker 
        docker-client 
        docker-client-latest 
        docker-common 
        docker-latest 
        docker-latest-logrotate 
        docker-logrotate 
        docker-engine

# install dependents
yum install -y yum-utils

# set yum repo
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# install docker
yum -y install docker-ce-20.10.9-3.el7 docker-ce-cli-20.10.9-3.el7 containerd.io

# start
systemctl enable docker --now

# docker config
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://12sotewv.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

sudo systemctl daemon-reload
sudo systemctl restart docker

1.8、cri-docker

cat > /lib/systemd/system/cri-docker.service <<EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=imaxun/pause:3.9
ExecReload=/bin/kill -s HUP
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitBurst=3

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF


cat > /lib/systemd/system/cri-docker.socket <<EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF
# 启动服务
systemctl daemon-reload
systemctl enable cri-docker --now

[root@Kuernetes1 ~]# wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.0/cri-dockerd-0.3.0.amd64.tgz
······
[root@Kubernetes1 ~]# tar -zxvf cri-dockerd.tgz 
cri-dockerd/
cri-dockerd/._cri-dockerd
tar: Ignoring unknown extended header keyword `LIBARCHIVE.xattr.com.apple.quarantine'
tar: Ignoring unknown extended header keyword `LIBARCHIVE.xattr.com.apple.metadata:kMDItemWhereFroms'
cri-dockerd/cri-dockerd
[root@Kubernetes3 ~]# chmod +x cri-dockerd/cri-dockerd
[root@Kubernetes1 ~]# cp cri-dockerd/cri-dockerd /usr/bin/
[root@Kubernetes2 ~]# sh cri-docker.sh 
Created symlink from /etc/systemd/system/multi-user.target.wants/cri-docker.service to /usr/lib/systemd/system/cri-docker.service.

1.9、证书

仅主节点即可

kubernetes证书机构配置（ca-config.json）相关名词解释

• “ca-config.json”：可以定义多个 profiles，分别指定不同的过期时间、使用场景等参数；后续在签名证书时使用某个 profile
  • server：服务端证书
  • client：客户端证书
  • peer：对等证书
• “signing”：表示该证书可用于签名其它证书
• “server auth”：表示client可以用该 CA 对 server 提供的证书进行验证
• “client auth”：表示server可以用该CA对client提供的证书进行验证

证书申请（ca-csr.json）相关名词解释

• CN：域名
• C：国家
• ST：省份
• L：城市
• O：组织名
• OU：部门

sudo tee /etc/kubernetes/pki/ca-config.json <<-'EOF'
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "server": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth"
        ]
      },
      "client": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "client auth"
        ]
      },
      "peer": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ]
      },
      "kubernetes": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ]
      },
      "etcd": {
        "expiry": "87600h",
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ]
      }
    }
  }
}
EOF

sudo tee /etc/kubernetes/pki/ca-csr.json <<-'EOF'
{
  "CN": "Ialso",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shanghai",
      "L": "Shanghai",
      "O": "kubernetes",
      "OU": "kubernetes"
    }
  ],
  "ca": {
    "expiry": "87600h"
  }
}
EOF

# 创建存放证书文件夹
[root@Kuernetes1 ~]# mkdir -p /etc/kubernetes/pki
# kubernetes证书机构配置
[root@Kuernetes1 pki]# sudo tee /etc/kubernetes/pki/ca-config.json <<-'EOF'
> {
>   "signing": {
>     "default": {
>       "expiry": "87600h"
>     },
>     "profiles": {
>       "server": {
>         "expiry": "87600h",
>         "usages": [
>           "signing",
>           "key encipherment",
>           "server auth"
>         ]
>       },
>       "client": {
>         "expiry": "87600h",
>         "usages": [
>           "signing",
>           "key encipherment",
>           "client auth"
>         ]
>       },
>       "peer": {
>         "expiry": "87600h",
>         "usages": [
>           "signing",
>           "key encipherment",
>           "server auth",
>           "client auth"
>         ]
>       },
>       "kubernetes": {
>         "expiry": "87600h",
>         "usages": [
>           "signing",
>           "key encipherment",
>           "server auth",
>           "client auth"
>         ]
>       },
>       "etcd": {
>         "expiry": "87600h",
>         "usages": [
>           "signing",
>           "key encipherment",
>           "server auth",
>           "client auth"
>         ]
>       }
>     }
>   }
> }
> EOF
······
# kubernetes证书机构证书申请
[root@Kuernetes1 pki]# sudo tee /etc/kubernetes/pki/ca-csr.json <<-'EOF'
> {
>   "CN": "Ialso",
>   "key": {
>     "algo": "rsa",
>     "size": 2048
>   },
>   "names": [
>     {
>       "C": "CN",
>       "ST": "Shanghai",
>       "L": "Shanghai",
>       "O": "kubernetes",
>       "OU": "kubernetes"
>     }
>   ],
>   "ca": {
>     "expiry": "87600h"
>   }
> }
> EOF
······
# 生成kubernetes机构证书
[root@Kuernetes1 pki]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2023/05/10 22:24:22 [INFO] generating a new CA key and certificate from CSR
2023/05/10 22:24:22 [INFO] generate received request
2023/05/10 22:24:22 [INFO] received CSR
2023/05/10 22:24:22 [INFO] generating key: rsa-2048
2023/05/10 22:24:22 [INFO] encoded CSR
2023/05/10 22:24:22 [INFO] signed certificate with serial number 613017233483743397580047557677682512319581508456
# 查看证书 ca-key.pem私钥；ca.pem公钥
[root@Kuernetes1 pki]# ls
ca-config.json  ca.csr  ca-csr.json  ca-key.pem  ca.pem

2、Kubernetes组件安装

2.1、etcd

1、安装

# 下载etcd：https://github.com/etcd-io/etcd/releases
[root@Kuernetes1 ~]# wget https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz
······
# 拷贝etcd安装包至其他master节点
[root@Kuernetes1 pki]# for i in Kuernetes2 Kuernetes3;do scp etcd-* root@$i:/root/;done
etcd-v3.5.6-linux-amd64.tar.gz                                                                                                                                100%   19MB 234.8MB/s   00:00    
etcd-v3.5.6-linux-amd64.tar.gz                                                                                                                                100%   19MB 118.2KB/s   02:41    
etcd-v3.5.6-linux-amd64.tar.gz                                                                                                                                100%   19MB 114.7KB/s   02:46
# 解压安装包
[root@Kuernetes1 ~]# tar -zxvf etcd-v3.5.6-linux-amd64.tar.gz
[root@Kuernetes1 ~]# mv /root/etcd-v3.5.6-linux-amd64/etcd /usr/bin/
[root@Kuernetes1 ~]# mv /root/etcd-v3.5.6-linux-amd64/etcdctl /usr/bin/
# 验证etcd
[root@Kuernetes1 ~]# etcdctl
······

2、证书

仅主节点即可

sudo tee /etc/kubernetes/pki/etcd/etcd-ca-csr.json <<-'EOF'
{
  "CN": "Ialso",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shanghai",
      "L": "Shanghai",
      "O": "etcd",
      "OU": "etcd"
    }
  ],
  "ca": {
    "expiry": "87600h"
  }
}
EOF

sudo tee /etc/kubernetes/pki/etcd/etcd-ialso-csr.json <<-'EOF'
{
  "CN": "etcd-ialso",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "Kuernetes1",
    "Kuernetes2",
    "Kuernetes3",
    "Kuernetes4",
    "Kuernetes5",
    "Kuernetes6"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Shanghai",
      "L": "Shanghai",
      "O": "etcd",
      "OU": "etcd-colony"
    }
  ]
}
EOF

cfssl gencert 
   -ca=/etc/kubernetes/pki/etcd/ca.pem 
   -ca-key=/etc/kubernetes/pki/etcd/ca-key.pem 
   -config=/etc/kubernetes/pki/ca-config.json 
   -profile=etcd 
   etcd-ialso-csr.json | cfssljson -bare /etc/kubernetes/pki/etcd/etcd

# etcd机构证书申请
[root@Kubernetes1 pki]# sudo tee /etc/kubernetes/pki/etcd/ca-csr.json <<-'EOF'
> {
>   "CN": "Ialso",
>   "key": {
>     "algo": "rsa",
>     "size": 2048
>   },
>   "names": [
>     {
>       "C": "CN",
>       "ST": "Shanghai",
>       "L": "Shanghai",
>       "O": "etcd",
>       "OU": "etcd"
>     }
>   ],
>   "ca": {
>     "expiry": "87600h"
>   }
> }
> EOF
······
# 生成etcd机构证书
[root@Kubernetes1 etcd]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare ca -
2023/05/11 01:04:38 [INFO] generating a new CA key and certificate from CSR
2023/05/11 01:04:38 [INFO] generate received request
2023/05/11 01:04:38 [INFO] received CSR
2023/05/11 01:04:38 [INFO] generating key: rsa-2048
2023/05/11 01:04:38 [INFO] encoded CSR
2023/05/11 01:04:38 [INFO] signed certificate with serial number 638045234947858116581635552444821777926630480846
# 查看证书 ca-key.pem私钥；ca.pem公钥
[root@Kubernetes1 etcd]# ls
ca.csr  ca-key.pem  ca.pem
# 在etcd机构为ialso签发证书
[root@Kubernetes1 etcd]# cfssl gencert 
>    -ca=/etc/kubernetes/pki/etcd/ca.pem 
>    -ca-key=/etc/kubernetes/pki/etcd/ca-key.pem 
>    -config=/etc/kubernetes/pki/ca-config.json 
>    -profile=etcd 
>    etcd-ialso-csr.json | cfssljson -bare /etc/kubernetes/pki/etcd/etcd
2023/05/11 01:27:09 [INFO] generate received request
2023/05/11 01:27:09 [INFO] received CSR
2023/05/11 01:27:09 [INFO] generating key: rsa-2048
2023/05/11 01:27:09 [INFO] encoded CSR
2023/05/11 01:27:09 [INFO] signed certificate with serial number 547412799563394483789087934200510450231255257959
# 查看证书 etcd-key.pem私钥；etcd.pem公钥
[root@Kubernetes1 etcd]# ls
ca.csr  ca-key.pem  ca.pem  etcd-ca-csr.json  etcd.csr  etcd-ialso-csr.json  etcd-key.pem  etcd.pem
# 拷贝证书至其他将要安装etcd的节点
[root@Kubernetes1 etcd]# for i in Kuernetes2 Kuernetes3;do scp -r /etc/kubernetes/pki/etcd/*.pem root@$i:/etc/kubernetes/pki/;done
······

3、启动

etcd配置文件：https://doczhcn.gitbook.io/etcd/index/index-1/configuration

# 三个节点配置文件只需要更换本机ip即可
name: 'etcd1'
data-dir: /etc/kubernetes/etcd/data
wal-dir: /etc/kubernetes/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'http://10.19.0.5:2380' # 本机ip+2380端口
listen-client-urls: 'http://10.19.0.510.19.0.5:2379,http://127.0.0.1:2379' # 本机ip+2379端口
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'http://10.19.0.5:2380' # 本机ip+2380端口
advertise-client-urls: 'http://10.19.0.5:2379' # 本机ip+2379端口
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'etcd1=http://10.19.0.5:2380,etcd2=http://10.19.0.9:2380,etcd3=http://10.19.0.11:2380' # etcd集群节点名称+ip
initial-cluster-token: 'etcd-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false

cat > /usr/lib/systemd/system/etcd.service  <<EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
ExecStart=/usr/bin/etcd --config-file=/etc/kubernetes/etcd/etcd-conf.yaml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
systemctl enable --now etcd.service

[root@Kubernetes1 ~]# systemctl enable --now etcd.service
● etcd.service - Etcd Service
   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Thu 2023-05-11 03:27:10 CST; 23s ago
     Docs: https://coreos.com/etcd/docs/latest
 Main PID: 27654 (etcd)
    Tasks: 8
   Memory: 21.1M
   CGroup: /system.slice/etcd.service
           └─27654 /usr/bin/etcd --config-file=/etc/kubernetes/etcd/etcd-conf.yaml

May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.185+0800","caller":"et...
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.185+0800","caller"...ts"}
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.186+0800","caller"...79"}
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.186+0800","caller"...ts"}
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.187+0800","caller"...79"}
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.187+0800","caller"...on"}
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.187+0800","caller"...on"}
May 11 03:27:10 Kubernetes1 systemd[1]: Started Etcd Service.
May 11 03:27:10 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:10.219+0800","caller":"et...
May 11 03:27:12 Kubernetes1 etcd[27654]: {"level":"info","ts":"2023-05-11T03:27:12.864+0800","caller"....5"}
Hint: Some lines were ellipsized, use -l to show in full.

4、测试

# 查看集群健康状态
[root@Kubernetes1 ~]# ETCDCTL_API=3 etcdctl  --cacert=/etc/kubernetes/pki/etcd/ca.pem  --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem  --endpoints="http://10.19.0.5:2379,http://10.19.0.9:2379,http://10.19.0.11:2379" endpoint health --write-out=table
+------------------------+--------+------------+-------+
|        ENDPOINT        | HEALTH |    TOOK    | ERROR |
+------------------------+--------+------------+-------+
|  http://10.19.0.9:2379 |   true |  3.32615ms |       |
| http://10.19.0.11:2379 |   true | 6.496657ms |       |
|  http://10.19.0.5:2379 |   true | 6.435083ms |       |
+------------------------+--------+------------+-------+
[root@Kubernetes3 ~]# etcdctl member list --write-out=table
+------------------+---------+-------+------------------------+------------------------+------------+
|        ID        | STATUS  | NAME  |       PEER ADDRS       |      CLIENT ADDRS      | IS LEARNER |
+------------------+---------+-------+------------------------+------------------------+------------+
| 268f6cee9eec2784 | started | etcd2 |  http://10.19.0.9:2380 |  http://10.19.0.9:2379 |      false |
| 73958ca2ca971e78 | started | etcd1 |  http://10.19.0.5:2380 |  http://10.19.0.5:2379 |      false |
| 9402ff1caa692dac | started | etcd3 | http://10.19.0.11:2380 | http://10.19.0.11:2379 |      false |
+------------------+---------+-------+------------------------+------------------------+------------+