本文最后更新于319 天前，其中的信息可能已经过时，如有错误请发送邮件到big_fw@foxmail.com

初始化准备

设置系统主机名以及 Host 文件的相互解析

hostnamectl  set-hostname  k8s-master01

安装依赖包

yum install -y conntrack ntpdate ntp ipvsadm ipset  iptables curl sysstat libseccomp wget  vim net-tools git

设置防火墙为 Iptables 并设置空规则

systemctl  stop firewalld  &&  systemctl  disable firewalld
yum -y install iptables-services  &&  systemctl  start iptables  &&  systemctl  enable iptables  &&  iptables -F  &&  service iptables save

关闭 SELINUX

swapoff -a && sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

调整内核参数，对于 K8S

cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM  
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf  /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf

调整系统时区

# 设置系统时区为 中国/上海
timedatectl set-timezone Asia/Shanghai
# 将当前的 UTC 时间写入硬件时钟
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
systemctl restart rsyslog 
systemctl restart crond

关闭系统不需要服务

systemctl stop postfix && systemctl disable postfix

设置 rsyslogd 和 systemd journald

mkdir /var/log/journal # 持久化保存日志的目录
mkdir /etc/systemd/journald.conf.d
cat > /etc/systemd/journald.conf.d/99-prophet.conf <<EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent

# 压缩历史日志
Compress=yes

SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000

# 最大占用空间 10G
SystemMaxUse=10G

# 单日志文件最大 200M
SystemMaxFileSize=200M

# 日志保存时间 2 周
MaxRetentionSec=2week

# 不将日志转发到 syslog
ForwardToSyslog=no
EOF
systemctl restart systemd-journald

升级系统内核为 4.44

CentOS 7.x 系统自带的 3.10.x 内核存在一些 Bugs，导致运行的 Docker、Kubernetes 不稳定，例如： rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
# 安装完成后检查 /boot/grub2/grub.cfg 中对应内核 menuentry 中是否包含 initrd16 配置，如果没有，再安装一次！
yum --enablerepo=elrepo-kernel install -y kernel-lt
# 设置开机从新内核启动
grub2-set-default 'CentOS Linux (4.4.189-1.el7.elrepo.x86_64) 7 (Core)'

kubeadm安装

kube-proxy开启ipvs的前置条件

modprobe br_netfilter

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

安装 Docker 软件 /podman/RKT/containerd

yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager \
  --add-repo \
  http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum install -y docker-ce

## 创建 /etc/docker 目录
mkdir /etc/docker

# 配置 daemon.
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "insecure-registries": ["harbor.xinxianghf.com"]
}
EOF

# 重启docker服务
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

关闭 NetworkManager

$ systemctl disable NetworkManager
$ systemctl stop NetworkManager

安装 Kubeadm （主从配置）

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum -y  install  kubeadm-1.20.15 kubectl-1.20.15 kubelet-1.20.15
systemctl enable kubelet.service

初始化主节点

  kubeadm config print init-defaults > kubeadm-config.yaml
    localAPIEndpoint:
        #集群IP(masterIP)
        advertiseAddress: 192.168.66.10
    #版本号需要改，因为他会自动去翻墙去官方下载这个版本，本地有的话就本地来，要是写别的本地没有就去翻墙了
    kubernetesVersion: v1.20.15
    networking:
      #添加podSubnet，就是一定不跟当前已有的网段冲突
      podSubnet: "10.244.0.0/16"
      serviceSubnet: 10.96.0.0/12

    #用IPVS。---是固定格式表示分割，多个yaml文件写在一个文件中，就需要---
    #不能有空格，有空格的表示子选项
    ---
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    kind: KubeProxyConfiguration
    featureGates:
      SupportIPVSProxyMode: true
    mode: ipvs

# 导入镜像

kubeadm init --config=kubeadm-config.yaml --experimental-upload-certs | tee kubeadm-init.log

加入主节点以及其余工作节点

执行安装日志中的加入命令即可

部署网络

calico 官方网站：https://docs.projectcalico.org

kubectl apply -f  https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl apply -f  https://docs.projectcalico.org/v3.3/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

安装之后的简易环境

把安装日志和安装配置文件放在新建的备份目录下，配置文件以后在加入新的东西的时候用的到

calico.yaml rbac-kdd.yaml这两个文件也建议保留，如果不想用calico了，换成flannel的话，直接把这两个文件delete -f删了就好，不然的话很难删的干净，或者直接把calico目录拿过去

常见目录

/var/lib/kubelet/config.yaml
有个kubectl配置文件放在了/var/lib/kubelet/config.yaml中
这里是一些配置参数，比较重要的

[root@kube-master01 ~]# cat /var/lib/kubelet/config.yaml
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 0s
    cacheUnauthorizedTTL: 0s
cgroupDriver: systemd
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
cpuManagerReconcilePeriod: 0s
evictionPressureTransitionPeriod: 0s
fileCheckFrequency: 0s
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 0s
imageMinimumGCAge: 0s
kind: KubeletConfiguration
logging: {}
nodeStatusReportFrequency: 0s
nodeStatusUpdateFrequency: 0s
rotateCertificates: true
runtimeRequestTimeout: 0s
shutdownGracePeriod: 0s
shutdownGracePeriodCriticalPods: 0s
# 静态POD的文件目录
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 0s
syncFrequency: 0s
volumeStatsAggPeriod: 0s
[root@kube-master01 ~]#

证书目录 /etc/kubernetes/pki

https双向认证

HTTPS 认证

单向 HTTPS
    找ca机构案发证书，大家都认可，互相就都相信了
    客户端认证服务器端（确定是不是真的百度或者淘宝，别是钓鱼网站）

双向 HTTPS 
    互相认证，比如ATM机，背后即使一个web界面，微型电脑，终端机（只不过是固定在那里，不能退出）
    信用卡插入的时候，小芯片里面装的就是公钥私钥，都是银行给你颁发的，当你插入的时候就相当于输入了公钥，
    ATM把这个秘钥传到数据中心，数据中心在把它的公钥发给你，如果能解开，那就是在我这办的，双向认证

    客户端认证服务器端
    服务器端认证客户端

Kubenretes内部有两套体系，一套是etcd为主的存储端，一套是APIServer为主的服务端，所以etcd内部也有一套证书文件

存储端
存储端只有ApiServer能访问etcd
    S Etcd
    C ApiServer

服务端
    S ApiServer

资源清单目录/etc/kubernetes/manifests

也就是静态Pod的路径

Kubernetes的启动逻辑，systemd先启动——kubelet再启动——kubelet把所有的Kubernetes的组件以Pod的方式拉起来

这个目录下就是kubelet启动Kubernetes时的组件启动配置文件

这里不管是什么名字，只要是yaml的文件，他都会把它启动起来，后期想要别的组件随着Kubernetes的开启而开启的话就可以加入这个目录里面

/root/.kube/

admin.conf
这就相当于访问集群需要的钥匙，还有访问方式

这里有集群的根证书，互相验证，还有用户名，客户端的证书和私钥也都在这里，双向认证，kubectl想访问server也需要秘钥的

[root@kube-master01 ~]# cat  .kube/config
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM2VENDQWRHZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQ0FYRFRJek1EY3lOekF6TURFeE5Wb1lEekl4TWpJd056QXpNRE13TVRFMVdqQVZNUk13RVFZRApWUVFERXdwcmRXSmxjbTVsZEdWek1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCnlWUHBQUzdCZzJrSG9RSUR1UTY4V25yMkh1M1FiWDRMTGlpUkJUQnhMZWxZcEJoSk1YeHRZcDZTRExWeHJUZ3QKZVg5UjhMbm9QTjFvSjdzNStvMG5Cdm92bWZ3NGowRGhWZ0dYZzVBYkJnSlVjaTFoMmRPNDdIYWdoUlQ1Qm5GcwpnaU1MN0ZhejdFc2dhemlBak8yaGEzNi9WMlVUR0c0Vm01YjFwNEtsK2xDSS9XQW1TZkhBbmZPRjRvTjdvSVoyCkw5RVZwQ1RQY0VSelFrYXhuUXNNVlk3NS9UdUNvdTArUHlLdEVFRHR5SStycWlBamFad2xPT1FsWWVQSFFXYWwKVGRrUXp5U2NLN0F6YTl0disrZ3RPOHJaM2xnbXJZYTRCYVhBcmtoMVFIellwMjU4SWVNWndyR1A0TmFCQ1FJRgorK2JpT1NMMnlMRHBDNDlORjByUWRRSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQXFRd0R3WURWUjBUCkFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVQKzgxNVRqdmwwQW84NUFjWU5jdTZHTlcyNEF3RFFZSktvWkkKaHZjTkFRRUxCUUFEZ2dFQkFLYWpnUWtPT3JHeitXaGVlOHgrM2FQMlJ0THpyTFQ3Q0lwTk4rZ2FWSUpRQVJOaQpiQmdkMzZ0WDNSM01xSWFreWN4aWd3S2tlcllGa0Y2SHJwdmkrWHBGZXgwUHNKUDlDR2JKQThPVE05MUdPWXR0CjZyVjVPNFBDa2FnemFDVkQ2ckY2QXdCdjRoS1ErTFlzcVZhNVNKbzJGRDUyandJcG1jcGpFZnY3bmFpUW1nY3IKdVNYdGg0Ym1SSTNDVEo5ZnlZWHRmbitETlNlSXE3TTFFSUpBaXpXTDM5TUJabTViUmh0anNYVmVNbStqck9zMAozUXc1aDIzcUpjT2JkYkh4UWJlTnl6N0YzK0hWL2s1WHF4dHdvS2t6SC8yTHFXVHgvMU9kenB2R2JLUWRwS2dOClh2STN0WE0wdGhnZnZlbUFRRHVOeGlmQ1ZGNzNjbVRmN3pDM2Zjaz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
    server: https://192.168.11.253:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
  user:
    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGVENDQWYyZ0F3SUJBZ0lJTE4zdXlGcDJKdTh3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWdGdzB5TXpBM01qY3dNekF4TVRWYUdBOHlNVEl5TURjd016QXpNREV4TjFvdwpOREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhHVEFYQmdOVkJBTVRFR3QxWW1WeWJtVjBaWE10CllXUnRhVzR3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRQzdYUFlqUVdzMzBYbWQKR2Z5elpOcFB6cllWR2ZiL29PeHpQUkJwcDJNMHNGYUV0bWFEOVJ3aUI5VTkzMXhpWnF4QjRaWlVBbGg4ZGVaeApTRFpYbkRKalpLMS9UYUJSeVkxR1ZPTTRDU01LZUo1Q2wxWWNkSzdWNVNJM3FhZWFhR3I3MExNa0E2NHpxNDRyCm9PYXZNd3pTbkU2amJiNUZvcjZENTZlUzNQZXJNNUZZcm1aWUZTek9LWkhiZjliQ2Q1RGlidEJZL0lTN1ladE8KNm9wYjB2YUgvaVZyMTNaTzFpYXRUQUR6YnRrS0gyMDhLMDRDZTdYZXBOV1JMVE4zZWNJbFk2ZHRXY3hIalBycwpXSGN1Z2pYb1N2OEtFOCtHeHlWb2R3REs0dlBnWFVlT090emVQZXVIR2pFVzZHMURNVTZOTy9QSmxGU3BqckQzCjBZR0ttS1VWQWdNQkFBR2pTREJHTUE0R0ExVWREd0VCL3dRRUF3SUZvREFUQmdOVkhTVUVEREFLQmdnckJnRUYKQlFjREFqQWZCZ05WSFNNRUdEQVdnQlEvN3pYbE9PK1hRQ2p6a0J4ZzF5N29ZMWJiZ0RBTkJna3Foa2lHOXcwQgpBUXNGQUFPQ0FRRUFCRng5OU9QaU96bG0vMEtVUUxsN1BDZ2xjMHRxZExEQTJFVXN6ZE8vNGRpU2pyOGVXRjNaCm96a0FCNi9RMlBWaU1WQXltREtpM1JvRi85M0hnM2VtdGZBejA2dktsc2MvL0tHTFVYa1NZaDNzdVBHbXpUclYKOCtVWmNic01HSFFlN3NZeDVrbkVQSFVuaTR2SlJEUFhRQWwybjIxakd1YUtpckhDSm9KaDBTYjlFRVpweHp4OQpPL0c5UnFCKzBtWFZtamszZ0h2Uyt1dkpqMTZTY1dJcHhlT3BOcWNPcFdkblZEbDJYdTlQdjM3VjFEdkEzam16ClNhcHRDbmovUi9hL0plNGdBemMvb21GZFg4RTRYdjd6VWdpdHJSeDJJZHk4LzFta1VUZTBYNjJQWTdrVVRKQ2IKVCtOZGR5bDc5V0JVdlVBa0Y4b0FUdmFOS2IxUW9NZzB1QT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
    client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdTF6MkkwRnJOOUY1blJuOHMyVGFUODYyRlJuMi82RHNjejBRYWFkak5MQldoTFptCmcvVWNJZ2ZWUGQ5Y1ltYXNRZUdXVkFKWWZIWG1jVWcyVjV3eVkyU3RmMDJnVWNtTlJsVGpPQWtqQ25pZVFwZFcKSEhTdTFlVWlONm1ubW1ocSs5Q3pKQU91TTZ1T0s2RG1yek1NMHB4T28yMitSYUsrZytlbmt0ejNxek9SV0s1bQpXQlVzemltUjIzL1d3bmVRNG03UVdQeUV1MkdiVHVxS1c5TDJoLzRsYTlkMlR0WW1yVXdBODI3WkNoOXRQQ3RPCkFudTEzcVRWa1MwemQzbkNKV09uYlZuTVI0ejY3RmgzTG9JMTZFci9DaFBQaHNjbGFIY0F5dUx6NEYxSGpqcmMKM2ozcmh4b3hGdWh0UXpGT2pUdnp5WlJVcVk2dzk5R0JpcGlsRlFJREFRQUJBb0lCQUFaTkRnb0pmUzJYWWR1aQpzQmpvWDE4bnVIL2xLNlZWS0I2SUF6RkthOCt0em5zUUYwcUF4NzF0UGdLaXJGeXl6MEEzOGZKSlVlaDlubzVZCnUxc2U2YUxtRTkrZ0lMQWZjeHpXWmYzUWNOazlUVFU0R2JGRFMwM3VNc1JrZERxQklIZnZnTmJKT09hTDhuQU4KTFRUZkpxTStsblp3am1HUkk0OFVEMUtsRTc4eUlnb2xZRVgya2NBWmFQaVZKdVg0cGQ2OFFCYnBtTHpkbTQ4ZAprNmhtTjV1UDlKOENRSTFQcEEvUThRUGIzR2xhYnB6d0VXeEFqM3RTQ1dHZktnd0NXZnVYeWRxeXRHcWRpM1FKCm9Cb0UvcStjRXE0VmZlNnQ3Zi9ZWFprWFI3WEhHbVNNSnhBdG1mcWdERm9ZVzBkMWFaejRva1RieE5Qb0RuZnMKNGpIUFQ5RUNnWUVBMnQyVzRtdm9JcGU2TFFjc1A4R3dGMUkvdFJodDJGZnFBcDVrdDFXOHBNemFJMVJGS2FmUwpVTm14V3RkdzhRbk4yZHZFZjYrWTgrR3prTVZBaDR6dXRXbzJrZjNpUDJHRXZ2VjFVSlFoSittOVNSVktNUnFzCjhJWVF5VlZhSjNwTFNncGFoK252dVYzWktzZWxyK0hpVlZzaFlwcFJwSy9yajVBNW1Vdm85YThDZ1lFQTJ5Y1EKdXRMQnY1Q3F2VmFpb2tGcHBHRURHdUtDb2xpQ082Ny9PdDNwdDBZY0ZxZTdQSEVaREQrOTNOY2d1SGhTZ2tKYQo2cUMxdUxXdVNJMmFlVjZtT3ZkZW9QNkFleGNOa2J2QnJzWmp3cWhId05zMnpOMEV3WFI2QnZoenpFYWRXbnBKCjErK1ZCOXIyems2MnU0c0xCQkhJNGxGbnFRc3dqNFR3WnNVL2huc0NnWUFRNTZpdDJIUFhxQ3lES2RpckhBVEQKNkdCdlZuRzFraVRMQ3Z6d2hEakw2ZDYzZDl6WHVOU3YxZ0N5NlkwWnNRNHJVNlE3MEk1WjJBY01tWS9BYUl4UAppdUtvS0JuL2wwS3ZsVG40ZlZpS1FxMlNKWWpUZmlINWVnRjE2NWZKRU5NYjV1elRMcWlMZ3NveFdiWmNmWEFECjg4UUNYN1VLcVdQaWROdGVFZFR1bFFLQmdRQy9HTXA2emlxS1c2QzJwMlA3Ym1TckpManJhWmFWNyszb0UwbVYKMEVuck9XUm0wTzN3MzRJQzcxWWt5eVpLRlFtRmNrMExDUVpNd1lmdTFpTmNNQTl0bmFUVFova3EzRWhGeXQ0eQpBK2FicDllaHNNV1JVQ3pFY0VlejZsUTNUV0FLdEZYTnlLbEFoMHZ0VERBRlMxSll3eFZUVzc2elpnRjdOR1hZCmg2bEVId0tCZ1FDN1hWN3JFRmNUZUw1NTZQcEYyYnJNNDRsNVBxMFBPYVpucklTZHhybk9sNzFmNGZMeWJkcWEKRTcxNTRrc2REYU5XNWNOZUExVURFbjFaRFZOeGRwUjhDYXZ1RjU1UGZKVmtpby9rUVd4anVJSXhaRG0wQmNiaAppVGtHRno1MTJ4WTVaaCtTNURPMnNQY2VDd3hFWnErd05LUEZNZWdHVEw3Rkk5RlZ6dWxXaGc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=