K8S 高可用集群安装

K8S 高可用集群安装主机名称地址安装组件k8s-m110.4.7.21docker、nginx、keepalived、control plane、work nodek8s-m210.4.7.22d

K8S 高可用集群安装

Nginx 负载均衡

四层代理

apt install nginx keepalived -ynmkdir /etc/nginx/tcp.d/necho 'include /etc/nginx/tcp.d/*.conf;' >>/etc/nginx/nginx.confncat >/etc/nginx/tcp.d/apiserver.conf <<EOFnstream {n upstream kube-apiserver {n server 10.4.7.21:6443 max_fails=3 fail_timeout=30s;n server 10.4.7.22:6443 max_fails=3 fail_timeout=30s;n server 10.4.7.23:6443 max_fails=3 fail_timeout=30s;n }n server {n listen 7443;n proxy_connect_timeout 2s;n proxy_timeout 900s;n proxy_pass kube-apiserver;n }n}nEOF

七层代理

cat >/etc/nginx/conf.d/k8s.com.conf <<EOFnupstream default_backend_traefik {n server 10.4.7.21:80 max_fails=3 fail_timeout=10s;n server 10.4.7.22:80 max_fails=3 fail_timeout=10s;n server 10.4.7.23:80 max_fails=3 fail_timeout=10s;n}nserver {n server_name *.k8s.com;n n location / {n proxy_pass http://default_backend_traefik;n proxy_set_header Host $http_host;n proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;n }n}nEOFn​nnginx -tnnginx -s reloadn​n# Master 同traefik 自签tlsncp ~/tls.* .n# Master2,3nscp root@10.4.7.21:/root/tls.* .n​ncat >/etc/nginx/conf.d/dashboard.k8s.com.conf <<'EOF'nserver {n listen 80;n server_name dashboard.k8s.com;n​n rewrite ^(.*)$ https://${server_name}$1 permanent;n}nserver {n listen 443 ssl;n server_name dashboard.k8s.com;n​n ssl_certificate "certs/tls.crt";n ssl_certificate_key "certs/tls.key";n ssl_session_cache shared:SSL:1m;n ssl_session_timeout 10m;n ssl_ciphers HIGH:!aNULL:!MD5;n ssl_prefer_server_ciphers on;n​n location / {n proxy_pass http://default_backend_traefik;n proxy_set_header Host $http_host;n proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;n }n}nEOF

Keepalived 高可用

创建端口监测脚本

cat >/etc/keepalived/check_port.sh <<'EOF'n#!/bin/bashn#keepalived 监控端口脚本n#使用方法：等待keepalived传入端口参数,检查改端口是否存在并返回结果nCHK_PORT=$1nif [ -n "$CHK_PORT" ];thenn PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`n if [ $PORT_PROCESS -eq 0 ];thenn echo "Port $CHK_PORT Is Not Used,End."n exit 1n finelsen echo "Check Port Cant Be Empty!"nfinEOFn​nchmod +x /etc/keepalived/check_port.shn# keepalived主ncat >/etc/keepalived/keepalived.conf <<'EOF'n! Configuration File for keepalivednglobal_defs {n router_id 10.4.7.21n}nvrrp_script chk_nginx {n script "/etc/keepalived/check_port.sh 7443"n interval 2n weight -20n}nvrrp_instance VI_1 {n state MASTERn interface ens33n virtual_router_id 251n priority 100n advert_int 1n mcast_src_ip 10.4.7.21n nopreemptn​n authentication {n auth_type PASSn auth_pass 11111111n }n track_script {n chk_nginxn }n virtual_ipaddress {n 10.4.7.20n }n}nEOFn# keepalived从ncat >/etc/keepalived/keepalived.conf <<'EOF'n! Configuration File for keepalivednglobal_defs {n router_id 10.4.7.22n}nvrrp_script chk_nginx {n script "/etc/keepalived/check_port.sh 7443"n interval 2n weight -20n}nvrrp_instance VI_1 {n state BACKUPn interface ens33n virtual_router_id 251n mcast_src_ip 10.4.7.22n priority 90n advert_int 1n authentication {n auth_type PASSn auth_pass 11111111n }n track_script {n chk_nginxn }n virtual_ipaddress {n 10.4.7.20n }n}nEOFn​n# 10.4.7.23 同 10.4.7.22 nsystemctl start keepalivednsystemctl enable keepalivednip addr|grep '10.4.7.20'

系统检查

# 修改主机名称nhostnamectl set-hostname k8s-m1n# 修改hosts文件necho "10.4.7.21 k8s-m1" >> /etc/hostsnecho "10.4.7.22 k8s-m2" >> /etc/hostsnecho "10.4.7.23 k8s-m3" >> /etc/hostsn# 查看ufw状态nufw statusn# 临时关闭swapnswapoff -an# 永久关闭swapnvi /etc/fstabn# /swap.img none swap sw 0 0nfree -mn# 同时调整k8s的swappiness参数necho "vm.swappiness=0" >> /etc/sysctl.d/k8s.confnsysctl -p /etc/sysctl.d/k8s.confn​n# Kubectl自动补全nsource <(kubectl completion bash)

Docker-CE

# 国内镜像ncurl -fsSL https://get.docker.com | bash -s docker --mirror Aliyunn# 查看dockers版本ndocker --versionn# 给普通用户添加权限nsudo usermod -aG docker $USERn# docker阿里云加速器ncurl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s https://b33dfgq9.mirror.aliyuncs.comn# 修改进程隔离工具n# docker默认cgroupfs k8s使用systemdnvim /etc/docker/daemon.jsonn{n...n "exec-opts": [ "native.cgroupdriver=systemd" ]n}n# 重启dockernsystemctl daemon-reload && systemctl restart docker

kubeadm

# 安装httpsnapt-get update && apt-get install -y apt-transport-httpsn# 添加密钥ncurl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -n# 添加源nadd-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main"n# 更新源napt-get updaten# 查看最新版本napt-cache madison kubelet kubectl kubeadmn# 安装指定的版本napt install -y kubelet kubectl kubeadmn# 查看kubelet版本nkubectl version --client=true -o yamln# 配置kubelet禁用swapnecho 'KUBELET_EXTRA_ARGS="--fail-swap-on=false"' > /etc/default/kubeletn# 重启kubeletnsystemctl daemon-reload && systemctl restart kubeletn# 查看kubelet服务启动状态(因缺少缺少很多参数配置文件,需要等待kubeadm init 后生成)njournalctl -u kubelet -f

初始化k8s-m1

cat > kubeadm-config.yaml <<EOFnapiVersion: kubeadm.k8s.io/v1beta2nkind: ClusterConfigurationnkubernetesVersion: v1.22.1nimageRepository: registry.aliyuncs.com/google_containersncontrolPlaneEndpoint: "10.4.7.21:6443"napiServer:n certSANs: #填写所有kube-apiserver节点的hostname、IP、VIPn - k8s-m1n - k8s-m2n - k8s-m3n - k8s-w1n - k8s-w2n - k8s-w3n - apiserver.lbn - 10.4.7.20nnetworking:n serviceSubnet: "10.96.0.0/16"n podSubnet: "10.244.0.0/16"n dnsDomain: "cluster.local"nEOFn​n# kubeadm initn# 根据您服务器网速的情况，您需要等候 3 - 10 分钟nkubeadm init --config=kubeadm-config.yaml --upload-certsn​n# docker pull corednsndocker pull registry.aliyuncs.com/google_containers/coredns:1.8.4ndocker tag registry.aliyuncs.com/google_containers/coredns:1.8.4 registry.aliyuncs.com/google_containers/coredns:v1.8.4n​n# 配置 kubectlnrm -rf /root/.kube/nmkdir /root/.kube/ncp -i /etc/kubernetes/admin.conf /root/.kube/confign​n# 安装 flannel 网络插件nkubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.ymln​n# master 去除污点nkubectl taint nodes --all node-role.kubernetes.io/master-

k8s-m2,k8s-m3

kubeadm join 10.4.7.21:6443 --token swcdwc.etn06vu7t11r1ywq n --discovery-token-ca-cert-hash sha256:0df815a693038402057acb22f2698375d563a03fd16e1050933d25210bda6f5b n --control-plane --certificate-key d192afd75926b20289a6a489272e161c4f4962c97de4d09dd76cc5d89463ba3c

Ingress-traefik

https://github.com/containous/traefik

crd

# traefik-crd.yaml n## IngressRoutenapiVersion: apiextensions.k8s.io/v1beta1nkind: CustomResourceDefinitionnmetadata:n name: ingressroutes.traefik.containo.usnspec:n scope: Namespacedn group: traefik.containo.usn version: v1alpha1n names:n kind: IngressRouten plural: ingressroutesn singular: ingressrouten---n## IngressRouteTCPnapiVersion: apiextensions.k8s.io/v1beta1nkind: CustomResourceDefinitionnmetadata:n name: ingressroutetcps.traefik.containo.usnspec:n scope: Namespacedn group: traefik.containo.usn version: v1alpha1n names:n kind: IngressRouteTCPn plural: ingressroutetcpsn singular: ingressroutetcpn---n## MiddlewarenapiVersion: apiextensions.k8s.io/v1beta1nkind: CustomResourceDefinitionnmetadata:n name: middlewares.traefik.containo.usnspec:n scope: Namespacedn group: traefik.containo.usn version: v1alpha1n names:n kind: Middlewaren plural: middlewaresn singular: middlewaren---n## TLSOptionnapiVersion: apiextensions.k8s.io/v1beta1nkind: CustomResourceDefinitionnmetadata:n name: tlsoptions.traefik.containo.usnspec:n scope: Namespacedn group: traefik.containo.usn version: v1alpha1n names:n kind: TLSOptionn plural: tlsoptionsn singular: tlsoptionn---n## TraefikServicenapiVersion: apiextensions.k8s.io/v1beta1nkind: CustomResourceDefinitionnmetadata:n name: traefikservices.traefik.containo.usnspec:n scope: Namespacedn group: traefik.containo.usn version: v1alpha1n names:n kind: TraefikServicen plural: traefikservicesn singular: traefikservice

rbac

# traefik-rbac.yaml n## ServiceAccountnapiVersion: v1nkind: ServiceAccountnmetadata:n namespace: kube-systemn name: traefik-ingress-controllern---n## ClusterRolenkind: ClusterRolenapiVersion: rbac.authorization.k8s.io/v1beta1nmetadata:n name: traefik-ingress-controllernrules:n - apiGroups: [""]n resources: ["services","endpoints","secrets"]n verbs: ["get","list","watch"]n - apiGroups: ["extensions"]n resources: ["ingresses"]n verbs: ["get","list","watch"]n - apiGroups: ["extensions"]n resources: ["ingresses/status"]n verbs: ["update"]n - apiGroups: ["traefik.containo.us"]n resources: ["middlewares"]n verbs: ["get","list","watch"]n - apiGroups: ["traefik.containo.us"]n resources: ["ingressroutes"]n verbs: ["get","list","watch"]n - apiGroups: ["traefik.containo.us"]n resources: ["ingressroutetcps"]n verbs: ["get","list","watch"]n - apiGroups: ["traefik.containo.us"]n resources: ["tlsoptions"]n verbs: ["get","list","watch"]n - apiGroups: ["traefik.containo.us"]n resources: ["traefikservices"]n verbs: ["get","list","watch"]n---n## ClusterRoleBindingnkind: ClusterRoleBindingnapiVersion: rbac.authorization.k8s.io/v1beta1nmetadata:n name: traefik-ingress-controllernroleRef:n apiGroup: rbac.authorization.k8s.ion kind: ClusterRolen name: traefik-ingress-controllernsubjects:n - kind: ServiceAccountn name: traefik-ingress-controllern namespace: kube-system

configMap

# traefik-config.yamlnkind: ConfigMapnapiVersion: v1nmetadata:n name: traefik-configndata:n traefik.yaml: |-n ping: "" ## 启用 Pingn serversTransport:n insecureSkipVerify: true ## Traefik 忽略验证代理服务的 TLS 证书n api:n insecure: true ## 允许 HTTP 方式访问 APIn dashboard: true ## 启用 Dashboardn debug: false ## 启用 Debug 调试模式n metrics:n prometheus: "" ## 配置 Prometheus 监控指标数据，并使用默认配置n entryPoints:n web:n address: ":80" ## 配置 80 端口，并设置入口名称为 webn websecure:n address: ":443" ## 配置 443 端口，并设置入口名称为 websecuren providers:n kubernetesCRD: "" ## 启用 Kubernetes CRD 方式来配置路由规则n kubernetesIngress: "" ## 启动 Kubernetes Ingress 方式来配置路由规则n log:n filePath: "" ## 设置调试日志文件存储路径，如果为空则输出到控制台n level: error ## 设置调试日志级别n format: json ## 设置调试日志格式n accessLog:n filePath: "" ## 设置访问日志文件存储路径，如果为空则输出到控制台n format: json ## 设置访问调试日志格式n bufferingSize: 0 ## 设置访问日志缓存行数n filters:n #statusCodes: ["200"] ## 设置只保留指定状态码范围内的访问日志n retryAttempts: true ## 设置代理访问重试失败时，保留访问日志n minDuration: 20 ## 设置保留请求时间超过指定持续时间的访问日志n fields: ## 设置访问日志中的字段是否保留（keep 保留、drop 不保留）n defaultMode: keep ## 设置默认保留访问日志字段n names: ## 针对访问日志特别字段特别配置保留模式n ClientUsername: drop n headers: ## 设置 Header 中字段是否保留n defaultMode: keep ## 设置默认保留 Header 中字段n names: ## 针对 Header 中特别字段特别配置保留模式n User-Agent: redactn Authorization: dropn Content-Type: keep

Set Lable

kubectl label nodes all IngressProxy=truenkubectl get nodes --show-labels

ds资源清单

# traefik-deploy.yamlnapiVersion: v1nkind: Servicenmetadata:n name: traefiknspec:n ports:n - name: webn port: 80n - name: websecuren port: 443n - name: adminn port: 8080n selector:n app: traefikn---napiVersion: apps/v1nkind: DaemonSetnmetadata:n name: traefik-ingress-controllern labels:n app: traefiknspec:n selector:n matchLabels:n app: traefikn template:n metadata:n name: traefikn labels:n app: traefikn spec:n serviceAccountName: traefik-ingress-controllern terminationGracePeriodSeconds: 1n containers:n - image: traefik:v2.1.2n name: traefik-ingress-lbn ports:n - name: webn containerPort: 80n hostPort: 80 ## 将容器端口绑定所在服务器的 80 端口n - name: websecuren containerPort: 443n hostPort: 443 ## 将容器端口绑定所在服务器的 443 端口n - name: adminn containerPort: 8080 ## Traefik Dashboard 端口n resources:n limits:n cpu: 2000mn memory: 1024Min requests:n cpu: 1000mn memory: 1024Min securityContext:n capabilities:n drop:n - ALLn add:n - NET_BIND_SERVICEn args:n - --configfile=/config/traefik.yamln - --kubernetes.endpoint=https://10.4.7.20:7443n volumeMounts:n - mountPath: "/config"n name: "config"n volumes:n - name: confign configMap:n name: traefik-config n tolerations: ## 设置容忍所有污点，防止节点被设置污点n - operator: "Exists"n nodeSelector: ## 设置node筛选器，在特定label的节点上启动n IngressProxy: "true"

IngressRoute

# traefik-dashboard-route.yamlnapiVersion: traefik.containo.us/v1alpha1nkind: IngressRoutenmetadata:n name: traefik-dashboard-routenspec:n entryPoints:n - webn routes:n # 这里设置你的域名n - match: Host(`traefik.k8s.com`)n kind: Rulen services:n - name: traefikn port: 8080nopenssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=cert.k8s.com"n​nkubectl create secret tls https --cert=tls.crt --key=tls.key -n kube-systemn​nkubectl apply -f traefik-crd.yamlnkubectl apply -f traefik-rbac.yaml -n kube-systemnkubectl apply -f traefik-config.yaml -n kube-systemnkubectl apply -f traefik-deploy.yaml -n kube-systemnkubectl apply -f traefik-dashboard-route.yaml -n kube-system

kubernetes-dashboard

kubernetes/dashboard

GITHUB_URL=https://github.com/kubernetes/dashboard/releasesnVERSION_KUBE_DASHBOARD=$(curl -w '%{url_effective}' -I -L -s -S ${GITHUB_URL}/latest -o /dev/null | sed -e 's|.*/||')nsudo k8s kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/${VERSION_KUBE_DASHBOARD}/aio/deploy/recommended.yaml

rbac.yaml

apiVersion: v1nkind: ServiceAccountnmetadata:n name: admin-usern namespace: kube-systemn​n---napiVersion: rbac.authorization.k8s.io/v1beta1nkind: ClusterRoleBindingnmetadata:n name: admin-usernroleRef:n apiGroup: rbac.authorization.k8s.ion kind: ClusterRolen name: cluster-adminnsubjects:n- kind: ServiceAccountn name: admin-usern namespace: kube-systemnkubectl -n kube-system describe secret admin-user-token | grep '^token'

ingress.yaml

apiVersion: traefik.containo.us/v1alpha1nkind: IngressRoutenmetadata:n name: kubernetes-dashboard-routen namespace: kubernetes-dashboardn annotations:n kubernetes.io/ingress.class: traefiknspec:n entryPoints:n - websecuren routes:n # 这里设置你的域名n - match: Host(`dashboard.k8s.com`)n kind: Rulen services:n - name: kubernetes-dashboardn port: 443n tls:n secretName: https

集群访问

echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profilensource ~/.bash_profile

测试集群

kubectl create deployment nginx-app --image=nginx --replicas=3n​nkubectl get pod -o widenkubectl get deploymentnkubectl expose deployment nginx-app --port=80 --type=NodePortnkubectl get svcncurl 10.4.7.21:32738