双流本地k8s搭建

配置表

| ip | 名称 | 配置 |
| --- | --- | --- |
| 10.2.8.111 | sl-k8s-master-01 | 4c8g |
| 10.2.8.112 | sl-k8s-master-02 | 4c8g |
| 10.2.8.113 | sl-k8s-master-03 | 4c8g |
| 10.2.8.114 | sl-k8s-node-01 | 8c16g |
| 10.2.8.115 | sl-k8s-node-02 | 8c16g |
| 10.2.8.116 | sl-k8s-node-03 | 8c16g |

搭建高可用负载均衡

思考了一下还是沿用以前的 keepalived+haproxy

比较方便

问了ai之所以大家选haproxy不选nginx 是因为四层负载ha更好，而且ha只做这个，所以选ha会更好

安装docker

```shell
sudo apt remove docker docker-engine docker.io containerd runc

sudo apt update && sudo apt upgrade -y
sudo apt install -y ca-certificates curl gnupg lsb-release

sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg

echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

sudo apt update
sudo apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin

sudo systemctl enable docker.service
sudo systemctl enable containerd.service
```

mkdir /etc/haproxy

# haproxy.cfg

```shell
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4096
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option                  http-server-close
    option                  forwardfor    except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

frontend  kube-apiserver
    mode                 tcp
    bind                 *:9443
    option               tcplog
    default_backend      kube-apiserver

listen stats
    mode                 http
    bind                 *:8888
    stats auth           admin:password
    stats refresh        5s
    stats realm          HAProxy\ Statistics
    stats uri            /stats
    log                  127.0.0.1 local3 err

backend kube-apiserver
    mode        tcp
    balance     roundrobin
    server  k8s-master1 10.2.8.111:6443 check
    server  k8s-master2 10.2.8.112:6443 check
    server  k8s-master3 10.2.8.113:6443 check
```

```shell
docker run -d --name k8s-haproxy \
  --net=host \
  --restart=always \
  -v /etc/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro \
  harbor.tqsys.cn/public/haproxy-debian:2.3
```

keepalived

```shell
# keepalived.conf - master 1
! Configuration File for keepalived

global_defs {
   router_id LVS_1
}

vrrp_script checkhaproxy
{
    script "/usr/bin/check-haproxy.sh"
    interval 2
    weight -30
}

vrrp_instance VI_1 {
    state MASTER
    interface ens34
    virtual_router_id 51
    priority 100
    advert_int 1
    
    # 单播配置
    unicast_peer {
        10.2.8.112
    }

virtual_ipaddress {
        10.2.8.110/24 dev ens34
    }

authentication {
        auth_type PASS
        auth_pass password
    }

track_script {
        checkhaproxy
    }
}

# 第五步：配置keepalived master2
# keepalived.conf - master 2
! Configuration File for keepalived

global_defs {
   router_id LVS_2
}

vrrp_script checkhaproxy
{
    script "/usr/bin/check-haproxy.sh"
    interval 2
    weight -30
}

vrrp_instance VI_1 {

state BACKUP
    interface ens34
    virtual_router_id 51
    priority 90
    advert_int 1

# 单播配置
    unicast_peer {
        10.2.8.111
    }

virtual_ipaddress {
        10.2.8.110/24 dev ens34
    }

authentication {
        auth_type PASS
        auth_pass password
    }

track_script {
        checkhaproxy
    }
}

```

```shell
# 第六步：创建check-proxy.sh
#!/bin/bash

count=`netstat -apn | grep 9443 | wc -l`
if [ $count -gt 0 ]; then
    exit 0
else
    exit 1
fi
```

mkdir /etc/keepalived

# 第七步：启动keepalived

docker run -d --name k8s-keepalived \

--restart=always \

--net=host \

--cap-add=NET_ADMIN --cap-add=NET_BROADCAST --cap-add=NET_RAW \

-v /etc/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf \

-v /etc/keepalived/check-haproxy.sh:/usr/bin/check-haproxy.sh \

harbor.tqsys.cn/public/keepalived:2.0.20  --copy-service

# 搭建k8s 系统优化
```shell
#关闭swap分区
sudo sed -i '/swap/s/^$.*$$/#\1/g' /etc/fstab

#配置内核参数
#创建/etc/sysctl.d/k8s.conf

cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
EOF

#加载内核模块
sudo modprobe overlay
sudo modprobe br_netfilter

#创建/etc/modules-load.d/k8s.conf
cat > /etc/modules-load.d/k8s.conf <<EOF
overlay
br_netfilter
EOF

#应用配置
sysctl -p /etc/sysctl.d/k8s.conf

#更换时区和时钟同步
sudo timedatectl set-timezone Asia/Shanghai  # 按需替换时区
sudo apt install chrony -y
sudo systemctl enable --now chrony

#编辑/etc/security/limits.conf
cat >> /etc/security/limits.conf <<EOF
* soft nofile 65535
* hard nofile 65535
* soft nproc 65535
* hard nproc 65535

EOF

#禁用透明大页（THP）
#编辑/etc/default/grub
GRUB_CMDLINE_LINUX="... transparent_hugepage=never"

#配置IPVS（可选，提升Service性能）
sudo apt install -y ipset ipvsadm

sudo modprobe ip_vs
sudo modprobe ip_vs_rr
sudo modprobe ip_vs_wrr
sudo modprobe ip_vs_sh
sudo modprobe nf_conntrack

cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

#持久化模块（创建/etc/modules-load.d/ipvs.conf）

```

containerd配置

```shell
#修改containerd 配置
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

#这步忘记的话,后面的pod会不停重启，且找不到原因 
SystemdCgroup = true  # 必须设为 true
sudo sed -i 's#registry.k8s.io#registry.aliyuncs.com/google_containers#g' /etc/containerd/config.toml

cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
systemctl restart  containerd

```

安装kubeadm

```shell
apt-get update && apt-get install -y apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/Release.key |
    gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.30/deb/ /" |
    tee /etc/apt/sources.list.d/kubernetes.list

apt-get update

apt-cache madison kubeadm | grep 1.30

apt-get install -y kubelet=1.30.12-1.1 kubeadm=1.30.12-1.1 kubectl=1.30.12-1.1

sudo kubeadm init \
  --control-plane-endpoint=10.2.8.110:9443 \
  --apiserver-advertise-address=10.2.8.111 \
  --pod-network-cidr=10.188.0.0/16 \
  --service-cidr=10.199.0.0/16 \
  --image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
  --cri-socket unix:///run/containerd/containerd.sock \
  --upload-certs

kubeadm join 10.2.8.110:9443 --token yrgunw.23jcjh5puvvugbz6 \
	--discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
	--control-plane --certificate-key 72c298551b025590f5123de51b80d511b32cb9f7711e57bf2dca3a3ea19e9092 \
  --cri-socket unix:///run/containerd/containerd.sock

sudo kubeadm join 10.2.8.110:9443 \
  --token yrgunw.23jcjh5puvvugbz6 \
  --discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
  --control-plane \
  --certificate-key 72c298551b025590f5123de51b80d511b32cb9f7711e57bf2dca3a3ea19e9092 \
  --apiserver-advertise-address=10.2.8.112 \
  --cri-socket unix:///run/containerd/containerd.sock

sudo kubeadm join 10.2.8.110:9443 \
  --token yrgunw.23jcjh5puvvugbz6 \
  --discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
  --control-plane \
  --certificate-key 72c298551b025590f5123de51b80d511b32cb9f7711e57bf2dca3a3ea19e9092 \
  --apiserver-advertise-address=10.2.8.113 \
  --cri-socket unix:///run/containerd/containerd.sock

sudo kubeadm join 10.2.8.110:9443 \
  --token yrgunw.23jcjh5puvvugbz6 \
  --discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
  --control-plane \
  --certificate-key 72c298551b025590f5123de51b80d511b32cb9f7711e57bf2dca3a3ea19e9092 \
  --apiserver-advertise-address=10.2.8.111 \
  --cri-socket unix:///run/containerd/containerd.sock

You can now join any number of the control-plane node running the following command on each as root:

kubeadm join 10.2.8.110:9443 --token yrgunw.23jcjh5puvvugbz6 \
	--discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
	--control-plane --certificate-key 72c298551b025590f5123de51b80d511b32cb9f7711e57bf2dca3a3ea19e9092

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.2.8.110:9443 --token yrgunw.23jcjh5puvvugbz6 \
	--discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
  --cri-socket unix:///run/containerd/containerd.sock

```

# node 节点加入
```shell
#要跟master一样初始化系统
sudo apt update && sudo apt upgrade -y
sudo apt install -y containerd.io

kubeadm join 10.2.8.110:9443 --token yrgunw.23jcjh5puvvugbz6 \
	--discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 \
  --cri-socket unix:///run/containerd/containerd.sock

kubeadm join 10.2.8.110:9443 --token gjo40y.rczdbd33uqashuoh --discovery-token-ca-cert-hash sha256:a0ed4fde64a2fd60c1815535feb9a50ad5006fcbe55e913a9fa4f1ce84101ca0 --cri-socket unix:///run/containerd/containerd.sock

#安装calico
curl https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml -O

#搭建ingress
 wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml

# 删除准入控制
kubectl delete  validatingwebhookconfigurations ingress-nginx-admission

```