| k8s

k8s安装

记录 k8s最新版 v1.28 (截至2023/8/18) 安装过程

虚拟机装备

三台虚拟机：

  • 主节点 ：k8s-master (2.8G 2C 20GB、192.168.64.211）

  • node节点：k8s-node01 (2.4G 1C 20GB、192.168.64.212)

  • node节点：k8s-node02 (2.4G 1C 20GB、192.168.64.210)

安装Docker

容器运行时环境使用Docker，只是在k8s v1.24以后需要额外安装cri-dockerd, k8s才能够正常识别到Docker。

1
2
3
4
5
6
7
8
9
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
systemctl start docker
systemctl enable docker
systemctl status docker
#验证
docker pull hello-world
docker run hello-world

安装cri-dockerd

1
2
3
4
5
6
wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.4/cri-dockerd-0.3.4-3.el7.x86_64.rpm
rpm -ivh cri-dockerd-0.3.4-3.el7.x86_64.rpm
systemctl daemon-reload
systemctl enable cri-docker.socket cri-docker
systemctl start cri-docker.socket cri-docker
systemctl status cri-docker.socket

安装kubernetes

  • 安装kubectl

    1
    2
    3
    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
    install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
    kubectl version --client

  • 安装kubeadm

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    systemctl stop firewalld
    systemctl disable firewalld 
    setenforce 0
    sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
    cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
    [kubernetes]
    name=Kubernetes
    baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
    enabled=1
    gpgcheck=0
    repo_gpgcheck=1
    gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
            http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
    exclude=kubelet kubeadm kubectl
    EOF
    yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
    systemctl enable --now kubelet

搭建K8s集群

所有机器都安装了容器运行时（Docker + cri-dockerd）以及kubernetes组件kubectl, kubeadm和kubelet。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
swapoff -a 
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
#安装runc
wget https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64
sudo install -m 755 runc.amd64 /usr/local/bin/runc
runc -v
sudo tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://tsvqojsz.mirror.aliyuncs.com"]
}
EOF
vi /usr/lib/systemd/system/cri-docker.service # 找到第10行ExecStart= 修改为
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.7
systemctl daemon-reload && systemctl restart docker cri-docker.socket cri-docker
systemctl status docker cri-docker.socket cri-docker
cat /etc/hostname
vi /etc/hosts
192.168.64.211 k8s-master
192.168.64.212 k8s-node01
192.168.64.210 k8s-node02

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

sudo sysctl --system
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

k8s-master执行初始化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
kubeadm init --node-name=k8s-master --image-repository=registry.aliyuncs.com/google_containers --cri-socket=unix:///var/run/cri-dockerd.sock --apiserver-advertise-address=192.168.64.211 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12
#重置kubeadm 比如之前有安装报错的
kubeadm reset   --cri-socket=unix:///var/run/cri-dockerd.sock
export KUBECONFIG=/etc/kubernetes/admin.conf
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >>  ~/.bash_profile
source ~/.bash_profile
#安装 flannel 网络插件
wget https://github.com/flannel-io/flannel/releases/download/v0.22.0/kube-flannel.yml
vi kube-flannel.yml
container:
    ......
    command：
    -  /opt/bin/flanneld
    arg:
    - --ip-masq
    - --kube-subnet-mgr
    - --iface=ens33
增加 - --iface=ens33 
kubectl apply -f kube-flannel.yml
vi /run/flannel/subnet.env
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

node节点加入master

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
scp /etc/kubernetes/admin.conf k8s-node01:/etc/kubernetes/
scp /etc/kubernetes/admin.conf k8s-node02:/etc/kubernetes/
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >>  ~/.bash_profile
source ~/.bash_profile

kubectl get nodes 如下报错，检查上面配置
E0704 21:50:22.095009   28309 memcache.go:265] couldn't get current server API group list: Get "http://localhost:8080/api?timeout=32s": dial tcp [::1]:8080: connect: connection refused

kubeadm join 192.168.64.211:6443 --token 4jqgg8.jqxz4pdzrbquecxb  --discovery-token-ca-cert-hash sha256:b2d7e30e8b3b11d6440f7c40159ac8282592be2b114f75443f455e91a540b029  --cri-socket=unix:///var/run/cri-dockerd.sock

kubectl get nodes
NAME         STATUS   ROLES           AGE   VERSION
k8s-master   Ready    control-plane   69m   v1.28.0
k8s-node01   Ready    <none>          45m   v1.28.0
k8s-node02   Ready    <none>          40m   v1.28.0

kubectl get pod -A
NAMESPACE      NAME                                 READY   STATUS    RESTARTS      AGE
kube-flannel   kube-flannel-ds-4cg4v                1/1     Running   1 (33m ago)   41m
kube-flannel   kube-flannel-ds-phc42                1/1     Running   2 (31m ago)   45m
kube-flannel   kube-flannel-ds-rpmqg                1/1     Running   1 (32m ago)   51m
kube-system    coredns-66f779496c-dqnqc             1/1     Running   1 (32m ago)   69m
kube-system    coredns-66f779496c-hssbb             1/1     Running   1 (32m ago)   69m
kube-system    etcd-k8s-master                      1/1     Running   4 (33m ago)   69m
kube-system    kube-apiserver-k8s-master            1/1     Running   4 (32m ago)   69m
kube-system    kube-controller-manager-k8s-master   1/1     Running   4 (33m ago)   69m
kube-system    kube-proxy-d2m2m                     1/1     Running   1 (33m ago)   45m
kube-system    kube-proxy-gqwjh                     1/1     Running   1 (33m ago)   41m
kube-system    kube-proxy-jkx9q                     1/1     Running   1 (33m ago)   69m
kube-system    kube-scheduler-k8s-master            1/1     Running   4 (32m ago)   69m

K8s常用命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
kubectl get pods 
kubectl get pods -n xxcm #查看指定空间下的pod
kubectl get pods -o wide #查看pod 带ip
kubectl describe pod xxpd  #查看某个pod
kubectl logs xxpd #查看pod中容器日志
kubectl exec -it podName  -c  containerName -n namespace bash
kubectl get service mongodb -o yaml
kubectl port-forward service/test-k8s 8888:8080
kubectl cp -n 命名空间 -c 容器名称 pod名称:app/docker_run.jar ./xxx.jar
kubectl apply -f app.yaml
kubectl get deployment
kubectl scale deployment test-k8s --replicas=5
kubectl port-forward pod-name 8090:8080

kubectl get nodes
kubectl get namespaces #空间名
kubectl get events
kubectl get cs #集群健康情况
kubectl cluster-info #运行情况
kubectl version 
kubectl api-versions
kubectl api-resources
kubectl get node --show-labels
kubectl get configmap -n xxns #查看指定空间下的configmap
kubectl describe configmap xxcm
kubectl edit configmap xxcm

kubectl create ns test
kubectl run httpd-app --image=reg.tstack.com/tstack/httpd:latest --replicas=2
kubectl create deployment nginx --image=nginx --port=8080 --replicas=3 

kubectl label  node 172.16.254.23 disktype=ssd
kubectl get node --show-labels

kubectl get cm
kubectl edit cm xxx-config
kubectl create configmap myconfigmap --from-literal=config1=xxx --from-literal=config2=yyy
kubectl create configmap myconfigmap --from-literal=./config1 --from-literal=./config2
kubectl create configmap myconfigmap --from-env-file=env.txt

kubectl get ingress
kubectl get ingress xx -o yaml
kubectl get svc XXX -O yaml

kubectl get pv 
kubectl get pvc

docker images
docker rmi id
docker import algo_centos7_container.tar centos7.4:centos7.4.1708
docker export id > centos7.tar
docker commit  容器ID  镜像名
docker tag 镜像名   niuey2001/vue:v1

vi Dockerfile
FROM nginx
COPY dist/  /usr/share/nginx/html/
docker build -t  niuey2001/vue:v1 .
docker build -t ccr.ccs.tencentyun.com/tsf_100010933916/tsf-apps:v1 .
docker login --username=xxxxx  ccr.ccs.tencentyun.com/xxxxx/tsf-apps
docker pull ccr.ccs.tencentyun.com/tsf_100010933916/tsf-app:v1

docker run --name nginx -d -p 4030:80 nginx
docker ps -a
docker stop cid

ESC