跨公网解决K8s组网

需要开放安全组

工作节点开放组

开放端口号	开放端口类型	备注
8472	UDP	Flannel vxlan 模式下的Overlay 网络通信
10250	TCP	kubelet log exec 等端口

控制面开放组

开放端口号	开放端口类型	备注
8472	UDP	Flannel vxlan 模式下的Overlay 网络通信
10250	TCP	kubelet log exec 等端口
6443	TCP	apiserver端口
2380	TCP	etcd端口

安装部署Master

创建虚拟网卡

# 公网ip
public_ip=xxx
sudo ip link add dummy-pub type dummy
sudo ip addr add $public_ip/32 dev dummy-pub

公网IP启动apiserver

参考:https://piwriw.github.io/2024/01/08/cloud/k8s/K8s%E5%85%AC%E7%BD%91%E9%83%A8%E7%BD%B2/

• 通过输出的master 加入token 节点加入到集群中

修改node-ip

 vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
 # example :Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=47.107.113.111"

# 重启kubelet
 systemctl daemon-reload
 systemctl restart kubelet
# 清除iptables
 iptables -F
 iptables -X
 iptables -t nat -F
 iptables -t nat -X
 # 重启docker
 systemctl restart docker

部署flannel插件

---
kind: Namespace
apiVersion: v1
metadata:
  name: kube-flannel
  labels:
    k8s-app: flannel
    pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  labels:
    k8s-app: flannel
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-flannel
---
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    k8s-app: flannel
  name: flannel
  namespace: kube-flannel
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-flannel
  labels:
    tier: node
    k8s-app: flannel
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "EnableNFTables": false,
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds
  namespace: kube-flannel
  labels:
    tier: node
    app: flannel
    k8s-app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugin
        image: docker.io/flannel/flannel-cni-plugin:v1.4.0-flannel1
        command:
        - cp
        args:
        - -f
        - /flannel
        - /opt/cni/bin/flannel
        volumeMounts:
        - name: cni-plugin
          mountPath: /opt/cni/bin
      - name: install-cni
        image: docker.io/flannel/flannel:v0.25.0
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: docker.io/flannel/flannel:v0.25.0
        command:
        - /opt/bin/flanneld
        args:
        - --public-ip=$(PUBLIC_IP)
        - --iface=eth0
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN", "NET_RAW"]
        env:
        - name: PUBLIC_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: EVENT_QUEUE_DEPTH
          value: "5000"
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
        - name: xtables-lock
          mountPath: /run/xtables.lock
      volumes:
      - name: run
        hostPath:
          path: /run/flannel
      - name: cni-plugin
        hostPath:
          path: /opt/cni/bin
      - name: cni
        hostPath:
          path: /etc/cni/net.d
      - name: flannel-cfg
        configMap:
          name: kube-flannel-cfg
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate

work节点

 vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
 # example :Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=47.107.113.111"

# 重启kubelet
 systemctl daemon-reload
 systemctl restart kubelet
# 清除iptables
 iptables -F
 iptables -X
 iptables -t nat -F
 iptables -t nat -X
 # 重启docker
 systemctl restart docker

测试

kubectl测试10250端口

kuebctl logs po 
# 检查是否能输出日志

测试svc和po网络

通过访问pod ip和nodeport来测试是否已经可达。

# nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:latest
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  selector:
    app: nginx
  ports:
    - protocol: TCP
      port: 80
      targetPort: 80
      nodePort: 30080
  type: NodePort

调试

 systemctl daemon-reload
 systemctl restart kubelet
# 清除iptables
 iptables -F
 iptables -X
 iptables -t nat -F
 iptables -t nat -X
 # 重启docker
 systemctl restart docker
 

如果work-node不是用kubecetl 的node-ip修改

可以用iptabels 映射

:star:添加节点iptable映射转换

  # 此时的internal_ip 和 public_ip 是节点是IP 而不是master 
iptables -t nat -I OUTPUT -d $internal_ip/32 -j DNAT --to-destination $public_ip