编译Kubemark

背景

Kubemark是k8s虚拟化节点的工具，用于测试。

目前找不到现成可用Kubamark镜像，所以自行编译。

安装

1
2
3
4
5
# gcc 环境准备
yum -y install gcc gcc-c++ autoconf pcre pcre-devel make automake                                                                                                                   
yum -y install wget httpd-tools vim 
yum -y install glibc-static.x86_64 
yum -y install binutils

Go环境准备：

LInuxGo部署过程

源码构建：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 需要安装git
yum install -y git 
# 拉取k8s代码 
git clone https://github.com/kubernetes/kubernetes.git
# 查看所有分支
git branch -a
# 强制切换到git 编译k8s分支
git checkout -f remotes/origin/release-1.24
# 
make WHAT='cmd/kubemark'

cp _output/bin/kubemark cluster/images/kubemark/
cd cluster/images/kubemark/
# 会拉取基础镜像，国内不能访问
# 修改为国内代理镜像
vim Dockerfile

make build

部署节点：

按照官网会出现缺少sh、bash的错误，因为google基础镜像没有这些

先给需要work节点打上label

1
kubectl label node kubemark=work

hollow-node_simplified_template.yaml:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
apiVersion: v1
kind: ReplicationController
metadata:
  name: hollow-node
  namespace: kubemark
spec:
  replicas: 50  # the node number you want to simulate
  selector:
    name: hollow-node
  template:
    metadata:
      labels:
        name: hollow-node
    spec:
      nodeSelector:  # leverage label to allocate to native node
        kubemark: work
      initContainers:
        - name: init-inotify-limit
          image: docker.io/busybox:latest
          imagePullPolicy: IfNotPresent
          command: ['sysctl', '-w', 'fs.inotify.max_user_instances=200'] # set as same as max_user_instance in actual node
          securityContext:
            privileged: true
      volumes:
        - name: rci-volume
          hostPath:
            path: /run/containerd
        - name: kubeconfig-volume
          secret:
            secretName: kubeconfig
        - name: logs-volume
          hostPath:
            path: /var/log
      containers:
        - name: hollow-kubelet
          image: piwriw/kubemark:v1.21 # the kubemark image you build
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 4194
            - containerPort: 10250
            - containerPort: 10255
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
          command:
            - /kubemark
          args:
            - --morph=kubelet
            - --name=$(NODE_NAME)
            - --kubeconfig=/kubeconfig/kubelet.kubeconfig
            - --alsologtostderr
            - --v=2
          volumeMounts:
            - name:  rci-volume
              mountPath: /run/containerd
            - name: kubeconfig-volume
              mountPath: /kubeconfig
              readOnly: true
            - name: logs-volume
              mountPath: /var/log
          resources:
            requests:    # the resource of hollow pod, can modify it.
#              cpu: 20m
              memory: 50M
          securityContext:
            privileged: true
        - name: hollow-proxy
          image: piwriw/kubemark:v1.21 # the kubemark image you build
          imagePullPolicy: IfNotPresent
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
          command:
            - /kubemark
          args:
            - --morph=proxy
            - --name=$(NODE_NAME)
            - --use-real-proxier=false
            - --kubeconfig=/kubeconfig/kubeproxy.kubeconfig
            - --alsologtostderr
            - --v=2
          volumeMounts:
            - name: kubeconfig-volume
              mountPath: /kubeconfig
              readOnly: true
            - name: logs-volume
              mountPath: /var/log
          resources:  # the resource of hollow pod, can modify it.
            requests:
#              cpu: 20m
              memory: 50M
      tolerations:
        - effect: NoExecute
          key: node.kubernetes.io/unreachable
          operator: Exists
        - effect: NoExecute
          key: node.kubernetes.io/not-ready
          operator: Exists

问题 rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.ImageService

1
2
3
4
5
6
7
8

cat > /etc/containerd/config.toml <<EOF
[plugins."io.containerd.grpc.v1.cri"]
systemd_cgroup = true
EOF

systemctl restart containerd