关于kubernetes:Kubernetesk8s实现IPv4IPv6网络双栈

背景

现在IPv4IP地址曾经应用结束，将来寰球会以IPv6地址为核心，会大力发展IPv6网络环境，因为IPv6能够实现给任何一个设施调配到公网IP，所以资源是十分丰盛的。

配置hosts

[root@k8s-master01 ~]# vim /etc/hosts[root@k8s-master01 ~]# cat /etc/hosts127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4::1         localhost localhost.localdomain localhost6 localhost6.localdomain62408:8207:78ce:7561::10 k8s-master012408:8207:78ce:7561::20 k8s-master022408:8207:78ce:7561::30 k8s-master032408:8207:78ce:7561::40 k8s-node012408:8207:78ce:7561::50 k8s-node022408:8207:78ce:7561::60 k8s-node032408:8207:78ce:7561::70 k8s-node042408:8207:78ce:7561::80 k8s-node0510.0.0.81 k8s-master0110.0.0.82 k8s-master0210.0.0.83 k8s-master0310.0.0.84 k8s-node0110.0.0.85 k8s-node0210.0.0.86 k8s-node0310.0.0.87 k8s-node0410.0.0.88 k8s-node0510.0.0.80 lb0110.0.0.90 lb0210.0.0.99 lb-vip[root@k8s-master01 ~]#

配置ipv6地址

[root@k8s-master01 ~]# vim /etc/sysconfig/network-scripts/ifcfg-ens160 [root@k8s-master01 ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens160TYPE=EthernetPROXY_METHOD=noneBROWSER_ONLY=noBOOTPROTO=noneDEFROUTE=yesIPV4_FAILURE_FATAL=noIPV6INIT=yesIPV6_AUTOCONF=noIPV6ADDR=2408:8207:78ce:7561::10/64IPV6_DEFAULTGW=2408:8207:78ce:7561::1IPV6_DEFROUTE=yesIPV6_FAILURE_FATAL=noNAME=ens160UUID=56ca7c8c-21c6-484f-acbd-349111b3ddb5DEVICE=ens160ONBOOT=yesIPADDR=10.0.0.81PREFIX=24GATEWAY=10.0.0.1DNS1=8.8.8.8DNS2=2408:8000:1010:1::8[root@k8s-master01 ~]#

留神：每一台主机都须要配置为动态IPv6地址！若不进行配置，在内核中开启IPv6数据包转发性能后会呈现IPv6异样。

sysctl参数启用ipv6

[root@k8s-master01 ~]# vim /etc/sysctl.d/k8s.conf[root@k8s-master01 ~]# cat /etc/sysctl.d/k8s.confnet.ipv4.ip_forward = 1net.bridge.bridge-nf-call-iptables = 1fs.may_detach_mounts = 1vm.overcommit_memory=1vm.panic_on_oom=0fs.inotify.max_user_watches=89100fs.file-max=52706963fs.nr_open=52706963net.netfilter.nf_conntrack_max=2310720net.ipv4.tcp_keepalive_time = 600net.ipv4.tcp_keepalive_probes = 3net.ipv4.tcp_keepalive_intvl =15net.ipv4.tcp_max_tw_buckets = 36000net.ipv4.tcp_tw_reuse = 1net.ipv4.tcp_max_orphans = 327680net.ipv4.tcp_orphan_retries = 3net.ipv4.tcp_syncookies = 1net.ipv4.tcp_max_syn_backlog = 16384net.ipv4.ip_conntrack_max = 65536net.ipv4.tcp_max_syn_backlog = 16384net.ipv4.tcp_timestamps = 0net.core.somaxconn = 16384net.ipv6.conf.all.disable_ipv6 = 0net.ipv6.conf.default.disable_ipv6 = 0net.ipv6.conf.lo.disable_ipv6 = 0net.ipv6.conf.all.forwarding = 1[root@k8s-master01 ~]# [root@k8s-master01 ~]# reboot

测试拜访公网IPv6

[root@k8s-master01 ~]# ping www.chenby.cn -6PING www.chenby.cn(2408:871a:5100:119:1d:: (2408:871a:5100:119:1d::)) 56 data bytes64 bytes from 2408:871a:5100:119:1d:: (2408:871a:5100:119:1d::): icmp_seq=1 ttl=53 time=10.6 ms64 bytes from 2408:871a:5100:119:1d:: (2408:871a:5100:119:1d::): icmp_seq=2 ttl=53 time=9.94 ms^C--- www.chenby.cn ping statistics ---2 packets transmitted, 2 received, 0% packet loss, time 1002msrtt min/avg/max/mdev = 9.937/10.269/10.602/0.347 ms[root@k8s-master01 ~]#

批改kube-apiserver如下配置

--service-cluster-ip-range=10.96.0.0/12,fd00::/108  --feature-gates=IPv6DualStack=true [root@k8s-master01 ~]# vim /usr/lib/systemd/system/kube-apiserver.service[root@k8s-master01 ~]# cat /usr/lib/systemd/system/kube-apiserver.service[Unit]Description=Kubernetes API ServerDocumentation=https://github.com/kubernetes/kubernetesAfter=network.target[Service]ExecStart=/usr/local/bin/kube-apiserver \      --v=2  \      --logtostderr=true  \      --allow-privileged=true  \      --bind-address=0.0.0.0  \      --secure-port=6443  \      --insecure-port=0  \      --advertise-address=192.168.1.81 \      --service-cluster-ip-range=10.96.0.0/12,fd00::/108  \      --feature-gates=IPv6DualStack=true \      --service-node-port-range=30000-32767  \      --etcd-servers=https://192.168.1.81:2379,https://192.168.1.82:2379,https://192.168.1.83:2379 \      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \      --client-ca-file=/etc/kubernetes/pki/ca.pem  \      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \      --service-account-issuer=https://kubernetes.default.svc.cluster.local \      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \      --authorization-mode=Node,RBAC  \      --enable-bootstrap-token-auth=true  \      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \      --requestheader-allowed-names=aggregator  \      --requestheader-group-headers=X-Remote-Group  \      --requestheader-extra-headers-prefix=X-Remote-Extra-  \      --requestheader-username-headers=X-Remote-User \      --enable-aggregator-routing=true      # --token-auth-file=/etc/kubernetes/token.csvRestart=on-failureRestartSec=10sLimitNOFILE=65535[Install]WantedBy=multi-user.target

批改kube-controller-manager如下配置

--feature-gates=IPv6DualStack=true--service-cluster-ip-range=10.96.0.0/12,fd00::/108--cluster-cidr=172.16.0.0/12,fc00::/48--node-cidr-mask-size-ipv4=24--node-cidr-mask-size-ipv6=64[root@k8s-master01 ~]# vim /usr/lib/systemd/system/kube-controller-manager.service[root@k8s-master01 ~]# cat /usr/lib/systemd/system/kube-controller-manager.service[Unit]Description=Kubernetes Controller ManagerDocumentation=https://github.com/kubernetes/kubernetesAfter=network.target[Service]ExecStart=/usr/local/bin/kube-controller-manager \      --v=2 \      --logtostderr=true \      --address=127.0.0.1 \      --root-ca-file=/etc/kubernetes/pki/ca.pem \      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \      --leader-elect=true \      --use-service-account-credentials=true \      --node-monitor-grace-period=40s \      --node-monitor-period=5s \      --pod-eviction-timeout=2m0s \      --controllers=*,bootstrapsigner,tokencleaner \      --allocate-node-cidrs=true \      --feature-gates=IPv6DualStack=true \      --service-cluster-ip-range=10.96.0.0/12,fd00::/108 \      --cluster-cidr=172.16.0.0/12,fc00::/48 \      --node-cidr-mask-size-ipv4=24 \      --node-cidr-mask-size-ipv6=64 \      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem \      --node-cidr-mask-size=24Restart=alwaysRestartSec=10s[Install]WantedBy=multi-user.target

批改kubelet如下配置

--feature-gates=IPv6DualStack=true[root@k8s-master01 ~]# vim /usr/lib/systemd/system/kubelet.service[root@k8s-master01 ~]# cat /usr/lib/systemd/system/kubelet.service[Unit]Description=Kubernetes KubeletDocumentation=https://github.com/kubernetes/kubernetesAfter=docker.serviceRequires=docker.service[Service]ExecStart=/usr/local/bin/kubelet \    --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig  \    --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \    --config=/etc/kubernetes/kubelet-conf.yml \    --network-plugin=cni  \    --cni-conf-dir=/etc/cni/net.d  \    --cni-bin-dir=/opt/cni/bin  \    --container-runtime=remote  \    --runtime-request-timeout=15m  \    --container-runtime-endpoint=unix:///run/containerd/containerd.sock  \    --cgroup-driver=systemd \    --node-labels=node.kubernetes.io/node='' \    --feature-gates=IPv6DualStack=trueRestart=alwaysStartLimitInterval=0RestartSec=10[Install]WantedBy=multi-user.target

批改kube-apiserver如下配置

#批改如下配置clusterCIDR: 172.16.0.0/12,fc00::/48 [root@k8s-master01 ~]# vim /etc/kubernetes/kube-proxy.yaml[root@k8s-master01 ~]# cat /etc/kubernetes/kube-proxy.yamlapiVersion: kubeproxy.config.k8s.io/v1alpha1bindAddress: 0.0.0.0clientConnection:  acceptContentTypes: ""  burst: 10  contentType: application/vnd.kubernetes.protobuf  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig  qps: 5clusterCIDR: 172.16.0.0/12,fc00::/48 configSyncPeriod: 15m0sconntrack:  max: null  maxPerCore: 32768  min: 131072  tcpCloseWaitTimeout: 1h0m0s  tcpEstablishedTimeout: 24h0m0senableProfiling: falsehealthzBindAddress: 0.0.0.0:10256hostnameOverride: ""iptables:  masqueradeAll: false  masqueradeBit: 14  minSyncPeriod: 0s  syncPeriod: 30sipvs:  masqueradeAll: true  minSyncPeriod: 5s  scheduler: "rr"  syncPeriod: 30skind: KubeProxyConfigurationmetricsBindAddress: 127.0.0.1:10249mode: "ipvs"nodePortAddresses: nulloomScoreAdj: -999portRange: ""udpIdleTimeout: 250ms[root@k8s-master01 ~]#

批改calico如下配置

# vim calico.yaml# calico-config ConfigMap处    "ipam": {        "type": "calico-ipam",        "assign_ipv4": "true",        "assign_ipv6": "true"    },    - name: IP      value: "autodetect"    - name: IP6      value: "autodetect"    - name: CALICO_IPV4POOL_CIDR      value: "172.16.0.0/16"    - name: CALICO_IPV6POOL_CIDR      value: "fc00::/48"    - name: FELIX_IPV6SUPPORT      value: "true"# kubectl apply -f calico.yaml

测试

#部署利用[root@k8s-master01 ~]# cat cby.yaml apiVersion: apps/v1kind: Deploymentmetadata:  name: chenbyspec:  replicas: 3  selector:    matchLabels:      app: chenby  template:    metadata:      labels:        app: chenby    spec:      containers:      - name: chenby        image: nginx        resources:          limits:            memory: "128Mi"            cpu: "500m"        ports:        - containerPort: 80---apiVersion: v1kind: Servicemetadata:  name: chenbyspec:  ipFamilyPolicy: PreferDualStack  ipFamilies:  - IPv6  - IPv4  type: NodePort  selector:    app: chenby  ports:  - port: 80    targetPort: 80[root@k8s-master01 ~]# kubectl  apply -f cby.yaml#查看端口[root@k8s-master01 ~]# kubectl  get svcNAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)        AGEchenby       NodePort    fd00::d80a   <none>        80:31535/TCP   54skubernetes   ClusterIP   10.96.0.1    <none>        443/TCP        22h[root@k8s-master01 ~]# #应用内网拜访[root@k8s-master01 ~]# curl -I http://[fd00::d80a]HTTP/1.1 200 OKServer: nginx/1.21.6Date: Fri, 29 Apr 2022 07:29:28 GMTContent-Type: text/htmlContent-Length: 615Last-Modified: Tue, 25 Jan 2022 15:03:52 GMTConnection: keep-aliveETag: "61f01158-267"Accept-Ranges: bytes[root@k8s-master01 ~]# #应用公网拜访[root@k8s-master01 ~]# curl -I http://[2408:8207:78ce:7561::10]:31535HTTP/1.1 200 OKServer: nginx/1.21.6Date: Fri, 29 Apr 2022 07:25:16 GMTContent-Type: text/htmlContent-Length: 615Last-Modified: Tue, 25 Jan 2022 15:03:52 GMTConnection: keep-aliveETag: "61f01158-267"Accept-Ranges: bytes[root@k8s-master01 ~]# [root@k8s-master01 ~]# curl -I http://10.0.0.81:31535HTTP/1.1 200 OKServer: nginx/1.21.6Date: Fri, 29 Apr 2022 07:26:16 GMTContent-Type: text/htmlContent-Length: 615Last-Modified: Tue, 25 Jan 2022 15:03:52 GMTConnection: keep-aliveETag: "61f01158-267"Accept-Ranges: bytes[root@k8s-master01 ~]#

https://www.oiox.cn/
https://www.chenby.cn/
https://blog.oiox.cn/
https://cby-chen.github.io/
https://blog.csdn.net/qq\_33921750
https://my.oschina.net/u/3981543
https://www.zhihu.com/people/...
https://segmentfault.com/u/hp...
https://juejin.cn/user/331578...
https://cloud.tencent.com/dev...
https://www.jianshu.com/u/0f8...
https://www.toutiao.com/c/use...
CSDN、GitHub、知乎、开源中国、思否、掘金、简书、腾讯云、今日头条、集体博客、全网可搜《小陈运维》
文章次要公布于微信公众号：《Linux运维交换社区》