kubeadm Networking.PodSubnet

kubeadm Networking.PodSubnet

kubeadm初始化k8s集群时可配置集群网络

其数据结构如下

// Networking contains elements describing cluster's networking configuration.
type Networking struct {
	// ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12".
	ServiceSubnet string
	// PodSubnet is the subnet used by pods.
	PodSubnet string
	// DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local".
	DNSDomain string
}

其中PodSubnet用来配置pod可分配IP段

1、podSubnet参数格式

根据validate代码

allErrs := field.ErrorList{}
// 参数格式为以“,”分隔的CIDR即ip/prefixlen
subnets, err := netutils.ParseCIDRs(strings.Split(subnetStr, ","))
	if err != nil {
		allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "couldn't parse subnet"))
		return allErrs
	}
	switch {
	// 不能有超过两个CIDR
	case len(subnets) > 2:
		allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking"))
	// 如果有两个CIDR，则必须是一个IPv4 CIDR一个IPv6 CIDR
	case len(subnets) == 2:
		areDualStackCIDRs, err := netutils.IsDualStackCIDRs(subnets)
		if err != nil {
			allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, err.Error()))
		} else if !areDualStackCIDRs {
			allErrs = append(allErrs, field.Invalid(fldPath, subnetStr, "expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking"))
		}
	}

podSubnet的格式支持单个IPv4 CIDR或单个IPv6 CIDR，或者同时包含1个IPv4 CIDR和1个IPv6 CIDR；即以下三种格式：

ipv4/prefixlen   例如 192.168.1.2/24
ipv6/prefixlen   例如 3004::2e:a/64
ipv4/prefixlen_ipv4,ipv6/prefixlen_ipv6  例如 192.168.1.2/24,3004::2e:a/64

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
  disableDefaultCNI: true
  podSubnet: 10.233.0.1/16

2、地址个数限制

podSubnet至少包含14个IP地址

// 获取CIDR中的ip地址个数
numAddresses := netutils.RangeSize(s)
// minAddrs=14，表示pod数量至少14，则podSubnet至少包含14个地址
// todo 暂不知为何pod数量至少14
if numAddresses < minAddrs {
	allErrs = append(allErrs, field.Invalid(fldPath, s.String(), fmt.Sprintf("subnet with %d address(es) is too small, the minimum is %d", numAddresses, minAddrs)))
}

3、地址内容限制

IPv6地址尽量避免配置为站点本地地址（fec0::/10，一般是路由器网口地址）

_, siteLocalNet, _ := netutils.ParseCIDRSloppy("fec0::/10")
if siteLocalNet.Contains(s.IP) || s.Contains(siteLocalNet.IP) {
klog.Warningf("the subnet %v contains IPv6 site-local addresses that belong to fec0::/10 which has been deprecated by rfc3879", s)
}

4、prefixlen的业务限制

由于podSubnet需要按子网划分给多个node，所以podSubnet的前缀长度必须小于node上可分配IP的前缀长度

另外，这两个前缀长度差值不能超过16（由于使用了roaring bitmaps压缩方式cidr_set bitmap should be compressed · Issue #44918 · kubernetes/kubernetes · GitHub）

// masksize，podNodenet的prefixlen
// nodeMask，配置的每个node上可分配IP的前缀长度，默认IPv4是24，IPv6是64
// nodeMask可通过ExtraArgs的node-cidr-mask-size-ipv6/4字段配置
if maskSize > nodeMask {
    allErrs = append(allErrs, field.Invalid(fldPath, podSubnet.String(), fmt.Sprintf("the size of pod subnet with mask %d is smaller than the size of node subnet with mask %d", maskSize, nodeMask)))
} else if (nodeMask - maskSize) > constants.PodSubnetNodeMaskMaxDiff {
// constants.PodSubnetNodeMaskMaxDiff=16
// node前缀长度和podSubnet的前缀长度差值不能超过16，todo why
    allErrs = append(allErrs, field.Invalid(fldPath, podSubnet.String(), fmt.Sprintf("pod subnet mask (%d) and node-mask (%d) difference is greater than %d", maskSize, nodeMask, constants.PodSubnetNodeMaskMaxDiff)))
}

apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.17.2
networking:
  podSubnet: "192.168.0.0/16"
controllerManager:
  extraArgs:
    "node-cidr-mask-size": "16"