k8s flannel vxlan

用到的知识:

linux bridge
linux network namespace
flannel backend
iptables

host 上的 flannel 配置文件

[root@k8snode028 bin]# cat /etc/cni/net.d/10-flannel.conflist 
{
  "name":"cni0",
  "cniVersion":"0.3.1",
  "plugins":[
    {
      "type":"flannel",
      "delegate":{
        "forceAddress":true,
        "hairpinMode": true,
        "isDefaultGateway":true
      }
    },
    {
      "type":"portmap",
      "capabilities":{
        "portMappings":true
      }
    }
  ]
}

host 上的 flannel 相关 bin 文件

ls /opt/cni/bin/
bridge  cnitool  dhcp  flannel  host-local  ipvlan  loopback  macvlan  noop  portmap  ptp  tuning  vlan

查看 namespace

ip netns list

查看 link

ip link

创建 red 的 namespace，并查看 link

ip netns exec red ip link

kubernetes host 上的 interface, ifconfig

...
cni0: flags=4163  mtu 1450
        inet 10.27.9.1  netmask 255.255.255.0  broadcast 0.0.0.0
...

flannel.1: flags=4163  mtu 1450
        inet 10.27.9.0  netmask 255.255.255.255  broadcast 0.0.0.0
...

kubernetes pod 内的 arp route 显示

[root@backend-6b7b58986c-jdnhn spring]# route  -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.27.9.1       0.0.0.0         UG    0      0        0 eth0
10.27.0.0       10.27.9.1       255.255.0.0     UG    0      0        0 eth0
10.27.9.0       0.0.0.0         255.255.255.0   U     0      0        0 eth0

node 节点上的 bridge 信息

yum install bridge-utils -y
[root@k8snode028 ~]# brctl show
bridge name	bridge id		STP enabled	interfaces
cni0		8000.0a580a1d0d01	no		veth0985a0b8
							veth1253dd53
							veth127dc912
							veth15b89d85
							veth30b9e182
							veth3a6a0f27
							veth500fcaf1
							veth5f994152
							veth6374a5e2
							veth67f0cb2e
							veth69772276
							veth6a95a62c
							veth6c8897b0
							veth781dd728
							veth88eddba5
							veth8d28dfe8
							veth8fd03c3e
							veth8fe5c104
							vetha529244f
							vethb2b2134b
							vethc070358a
							vethccd69c78
							vethd6f2ae04
							vethdbaec209
							vethe0e0bd3a
							vethe3939f54
							vethec7d63a5
							vethefdaa81a
							vethf836c26b
							vethfb0f86bc
							vethfec66a08
docker0		8000.0242fbd0e139	no

node 节点上的路由信息

[root@k8snode028 network-scripts]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.18.19.1      0.0.0.0         UG    0      0        0 bond0
10.18.19.0      0.0.0.0         255.255.255.0   U     0      0        0 bond0
10.27.0.0       10.27.0.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.1.0       10.27.1.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.2.0       10.27.2.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.3.0       10.27.3.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.4.0       10.27.4.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.5.0       10.27.5.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.6.0       10.27.6.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.7.0       10.27.7.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.8.0       10.27.8.0       255.255.255.0   UG    0      0        0 flannel.1
10.27.9.0       0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.27.10.0      10.27.10.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.11.0      10.27.11.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.12.0      10.27.12.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.13.0      10.27.13.0      255.255.255.0   UG    0      0        0 flannel.1
10.27.14.0      10.27.14.0      255.255.255.0   UG    0      0        0 flannel.1
169.254.0.0     0.0.0.0         255.255.0.0     U     1006   0        0 bond0
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0

node 节点上查看 bridge (相当于本机虚拟交换机)

ip link show type bridge 或 bridge link show
7: docker0:  mtu 1500 qdisc noqueue state UP mode DEFAULT group default 
    link/ether 02:42:1c:85:aa:91 brd ff:ff:ff:ff:ff:ff
12: cni0:  mtu 1450 qdisc noqueue state UP mode DEFAULT group default qlen 1000
    link/ether 0a:58:0a:1b:09:01 brd ff:ff:ff:ff:ff:ff

node 节点上查看所有的 veth

ip link show type veth
25: veth7443cee@if24:  mtu 1500 qdisc noqueue master docker0 state UP mode DEFAULT group default 
    link/ether 2e:03:af:2d:5e:46 brd ff:ff:ff:ff:ff:ff link-netnsid 5
...

node 上的 flannel 容器里的信息

[root@k8snode028 bin]# docker exec 1a84ab994e5d cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.27.0.0/16
FLANNEL_SUBNET=10.27.9.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

从一台 node pod 到另一台 node pod 网段(10.29.13.0)，查询找到 mac 地址

arp -e | grep 10.29.13.0
10.29.13.0               ether   42:32:74:de:24:50   CM                    flannel.1

再通过 mac 地址找到是哪台目标 node

bridge fdb show | grep 42:32:74:de:24:50
42:32:74:de:24:50 dev flannel.1 dst 10.18.19.129 self permanent

参考:
https://xuxinkun.github.io/2019/06/05/flannel-vxlan/