Created
March 23, 2026 20:02
-
-
Save trozet/c2c03e30ff1fd67e671df59ea7fa6819 to your computer and use it in GitHub Desktop.
Testing out BGP no overlay with CDN+CUDN
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ####### CDN - managed with SNAT=true | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list ovn_cluster_router | |
| IPv4 Routes | |
| Route Table <main>: | |
| 100.64.0.2 100.64.0.2 dst-ip | |
| 10.244.0.0/24 100.64.0.2 src-ip | |
| 10.244.0.0/16 100.64.0.2 src-ip | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_ovn-control-plane | |
| IPv4 Routes | |
| Route Table <main>: | |
| 10.244.1.0/24 172.18.0.4 dst-ip rtoe-GR_ovn-control-plane | |
| 10.244.2.0/24 172.18.0.2 dst-ip rtoe-GR_ovn-control-plane | |
| 169.254.0.0/17 169.254.0.4 dst-ip rtoe-GR_ovn-control-plane | |
| 10.244.0.0/16 100.64.0.1 dst-ip | |
| 0.0.0.0/0 172.18.0.1 dst-ip rtoe-GR_ovn-control-plane | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-nat-list GR_ovn-control-plane | |
| TYPE GATEWAY_PORT MATCH EXTERNAL_IP EXTERNAL_PORT LOGICAL_IP EXTERNAL_MAC LOGICAL_PORT | |
| snat 172.18.0.3 32768-60999 100.64.0.2 | |
| snat 172.18.0.3 32768-60999 10.244.0.0/16 | |
| [root@ovn-control-plane ~]# ovn-nbctl list nat | |
| _uuid : bc8bc908-c377-41ff-afda-1097ebce1632 | |
| allowed_ext_ips : [] | |
| exempted_ext_ips : fa4d92da-8a9e-4b4a-8d3f-f8d874c9ffeb | |
| external_ids : {} | |
| external_ip : "172.18.0.3" | |
| external_mac : [] | |
| external_port_range : "32768-60999" | |
| gateway_port : [] | |
| logical_ip : "10.244.0.0/16" | |
| logical_port : [] | |
| match : "" | |
| options : {stateless="false"} | |
| priority : 0 | |
| type : snat | |
| ^shouldn't this match on !east/west? | |
| ###### Now create CUDN | |
| trozet@yard53:~$ k get routeadvertisements | |
| NAME STATUS | |
| default Not Accepted: configuration pending: no FRRConfigurations selected | |
| ovnk-managed-ebada5168620c5fe Accepted | |
| ^ default shows not accepted | |
| trozet@yard53:~$ k get routeadvertisements default -o yaml | |
| apiVersion: k8s.ovn.org/v1 | |
| kind: RouteAdvertisements | |
| metadata: | |
| annotations: | |
| kubectl.kubernetes.io/last-applied-configuration: | | |
| {"apiVersion":"k8s.ovn.org/v1","kind":"RouteAdvertisements","metadata":{"annotations":{},"name":"default"},"spec":{"advertisements":["PodNetwork"],"frrConfigurationSelector":{"matchLabels":{"name":"receive-all"}},"networkSelectors":[{"networkSelectionType":"DefaultNetwork"}],"nodeSelector":{}}} | |
| creationTimestamp: "2026-03-23T18:48:38Z" | |
| generation: 1 | |
| name: default | |
| resourceVersion: "1362" | |
| uid: f3662a93-cf59-4efc-bd9f-da1dbe8faf7a | |
| spec: | |
| advertisements: | |
| - PodNetwork | |
| frrConfigurationSelector: | |
| matchLabels: | |
| name: receive-all | |
| networkSelectors: | |
| - networkSelectionType: DefaultNetwork | |
| nodeSelector: {} | |
| status: | |
| conditions: | |
| - lastTransitionTime: "2026-03-23T18:49:26Z" | |
| message: 'configuration pending: no FRRConfigurations selected' | |
| observedGeneration: 1 | |
| reason: ConfigurationPending | |
| status: "False" | |
| type: Accepted | |
| status: 'Not Accepted: configuration pending: no FRRConfigurations selected' | |
| ^ But FRR managed config is advertising and receiving CDN pod network: | |
| trozet@yard53:~$ k get frrconfigurations.frrk8s.metallb.io -n frr-k8s-system ovnk-generated-26sx2 -o yaml | |
| apiVersion: frrk8s.metallb.io/v1beta1 | |
| kind: FRRConfiguration | |
| metadata: | |
| annotations: | |
| k8s.ovn.org/route-advertisements: ovnk-managed-ebada5168620c5fe/ovnk-managed-c9ae298d71aa4275/ovn-control-plane | |
| creationTimestamp: "2026-03-23T18:49:26Z" | |
| generateName: ovnk-generated- | |
| generation: 1 | |
| labels: | |
| k8s.ovn.org/route-advertisements: ovnk-managed-ebada5168620c5fe | |
| name: ovnk-generated-26sx2 | |
| namespace: frr-k8s-system | |
| resourceVersion: "1365" | |
| uid: 6722ab34-2dae-44a6-8ee9-0511a6bad52c | |
| spec: | |
| bgp: | |
| routers: | |
| - asn: 64512 | |
| neighbors: | |
| - address: 172.18.0.2 | |
| asn: 64512 | |
| disableMP: true | |
| dualStackAddressFamily: false | |
| passwordSecret: {} | |
| toAdvertise: | |
| allowed: | |
| mode: filtered | |
| prefixes: | |
| - 10.244.0.0/24 | |
| toReceive: | |
| allowed: | |
| mode: filtered | |
| prefixes: | |
| - ge: 24 | |
| le: 24 | |
| prefix: 10.244.0.0/16 | |
| - address: 172.18.0.4 | |
| asn: 64512 | |
| disableMP: true | |
| dualStackAddressFamily: false | |
| passwordSecret: {} | |
| toAdvertise: | |
| allowed: | |
| mode: filtered | |
| prefixes: | |
| - 10.244.0.0/24 | |
| toReceive: | |
| allowed: | |
| mode: filtered | |
| prefixes: | |
| - ge: 24 | |
| le: 24 | |
| prefix: 10.244.0.0/16 | |
| prefixes: | |
| - 10.244.0.0/24 | |
| nodeSelector: | |
| matchLabels: | |
| kubernetes.io/hostname: ovn-control-plane | |
| raw: {} | |
| #### CUDN in NBDB | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-nat-list GR_cluster_udn_l3.primary_ovn-control-plane | |
| TYPE GATEWAY_PORT MATCH EXTERNAL_IP EXTERNAL_PORT LOGICAL_IP EXTERNAL_MAC LOGICAL_PORT | |
| snat 169.254.0.11 32768-60999 100.65.0.2 | |
| snat 169.254.0.11 32768-60999 10.20.0.0/16 | |
| ^ same outbound SNAT missing match criteriafor the CUDN | |
| trozet@yard53:~$ cat ~/cudn-no-overlay.yaml | |
| apiVersion: k8s.ovn.org/v1 | |
| kind: ClusterUserDefinedNetwork | |
| metadata: | |
| name: l3-primary | |
| labels: | |
| bgp: enabled | |
| spec: | |
| namespaceSelector: | |
| matchExpressions: | |
| - key: kubernetes.io/metadata.name | |
| operator: In | |
| values: | |
| - udn-test | |
| - udn-test2 | |
| #matchLabels: | |
| #kubernetes.io/metadata.name: udn-test | |
| network: | |
| topology: Layer3 | |
| layer3: | |
| role: Primary | |
| subnets: | |
| - cidr: 10.20.0.0/16 | |
| transport: "NoOverlay" | |
| noOverlay: | |
| outboundSNAT: "Enabled" | |
| routing: "Managed" | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list cluster_udn_l3.primary_ovn_cluster_router | |
| IPv4 Routes | |
| Route Table <main>: | |
| 10.96.0.1 10.20.0.2 dst-ip | |
| 10.96.0.10 10.20.0.2 dst-ip | |
| 100.65.0.2 100.65.0.2 dst-ip | |
| 10.20.0.0/24 100.65.0.2 src-ip | |
| 10.20.0.0/16 100.65.0.2 src-ip | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_cluster_udn_l3.primary_ovn-control-plane | |
| IPv4 Routes | |
| Route Table <main>: | |
| 169.254.0.0/17 169.254.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.20.0.0/16 100.65.0.1 dst-ip | |
| 0.0.0.0/0 172.18.0.1 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| ^ missing routes for 10.20.0.0 | |
| [root@ovn-control-plane ~]# ip route show | |
| default via 172.18.0.1 dev breth0 | |
| 10.96.0.0/16 via 169.254.0.4 dev breth0 src 169.254.0.2 mtu 1500 | |
| 10.244.0.0/24 dev ovn-k8s-mp0 proto kernel scope link src 10.244.0.2 | |
| 10.244.0.0/16 via 10.244.0.1 dev ovn-k8s-mp0 | |
| 10.244.1.0/24 nhid 34 via 172.18.0.4 dev breth0 proto bgp metric 20 | |
| 10.244.2.0/24 nhid 33 via 172.18.0.2 dev breth0 proto bgp metric 20 | |
| 169.254.0.0/17 dev breth0 proto kernel scope link src 169.254.0.2 | |
| 169.254.0.1 dev breth0 src 172.18.0.3 | |
| 169.254.0.3 via 10.244.0.1 dev ovn-k8s-mp0 | |
| 172.18.0.0/16 dev breth0 proto kernel scope link src 172.18.0.3 | |
| 172.19.0.0/16 dev eth1 proto kernel scope link src 172.19.0.3 | |
| ### looks like managed + CUDN doesn't work, create my own RA | |
| trozet@yard53:~$ cat ~/ra.yaml | |
| apiVersion: k8s.ovn.org/v1 | |
| kind: RouteAdvertisements | |
| metadata: | |
| name: trozet-cudn | |
| spec: | |
| nodeSelector: {} | |
| frrConfigurationSelector: {} | |
| networkSelectors: | |
| - networkSelectionType: ClusterUserDefinedNetworks | |
| clusterUserDefinedNetworkSelector: | |
| networkSelector: | |
| matchLabels: | |
| bgp: "enabled" | |
| advertisements: | |
| - "PodNetwork" | |
| # - "EgressIP" | |
| trozet@yard53:~$ k create -f ~/ra.yaml | |
| routeadvertisements.k8s.ovn.org/trozet-cudn created | |
| trozet@yard53:~$ k get routeadvertisements | |
| NAME STATUS | |
| default Not Accepted: configuration pending: no FRRConfigurations selected | |
| ovnk-managed-ebada5168620c5fe Accepted | |
| trozet-cudn Accepted | |
| ## routes show up | |
| [root@ovn-control-plane ~]# ip route show | |
| default via 172.18.0.1 dev breth0 | |
| 10.20.1.0/24 nhid 34 via 172.18.0.4 dev breth0 proto bgp metric 20 | |
| 10.20.2.0/24 nhid 33 via 172.18.0.2 dev breth0 proto bgp metric 20 | |
| 10.96.0.0/16 via 169.254.0.4 dev breth0 src 169.254.0.2 mtu 1500 | |
| 10.244.0.0/24 dev ovn-k8s-mp0 proto kernel scope link src 10.244.0.2 | |
| 10.244.0.0/16 via 10.244.0.1 dev ovn-k8s-mp0 | |
| 10.244.1.0/24 nhid 34 via 172.18.0.4 dev breth0 proto bgp metric 20 | |
| 10.244.2.0/24 nhid 33 via 172.18.0.2 dev breth0 proto bgp metric 20 | |
| 169.254.0.0/17 dev breth0 proto kernel scope link src 169.254.0.2 | |
| 169.254.0.1 dev breth0 src 172.18.0.3 | |
| 169.254.0.3 via 10.244.0.1 dev ovn-k8s-mp0 | |
| 172.18.0.0/16 dev breth0 proto kernel scope link src 172.18.0.3 | |
| 172.19.0.0/16 dev eth1 proto kernel scope link src 172.19.0.3 | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_cluster_udn_l3.primary_ovn-control-plane | |
| IPv4 Routes | |
| Route Table <main>: | |
| 10.20.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.20.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.244.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.244.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 169.254.0.0/17 169.254.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.20.0.0/16 100.65.0.1 dst-ip | |
| 0.0.0.0/0 172.18.0.1 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| ### Create a 2nd CUDN | |
| trozet@yard53:~$ k create -f ~/cudn-no-overlay2.yaml | |
| clusteruserdefinednetwork.k8s.ovn.org/l3-primary-2 created | |
| trozet@yard53:~$ cat ~/cudn-no-overlay2.yaml | |
| apiVersion: k8s.ovn.org/v1 | |
| kind: ClusterUserDefinedNetwork | |
| metadata: | |
| name: l3-primary-2 | |
| labels: | |
| bgp: enabled | |
| spec: | |
| namespaceSelector: | |
| matchExpressions: | |
| - key: kubernetes.io/metadata.name | |
| operator: In | |
| values: | |
| - udn-test3 | |
| - udn-test4 | |
| #matchLabels: | |
| #kubernetes.io/metadata.name: udn-test | |
| network: | |
| topology: Layer3 | |
| layer3: | |
| role: Primary | |
| subnets: | |
| - cidr: 10.30.0.0/16 | |
| transport: "NoOverlay" | |
| noOverlay: | |
| outboundSNAT: "Enabled" | |
| routing: "Managed" | |
| trozet@yard53:~$ k get cudn l3-primary-2 -o yaml | grep "Transport has been" | |
| message: Transport has been configured as 'no-overlay'. | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| IPv4 Routes | |
| Route Table <main>: | |
| 10.20.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.20.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.30.0.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.30.1.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.244.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.244.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 169.254.0.0/17 169.254.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| 10.30.0.0/16 100.65.0.1 dst-ip | |
| 0.0.0.0/0 172.18.0.1 dst-ip rtoe-GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| ^ 2nd CUDN has routes from 1st CUDN | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-route-list GR_cluster_udn_l3.primary_ovn-control-plane | |
| IPv4 Routes | |
| Route Table <main>: | |
| 10.20.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.20.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.30.0.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.30.1.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.244.1.0/24 172.18.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.244.2.0/24 172.18.0.2 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 169.254.0.0/17 169.254.0.4 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| 10.20.0.0/16 100.65.0.1 dst-ip | |
| 0.0.0.0/0 172.18.0.1 dst-ip rtoe-GR_cluster_udn_l3.primary_ovn-control-plane | |
| ^vice versa for 1st CUDN | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-nat-list GR_cluster_udn_l3.primary.2_ovn-control-plane | |
| TYPE GATEWAY_PORT MATCH EXTERNAL_IP EXTERNAL_PORT LOGICAL_IP EXTERNAL_MAC LOGICAL_PORT | |
| snat 169.254.0.13 32768-60999 100.65.0.2 | |
| snat ip4.dst == $a71297 169.254.0.13 32768-60999 10.30.0.0/16 | |
| ^ this nat has match criteria, but its only to node IPs? | |
| _uuid : 8b30a6d0-ffbe-4b4c-a279-a83814d63887 | |
| addresses : ["172.18.0.2", "172.18.0.3", "172.18.0.4", "172.19.0.2", "172.19.0.3", "172.19.0.4"] | |
| external_ids : {ip-family=v4, "k8s.ovn.org/id"="default-network-controller:EgressIP:node-ips:v4:default", "k8s.ovn.org/name"=node-ips, "k8s.ovn.org/owner-controller"=default-network-controller, "k8s.ovn.org/owner-type"=EgressIP, network=default} | |
| name : a712973235162149816 | |
| Now all the sudden CUDN1 has match criteria in its NAT: | |
| [root@ovn-control-plane ~]# ovn-nbctl lr-nat-list GR_cluster_udn_l3.primary_ovn-control-plane | |
| TYPE GATEWAY_PORT MATCH EXTERNAL_IP EXTERNAL_PORT LOGICAL_IP EXTERNAL_MAC LOGICAL_PORT | |
| snat 169.254.0.11 32768-60999 100.65.0.2 | |
| snat ip4.dst == $a71297 169.254.0.11 32768-60999 10.20.0.0/16 | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment