Other then in my L3VPN EVPN with VyOS post, I decided to drop OSPF as IGP (Interior Gateway Protocol) and solely build on BGP.
set interfaces dummy dum0 address '1.1.1.1/32'
set interfaces ethernet eth0 address '100.0.0.11/31'
set interfaces ethernet eth0 description 'P2P-to-INET'
set interfaces ethernet eth0 mtu '1400'
set interfaces wireguard wg1 address '100.0.0.1/30'
set interfaces wireguard wg1 mtu '1600'
set interfaces wireguard wg1 peer PE1 address '100.0.0.13'
set interfaces wireguard wg1 peer PE1 allowed-ips '0.0.0.0/0'
set interfaces wireguard wg1 peer PE1 port '10000'
set interfaces wireguard wg1 peer PE1 public-key 'FZ1w0d3Wqh6ZLqGGjGCDm5HLfIJW8angDX1QNrb92Gk='
set interfaces wireguard wg1 port '10000'
set interfaces wireguard wg1 private-key 'KI+bywaKMTI432W7Th42j0+6SXkyykhh9prMrN4ppX8='
set protocols bgp address-family ipv4-unicast network 1.1.1.1/32
set protocols bgp neighbor 100.0.0.2 description 'PE2'
set protocols bgp neighbor 100.0.0.2 peer-group 'IPv4-UNDERLAY'
set protocols bgp parameters log-neighbor-changes
set protocols bgp parameters router-id '1.1.1.1'
set protocols bgp peer-group IPv4-UNDERLAY address-family ipv4-unicast
set protocols bgp peer-group IPv4-UNDERLAY bfd
set protocols bgp peer-group IPv4-UNDERLAY remote-as 'external'
set protocols bgp system-as '65001'
set protocols static route 0.0.0.0/0 next-hop 100.0.0.10
set interfaces dummy dum0 address '2.2.2.2/32'
set interfaces ethernet eth0 address '100.0.0.13/31'
set interfaces ethernet eth0 description 'P2P-to-INET'
set interfaces ethernet eth0 mtu '1400'
set interfaces wireguard wg1 address '100.0.0.2/30'
set interfaces wireguard wg1 mtu '1600'
set interfaces wireguard wg1 peer PE1 address '100.0.0.11'
set interfaces wireguard wg1 peer PE1 allowed-ips '0.0.0.0/0'
set interfaces wireguard wg1 peer PE1 port '10000'
set interfaces wireguard wg1 peer PE1 public-key '+HPVoPjLXWGobntuJ/F+gMXYP3o1P35vtUkPSdGVERU='
set interfaces wireguard wg1 port '10000'
set interfaces wireguard wg1 private-key 'eIO0pGevEZI8Jv0d1VZZ1s6enV493OSxRiwtXRUGaX4='
set protocols bgp address-family ipv4-unicast network 2.2.2.2/32
set protocols bgp neighbor 100.0.0.1 description 'PE1'
set protocols bgp neighbor 100.0.0.1 peer-group 'IPv4-UNDERLAY'
set protocols bgp parameters log-neighbor-changes
set protocols bgp parameters router-id '2.2.2.2'
set protocols bgp peer-group IPv4-UNDERLAY address-family ipv4-unicast
set protocols bgp peer-group IPv4-UNDERLAY bfd
set protocols bgp peer-group IPv4-UNDERLAY remote-as 'external'
set protocols bgp system-as '65002'
set protocols static route 0.0.0.0/0 next-hop 100.0.0.12
vyos@PE1:~$ show bgp summary
IPv4 Unicast Summary (VRF default):
BGP router identifier 1.1.1.1, local AS number 65001 vrf-id 0
BGP table version 2
RIB entries 3, using 576 bytes of memory
Peers 1, using 20 KiB of memory
Peer groups 2, using 128 bytes of memory
Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc
100.0.0.2 4 65002 201 202 2 0 0 03:16:26 1 2 PE2
vyos@PE1:~$ show bgp ipv4
BGP table version is 2, local router ID is 1.1.1.1, vrf id 0
Default local pref 100, local AS 65001
Status codes: s suppressed, d damped, h history, * valid, > best, = multipath,
i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes: i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found
Network Next Hop Metric LocPrf Weight Path
*> 1.1.1.1/32 0.0.0.0 0 32768 i
*> 2.2.2.2/32 100.0.0.2 0 0 65002 i
set interfaces bridge br1 enable-vlan
set interfaces bridge br1 member interface vxlan1
set interfaces vxlan vxlan1 parameters external
set interfaces vxlan vxlan1 parameters nolearning
set protocols bgp address-family l2vpn-evpn advertise-all-vni
set protocols bgp address-family l2vpn-evpn advertise-svi-ip
set protocols bgp peer-group EVPN-OVERLAY address-family l2vpn-evpn
set protocols bgp peer-group EVPN-OVERLAY bfd
set protocols bgp peer-group EVPN-OVERLAY ebgp-multihop '3'
set protocols bgp peer-group EVPN-OVERLAY remote-as 'external'
set protocols bgp peer-group EVPN-OVERLAY update-source 'dum0'
PE1
set interfaces vxlan vxlan1 source-address '1.1.1.1'
set protocols bgp neighbor 2.2.2.2 description 'PE2'
set protocols bgp neighbor 2.2.2.2 peer-group 'EVPN-OVERLAY'
set interfaces vxlan vxlan1 source-address '2.2.2.2'
set protocols bgp neighbor 1.1.1.1 description 'PE2'
set protocols bgp neighbor 1.1.1.1 peer-group 'EVPN-OVERLAY'
vyos@PE1:~$ show bgp l2vpn evpn summary
BGP router identifier 1.1.1.1, local AS number 65001 vrf-id 0
BGP table version 0
RIB entries 23, using 4416 bytes of memory
Peers 1, using 20 KiB of memory
Peer groups 2, using 128 bytes of memory
Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc
2.2.2.2 4 65002 225 225 7 0 0 03:27:51 11 21 PE2
set interfaces bridge br1 member interface eth1 native-vlan '20'
set interfaces bridge br1 vif 20
set interfaces vxlan vxlan1 vlan-to-vni 20 vni '10020'
set protocols bgp address-family l2vpn-evpn vni 10020 rd '1.1.1.1:10020'
set interfaces bridge br1 member interface eth1 native-vlan '20'
set interfaces bridge br1 vif 20
set interfaces vxlan vxlan1 vlan-to-vni 20 vni '10020'
set protocols bgp address-family l2vpn-evpn vni 10020 rd '2.2.2.2:10020'
vyos@PE1:~$ show evpn mac vni 10020
Number of MACs (local and remote) known for this VNI: 4
Flags: N=sync-neighs, I=local-inactive, P=peer-active, X=peer-proxy
MAC Type Flags Intf/Remote ES/VTEP VLAN Seq #'s
50:00:00:03:00:01 local eth1 20 0/0
50:00:00:04:00:01 remote 2.2.2.2 0/0
3e:20:1f:50:be:6d remote 2.2.2.2 0/0
aa:34:29:79:1f:54 local br1.20 20 0/0
vyos@PE1:~$ show bgp l2vpn evpn route vni 10020
BGP table version is 5, local router ID is 1.1.1.1
Status codes: s suppressed, d damped, h history, * valid, > best, i - internal
Origin codes: i - IGP, e - EGP, ? - incomplete
EVPN type-1 prefix: [1]:[EthTag]:[ESI]:[IPlen]:[VTEP-IP]:[Frag-id]
EVPN type-2 prefix: [2]:[EthTag]:[MAClen]:[MAC]:[IPlen]:[IP]
EVPN type-3 prefix: [3]:[EthTag]:[IPlen]:[OrigIP]
EVPN type-4 prefix: [4]:[ESI]:[IPlen]:[OrigIP]
EVPN type-5 prefix: [5]:[EthTag]:[IPlen]:[IP]
Network Next Hop Metric LocPrf Weight Path
*> [2]:[0]:[48]:[3e:20:1f:50:be:6d]:[128]:[fe80::3c20:1fff:fe50:be6d]
2.2.2.2 0 65002 i
RT:65002:10020 ET:8
*> [2]:[0]:[48]:[aa:34:29:79:1f:54]:[128]:[fe80::a834:29ff:fe79:1f54]
1.1.1.1 32768 i
ET:8 RT:65001:10020
*> [3]:[0]:[32]:[1.1.1.1]
1.1.1.1 32768 i
ET:8 RT:65001:10020
*> [3]:[0]:[32]:[2.2.2.2]
2.2.2.2 0 65002 i
RT:65002:10020 ET:8
set interfaces bridge br1 member interface eth2 native-vlan '3002'
set interfaces bridge br1 vif 3002 vrf 'black'
set interfaces bridge br1 vif 4000 vrf 'black'
set interfaces vxlan vxlan1 vlan-to-vni 3002 vni '13002'
set interfaces vxlan vxlan1 vlan-to-vni 4000 vni '10200'
set policy prefix-list AS200-IN rule 10 action 'permit'
set policy prefix-list AS200-IN rule 10 prefix '200.0.0.0/16'
set policy prefix-list DFLT-ONLY rule 10 action 'permit'
set policy prefix-list DFLT-ONLY rule 10 prefix '0.0.0.0/0'
set vrf name black protocols bgp address-family l2vpn-evpn advertise ipv4 unicast
set vrf name black protocols bgp parameters log-neighbor-changes
set vrf name black protocols bgp peer-group AS200 address-family ipv4-unicast default-originate
set vrf name black protocols bgp peer-group AS200 address-family ipv4-unicast prefix-list export 'DFLT-ONLY'
set vrf name black protocols bgp peer-group AS200 address-family ipv4-unicast prefix-list import 'AS200-IN'
set vrf name black protocols bgp peer-group AS200 local-as 100 no-prepend replace-as
set vrf name black protocols bgp peer-group AS200 remote-as '200'
set vrf name black table '10200'
set vrf name black vni '10200'
set interfaces bridge br1 vif 3002 address '100.0.0.50/31'
set interfaces bridge br1 vif 3002 description 'P2P-to-B_CE1'
set protocols bgp address-family l2vpn-evpn vni 10200 rd '1.1.1.1:10200'
set protocols bgp address-family l2vpn-evpn vni 13002 rd '1.1.1.1:13002'
set vrf name black protocols bgp neighbor 100.0.0.51 description 'B_CE1'
set vrf name black protocols bgp neighbor 100.0.0.51 peer-group AS200
set vrf name black protocols bgp parameters router-id '1.1.1.1'
set vrf name black protocols bgp system-as '65001'
set interfaces bridge br1 vif 3002 address '100.0.0.52/31'
set interfaces bridge br1 vif 3002 description 'P2P-to-B_CE2'
set protocols bgp address-family l2vpn-evpn vni 10200 rd '2.2.2.2:10200'
set protocols bgp address-family l2vpn-evpn vni 13002 rd '2.2.2.2:13002'
set vrf name black protocols bgp neighbor 100.0.0.53 peer-group AS200
set vrf name black protocols bgp neighbor 100.0.0.53 description 'B_CE2'
set vrf name black protocols bgp parameters router-id '2.2.2.2'
set vrf name black protocols bgp system-as '65002'
Once the ISP side is up we also need to configure the customer edge (CE) devices and establish the necessary BGP peering to both FooBar ISP and out iBGP session for redundancy between B_CE1 and B_CE2.
set interfaces dummy dum0 address '200.0.0.1/32'
set interfaces ethernet eth0 address '200.0.0.254/31'
set interfaces ethernet eth0 description 'P2P-to-B_CE2'
set interfaces ethernet eth2 address '100.0.0.51/31'
set interfaces ethernet eth2 description 'P2P-to-PE1'
set protocols bgp address-family ipv4-unicast network 200.0.0.0/16
set protocols bgp address-family ipv4-unicast network 200.0.0.1/32
set protocols bgp neighbor 100.0.0.50 address-family ipv4-unicast
set protocols bgp neighbor 100.0.0.50 description 'PE1'
set protocols bgp neighbor 100.0.0.50 remote-as '100'
set protocols bgp neighbor 200.0.0.255 address-family ipv4-unicast nexthop-self
set protocols bgp neighbor 200.0.0.255 description 'B_CE2'
set protocols bgp neighbor 200.0.0.255 remote-as 'internal'
set protocols bgp parameters log-neighbor-changes
set protocols bgp system-as '200'
set protocols static route 200.0.0.0/16 blackhole
set interfaces dummy dum0 address '200.0.0.2/32'
set interfaces ethernet eth0 address '200.0.0.255/31'
set interfaces ethernet eth0 description 'P2P-to-B_CE1'
set interfaces ethernet eth2 address '100.0.0.53/31'
set interfaces ethernet eth2 description 'P2P-to-PE2'
set protocols bgp address-family ipv4-unicast network 200.0.0.0/16
set protocols bgp address-family ipv4-unicast network 200.0.0.2/32
set protocols bgp neighbor 100.0.0.52 address-family ipv4-unicast
set protocols bgp neighbor 100.0.0.52 description 'PE2'
set protocols bgp neighbor 100.0.0.52 remote-as '100'
set protocols bgp neighbor 200.0.0.254 address-family ipv4-unicast nexthop-self
set protocols bgp neighbor 200.0.0.254 description 'B_CE1'
set protocols bgp neighbor 200.0.0.254 remote-as 'internal'
set protocols bgp parameters log-neighbor-changes
set protocols bgp system-as '200'
set protocols static route 200.0.0.0/16 blackhole
Verify our BGP sessions (both internal and external) are up.
vyos@B-CE1:~$ show bgp summary
IPv4 Unicast Summary (VRF default):
BGP router identifier 200.0.0.254, local AS number 200 vrf-id 0
BGP table version 18
RIB entries 5, using 960 bytes of memory
Peers 2, using 41 KiB of memory
Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt Desc
100.0.0.50 4 100 295 300 18 0 0 00:08:02 1 4 PE1
200.0.0.255 4 200 291 291 17 0 0 04:41:22 3 3 B_CE2
Validate the IPv4 prefixes we learn from our peers
Check out routes
vyos@B-CE1:~$ show bgp ipv4
BGP table version is 18, local router ID is 200.0.0.254, vrf id 0
Default local pref 100, local AS 200
Status codes: s suppressed, d damped, h history, * valid, > best, = multipath,
i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes: i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found
Network Next Hop Metric LocPrf Weight Path
* i0.0.0.0/0 200.0.0.255 0 100 0 100 i
*> 100.0.0.50 0 0 100 i
* i200.0.0.0/16 200.0.0.255 0 100 0 i
*> 0.0.0.0 0 32768 i
*> 200.0.0.1/32 0.0.0.0 0 32768 i
*>i200.0.0.2/32 200.0.0.255 0 100 0 i
On the provider side, we can now take a look at all our IPv4 forwarding information base (FIB) routes
vyos@PE1:~$ show ip route vrf all
Codes: K - kernel route, C - connected, S - static, R - RIP,
O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
f - OpenFabric,
> - selected route, * - FIB route, q - queued, r - rejected, b - backup
t - trapped, o - offload failure
VRF black:
C>* 100.0.0.50/31 is directly connected, br1.3002, 09:59:33
B>* 100.0.0.52/32 [20/0] via 2.2.2.2, br1.4000 onlink, weight 1, 09:59:07
B>* 100.0.0.53/32 [20/0] via 2.2.2.2, br1.4000 onlink, weight 1, 09:59:07
B>* 200.0.0.0/16 [20/0] via 100.0.0.51, br1.3002, weight 1, 05:26:04
VRF default:
S>* 0.0.0.0/0 [1/0] via 100.0.0.10, eth0, weight 1, 09:59:24
C>* 1.1.1.1/32 is directly connected, dum0, 09:59:34
B>* 2.2.2.2/32 [20/0] via 100.0.0.2, wg1, weight 1, 09:59:08
C>* 100.0.0.0/30 is directly connected, wg1, 09:59:28
C>* 100.0.0.10/31 is directly connected, eth0, 09:59:28
vyos@PE2:~$ show bgp vrf all ipv4
Instance default:
BGP table version is 2, local router ID is 2.2.2.2, vrf id 0
Default local pref 100, local AS 65002
Status codes: s suppressed, d damped, h history, * valid, > best, = multipath,
i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes: i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found
Network Next Hop Metric LocPrf Weight Path
*> 1.1.1.1/32 100.0.0.1 0 0 65001 i
*> 2.2.2.2/32 0.0.0.0 0 32768 i
Instance black:
BGP table version is 15, local router ID is 2.2.2.2, vrf id 8
Default local pref 100, local AS 65002
Status codes: s suppressed, d damped, h history, * valid, > best, = multipath,
i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes: i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found
Network Next Hop Metric LocPrf Weight Path
*> 100.0.0.50/32 1.1.1.1< 0 65001 i
*> 100.0.0.51/32 1.1.1.1< 0 65001 i
*> 200.0.0.0/16 100.0.0.53 0 0 200 i
* 1.1.1.1< 0 0 65001 200 i
Conclusion
Stay tuned; I'm already considering another topic - EVPN ESI multihoming!