1. Running VPP L3 forward application
1.1. Prerequisites
Linux booted on x86 Host and Octeon DPU
Login to x-86 host and Octeon DPU
1.2. DAO Environment Setup
DAO should be preinstalled as given in below link. Downloading and installing DAO packages
1.3. Steps to run on OCTEON DPU
Prepare vpp startup configuration file as below. This is the minimal startup.conf required to start vpp octeon plugin with 2 devices (see plugin and devices section). NOTE: Default vpp configuration file is placed at /etc/vpp/startup.conf path.
root@localhost:~#cat /etc/vpp/startup.conf
unix
{
log /var/log/vpp/vpp.log
cli-listen /run/vpp/cli.sock
cli-no-pager
full-coredump
}
socksvr
{
socket-name /run/vpp/api.sock
}
statseg
{
per-node-counters on
}
plugins
{
plugin default
{
disable
}
plugin dev_octeon_plugin.so
{
enable
}
plugin perfmon_plugin.so
{
enable
}
}
cpu
{
main-core 1
corelist-workers 2
}
buffers
{
buffers-per-numa 107520
}
devices
{
dev pci/0002:02:00.0
{
driver octeon
port 0
{
name eth0
num-tx-queues 16
num-rx-queues 17
}
}
dev pci/0002:03:00.0
{
driver octeon
port 0
{
name eth1
num-tx-queues 16
num-rx-queues 17
}
}
}
Bind vpp pktio interfaces to vfio-pci dpdk driver.
## Identify devices using below command
root@localhost:~# lspci | grep "Octeon Tx2 RVU Physical Function"
0002:02:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 RVU Physical Function (rev 54)
0002:03:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 RVU Physical Function (rev 54)
0002:04:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 RVU Physical Function (rev 54)
0002:05:00.0 Ethernet controller: Cavium, Inc. Octeon Tx2 RVU Physical Function (rev 54)
root@localhost:~#
## Identify interface nake corresponding to each device (see if=<interface name>)
root@localhost:~# dpdk-devbind.py -s | grep 0002:02:00.0
0002:02:00.0 'Octeon Tx2 RVU Physical Function a063' numa_node=0 if=enP2p2s0 drv=rvu_nicpf unused=vfio-pci
root@localhost:~#
## Identify link speed and link status
root@localhost:~# ethtool enP2p2s0 | grep -iE "Speed|Link detected"
Speed: 25000Mb/s
Link detected: yes
root@localhost:~#
## From above detail this link is 25GBPS and status is "yes", so I can use this.
## Perform similar check for other devices.
## Identified 2 devices (0002:02:00.0 0002:03:00.0)
## Bind these 2 devices to vfio-pci driver.
## Update these device PCI addresses in startup.conf.
root@localhost:~# dpdk-devbind.py -b vfio-pci 0002:02:00.0 0002:03:00.0
Run vpp
root@localhost:~# vpp -c /etc/vpp/startup.conf
Run vppctl and apply the vpp interface configurations.
root@localhost:~# vppctl
_______ _ _ _____ ___
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
eth0 1 down 9000/0/0/0
eth1 2 down 9000/0/0/0
local0 0 down 0/0/0/0
vpp#
vpp#set int state eth0 up
vpp#set int state eth1 up
vpp#set int ip address eth0 10.10.10.1/24
vpp#set int ip address eth1 20.20.20.1/24
vpp#ip neighbor eth0 10.10.10.2 0c:42:a1:67:d5:e7
vpp#ip neighbor eth1 20.20.20.2 0c:42:a1:67:d5:e6
## MAC addresses 0c:42:a1:67:d5:e7 and 0c:42:a1:67:d5:e7 are taken from x-86 interfaces.(Refer section 1.4.1)
## To see the MAC addresses of VPP interfaces run below command in vppctl and check Ethernet address(MAC).
vpp#DBGvpp# show hardware-interfaces eth0
Name Idx Link Hardware
eth0 1 down eth0
Link speed: unknown
Ethernet address a2:6e:ed:21:b4:9e
Device:
Driver is 'octeon', bus is 'pci', description is 'Marvell Octeon Resource Virtualization Unit PF'
PCIe address is 0002:02:00.0, port is P0, speed is unknown speed x0 (max unknown speed x0)
Assigned process node is 'pci/0002:02:00.0-process'
Device Specific Arguments:
Name Value Default Description
n_desc <not set> 16384 number of cpt descriptors, applicable to cpt devices only
Port 0:
Hardware Address is a2:6e:ed:21:b4:9e, 1 RX queues (max 64), 2 TX queues (max 64)
Max RX frame size is 16380 (max supported 16380)
Caps: rss
RX Offloads: ip4-cksum
TX Offloads: ip4-cksum
Device Specific Port Status:
Device Specific Port Arguments:
Name Value Default Description
allmulti <not set> 0 Set allmulti mode, applicable to network devices only
eth_pause_frame <not set> 0 Enable ethernet pause frame support, applicable to network devices only
switch_header <not set> '' Enable switch header and set specific switch header type, applicable to network devices only
Interface assigned, interface name is 'eth0', RX node is 'eth0-rx'
RX queue 0:
Size is 1024, buffer pool index is 0
Polling thread is 1, enabled, not-started, polling mode
TX queue 0:
Size is 1024
Used by thread 0
TX queue 1:
Size is 1024
Used by thread 1
vpp#
1.4. Steps to run on x-86 Host
Identify interfaces which are connected to Octeon DPU
## Identify devices
root@transport-2:~# lspci | grep Mellanox
5e:00.0 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex]
5e:00.1 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex]
b1:00.0 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
b1:00.1 Ethernet controller: Mellanox Technologies MT27700 Family [ConnectX-4]
d9:00.0 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex]
d9:00.1 Ethernet controller: Mellanox Technologies MT28800 Family [ConnectX-5 Ex]
root@transport-2:~#
## Identify interface name (see if=<Interface name>)
root@transport-2:~# dpdk-devbind.py -s | grep d9:00.1
0000:d9:00.1 'MT28800 Family [ConnectX-5 Ex] 1019' numa_node=1 if=enp217s0f1np1 drv=mlx5_core unused=vfio-pci *Active*
root@transport-2:~#
## Identify link speed and link status
root@transport-2:~# ethtool enp217s0f1np1 | grep -iE "Speed|Link detected"
Speed: 25000Mb/s
Link detected: yes
root@transport-2:~#
## Identify MAC address
root@transport-2:~# ifconfig enp217s0f1np1 | grep ether
ether 0c:42:a1:67:d5:e7 txqueuelen 1000 (Ethernet)
root@transport-2:~#
## Identified interfaces are enp217s0f1np1 and enp217s0f0np0
NOTE: To confirm links on both x-86 and Octeon side, verify using ping.
For example:
Interface on Octeon : enP2p2s0
Interface on x-86 : enp217s0f0np0
Assign IP address to both interfaces in same subnet.
#ifconfig enP2p2s0 20.20.20.1/24 up
#ifconfig enp217s0f0np0 20.20.20.2/24 up
Ping each other and verify link connectivity.
Assign IP addresses to interfaces
root@localhost:~#ifconfig enp217s0f1np1 10.10.10.2/24 up
root@localhost:~#ifconfig enp217s0f0np0 20.20.20.2/24 up
Install scapy traffic generator on x-86 host (If not already installed).
root@localhost:~#apt-get install scapy
Run scapy and send packets
root@localhost:~#scapy
INFO: Can't import PyX. Won't be able to use psdump() or pdfdump().
WARNING: No alternative Python interpreters found ! Using standard Python shell instead.
INFO: Using the default Python shell: History is disabled.
aSPY//YASa
apyyyyCY//////////YCa |
sY//////YSpcs scpCY//Pp | Welcome to Scapy
ayp ayyyyyyySCP//Pp syY//C | Version 2.6.1
AYAsAYYYYYYYY///Ps cY//S |
pCCCCY//p cSSps y//Y | https://github.com/secdev/scapy
SPPPP///a pP///AC//Y |
A//A cyP////C | Have fun!
p///Ac sC///a |
P////YCpc A//A | Craft packets like I craft my beer.
scccccp///pSP///p p//Y | -- Jean De Clerck
sY/////////y caa S//P |
cayCyayP//Ya pY/Ya
sY/PsY////YCc aC//Yp
sc sccaCY//PCypaapyCP//YSs
spCPY//////YPSps
ccaacs
>>>sendp(Ether(dst="f2:b8:4c:4b:55:d9",src="0c:42:a1:67:d5:e7")/IP(src="10.10.10.2",dst="20.20.20.2",len=60)/UDP(dport=4000,sport=5000,len=40)/Raw(RandString(size=32)), iface="enp217s0f1np1", return_packets=True, count=10)
..........
Sent 10 packets.
<PacketList: TCP:0 UDP:10 ICMP:0 Other:0>
>>>
>>>sendp(Ether(dst="ee:bf:45:74:e9:12",src="0c:42:a1:67:d5:e6")/IP(src="20.20.20.2",dst="10.10.10.2",len=60)/UDP(dport=5000,sport=4000,len=40)/Raw(RandString(size=32)), iface="enp217s0f0np0", return_packets=True, count=10)
..........
Sent 10 packets.
<PacketList: TCP:0 UDP:10 ICMP:0 Other:0>
>>>
## Destination MACs are taken from vpp eth0 and eth1 interfaces. (f2:b8:4c:4b:55:d9 and ee:bf:45:74:e9:12). (Refer section 1.3.4)
1.5. Verify Traffic On Octeon DPU
root@localhost:~# vppctl
_______ _ _ _____ ___
__/ __/ _ \ (_)__ | | / / _ \/ _ \
_/ _// // / / / _ \ | |/ / ___/ ___/
/_/ /____(_)_/\___/ |___/_/ /_/
vpp#
vpp# show int
Name Idx State MTU (L3/IP4/IP6/MPLS) Counter Count
eth0 1 up 9000/0/0/0 rx packets 10
rx bytes 740
tx packets 10
tx bytes 740
ip4 10
eth1 2 up 9000/0/0/0 rx packets 10
rx bytes 740
tx packets 10
tx bytes 740
ip4 10
local0 0 down 0/0/0/0
vpp#