sNAT Hardware offload cannot be implemented
wanghuiheze opened this issue · 4 comments
wanghuiheze commented
Test the sNAT hardware offload
Experimental environment:
central 10.185.11.71/24 - OVN Central
gateway01 172.31.5.23/24 - OVN Gateway Node
node1 172.31.1.21/24 – will serve as an OVN Host
client 172.31.6.24/24 - physical network node
#We use namespace instead of VM
namespace 10.199.100.10 - in node1 Node
tunnel:geneve
#OVS version
[root@gateway01 ~]# ovs-vsctl -V
ovs-vsctl (Open vSwitch) 2.17.7-e054917
DB Schema 8.3.1
[root@node01 ~]# ovs-vsctl -V
ovs-vsctl (Open vSwitch) 2.17.6-c5d234f
DB Schema 8.3.0
#OVN version
[root@newcentral ~]# ovn-nbctl -V
ovn-nbctl 23.03.0
Open vSwitch Library 3.1.0
DB Schema 7.0.0
[root@newcentral ~]# ovn-sbctl -V
ovn-sbctl 23.03.0
Open vSwitch Library 3.1.0
DB Schema 20.27.0
#Kernal version
[root@gateway01 ~]# uname -a
Linux gateway01 5.14.0-284.11.1.el9_2.x86_64 #1 SMP PREEMPT_DYNAMIC Tue May 9 05:49:00 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux
[root@gateway01 ~]# cat /etc/redhat-release
AlmaLinux release 9.2 (Turquoise Kodkod)
[root@node01 ~]# uname -a
Linux node01 4.18.0-425.3.1.el8.x86_64 #1 SMP Tue Nov 8 14:08:25 EST 2022 x86_64 x86_64 x86_64 GNU/Linux
[root@node01 ~]# cat /etc/redhat-release
AlmaLinux release 8.7 (Stone Smilodon)
____ ____
| client | 172.31.4.0/24 Physical Network
---------
|
____|____
| switch | localSW1
---------
____|____ snat 123.123.123.123 10.199.100.0/24
| router | GR1 port 'GR1-localSW1': 172.31.5.230/24
--------- port 'GR1-joinSW': 172.16.255.3/24
____|____
| switch | joinSW 172.16.255.0/24
---------
|
____|____
| router | dr port 'dr1-joinSW': 172.16.255.2/24
--------- port 'dr-sw1': 10.199.100.1/24
/ \ port 'dr-sw2': 10.199.200.1/24
_______/_ _\_______
| sw1 | | sw2 | sw1:10.199.100.0/24
--------- --------- sw2:10.199.200.0/24
/
_______/_
| sw1-vm1 |
---------
10.199.100.10
View the ovn database:
[root@newcentral ~]# ovn-nbctl show
switch 9b507f28-6635-43d5-a782-f8561d01a6ff (subnet_04b62bbe1f147000)
port subnet_04b62bbe1f147000-router
type: router
router-port: vpc_049e374424847000-subnet_04b62bbe1f147000
port i_04b636e391c47000
addresses: ["00:00:19:92:00:20 10.199.200.20/24"]
switch 84b95eb4-8244-4fd4-b39a-1555e952a0b4 (vpc_04a7d57200347000-vbr)
port ri_04af32c445847000
type: router
router-port: vbr_04a0be3b4b247000-vpc_04a7d57200347000
port ri_04aacfb978547000
type: router
router-port: vbr_04a0be3427447000-vpc_04a7d57200347000
port vpc_04a7d57200347000-join-port
type: router
router-port: vpc_04a7d57200347000-vbr-port
switch 939b6ec0-be19-49b4-aa82-dbfa626d2022 (vbr_04a0be3b4b247000-local)
port vbr_04a0be3b4b247000-local-network
type: localnet
addresses: ["unknown"]
port vbr_04a0be3b4b247000-local-router
type: router
router-port: vbr_04a0be3b4b247000-route-local
switch 80573916-6550-4a34-8570-bc67caad05cf (vpc_04afbc45e9847000-vbr)
port vpc_04afbc45e9847000-join-port
type: router
router-port: vpc_04afbc45e9847000-vbr-port
switch 74a43bfe-79ea-4400-b02e-3bd7043e6dc0 (subnet_049e37f43f047000)
port subnet_049e37f43f047000-router
type: router
router-port: vpc_049e374424847000-subnet_049e37f43f047000
port i_04b0bdce8661c000
addresses: ["AE:91:91:13:D9:FE 10.199.100.12/24"]
port i_04b0bdcbade1c000
addresses: ["AE:BE:BE:6B:0B:83 10.199.100.11/24"]
port i_04aacda5ceb47000
addresses: ["00:00:19:91:00:10 10.199.100.10/24"]
switch 5832950f-ec29-42e9-ad37-09dd194638ce (vbr_04a0be3427447000-local)
port vbr_04a0be3427447000-local-network
type: localnet
addresses: ["unknown"]
port vbr_04a0be3427447000-local-router
type: router
router-port: vbr_04a0be3427447000-route-local
switch 9e5e34e3-259f-4078-a833-6efa09ed6d15 (vpc_049e374424847000-vbr)
port ri_04a0be3cb4347000
type: router
router-port: vbr_04a0be3427447000-vpc_049e374424847000
port vpc_049e374424847000-join-port
type: router
router-port: vpc_049e374424847000-vbr-port
port ri_04a0be4962547000
type: router
router-port: vbr_04a0be3b4b247000-vpc_049e374424847000
router bd40b34c-8cf1-46a1-8dd9-9d5ef42e07a2 (vpc_04a7d57200347000)
port vpc_04a7d57200347000-pcc_04aacd2929547000
mac: "AE:56:56:A2:49:B2"
networks: ["10.31.0.2/24"]
route-table: vtb_04a7d57200347000
port vpc_04a7d57200347000-vbr-port
mac: "AE:EF:EF:DB:2F:F3"
networks: ["172.16.255.2/24"]
route-table: vtb_04a7d57200347000
router 1ea16498-bf2b-4182-90e2-64bab54bd1fb (vbr_04a0be3427447000)
port vbr_04a0be3427447000-vpc_04a7d57200347000
mac: "AE:DD:DD:E4:9E:58"
networks: ["172.16.255.3/24"]
port vbr_04a0be3427447000-route-local
mac: "AE:A9:A9:0C:6F:C8"
networks: ["172.31.5.230/24"]
port vbr_04a0be3427447000-vpc_049e374424847000
mac: "AE:CD:CD:04:D7:D4"
networks: ["172.16.255.3/24"]
nat 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
external ip: "123.123.123.123"
logical ip: "10.199.100.0/24"
type: "snat"
router 0ef0e9c1-4352-4ba0-9002-772f6cc7366b (vpc_049e374424847000)
port vpc_049e374424847000-pcc_04aacd2929547000
mac: "AE:56:56:A2:49:B1"
networks: ["10.31.0.1/24"]
route-table: vtb_049e374424847000
port vpc_049e374424847000-subnet_049e37f43f047000
mac: "AE:95:95:65:27:A6"
networks: ["10.199.100.1/24"]
route-table: vtb_049e374424847000
port vpc_049e374424847000-subnet_04b62bbe1f147000
mac: "AE:8D:8D:55:BA:26"
networks: ["10.199.200.1/24"]
route-table: vtb_049e374424847000
port vpc_049e374424847000-vbr-port
mac: "AE:03:03:17:EB:DF"
networks: ["172.16.255.2/24"]
route-table: vtb_049e374424847000
router db24641e-7597-47fc-81d9-de2ec15ca016 (vpc_04afbc45e9847000)
port vpc_04afbc45e9847000-vbr-port
mac: "AE:BC:BC:4F:98:7C"
networks: ["172.16.255.2/24"]
[root@newcentral ~]# ovn-nbctl show vbr_04a0be3427447000
router 1ea16498-bf2b-4182-90e2-64bab54bd1fb (vbr_04a0be3427447000)
port vbr_04a0be3427447000-vpc_04a7d57200347000
mac: "AE:DD:DD:E4:9E:58"
networks: ["172.16.255.3/24"]
port vbr_04a0be3427447000-route-local
mac: "AE:A9:A9:0C:6F:C8"
networks: ["172.31.5.230/24"]
port vbr_04a0be3427447000-vpc_049e374424847000
mac: "AE:CD:CD:04:D7:D4"
networks: ["172.16.255.3/24"]
nat 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
external ip: "123.123.123.123"
logical ip: "10.199.100.0/24"
type: "snat"
#NAT table
[root@newcentral ~]# ovn-nbctl list nat
_uuid : 9bd37dcd-2a01-4ff6-9089-bce2558fc5b9
allowed_ext_ips : []
exempted_ext_ips : []
external_ids : {}
external_ip : "123.123.123.123"
external_mac : []
external_port_range : ""
gateway_port : []
logical_ip : "10.199.100.0/24"
logical_port : []
options : {stateless="false"}
type : snat
#Logical_router table
[root@newcentral ~]# ovn-nbctl find logical_router name="vbr_04a0be3427447000"
_uuid : 1ea16498-bf2b-4182-90e2-64bab54bd1fb
copp : []
enabled : []
external_ids : {vendor=sina}
load_balancer : []
load_balancer_group : []
name : vbr_04a0be3427447000
nat : [9bd37dcd-2a01-4ff6-9089-bce2558fc5b9]
options : {chassis=gateway01}
policies : []
ports : [52a9cb30-1a9f-4855-95a7-f7bc5d741d82, 5bcf8982-f784-405c-8ada-ef73bd57c4a6, 77b73cdb-8c91-4221-8fac-4253ac5a0e2d]
static_routes : [3919f68b-cc08-46c2-95c0-93b4d5305e44, 3a71b83b-1333-447f-a3d3-84ec0483fc96, 437f7eaa-4fe5-41f4-817e-30e09ca8228a, 90322d85-0f43-4cc5-9916-60000d620da5]
[root@newcentral ~]# ovn-sbctl show
Chassis gateway01
hostname: gateway01
Encap geneve
ip: "172.31.5.23"
options: {csum="true"}
Port_Binding vbr_04a0be3427447000-vpc_04a7d57200347000
Port_Binding vbr_04a0be3427447000-route-local
Port_Binding vbr_04a0be3427447000-vpc_049e374424847000
Port_Binding ri_04aacfb978547000
Port_Binding vbr_04a0be3427447000-local-router
Port_Binding ri_04a0be3cb4347000
Chassis node01
hostname: node01
Encap geneve
ip: "172.31.1.21"
options: {csum="true"}
Port_Binding i_04aacda5ceb47000
Some information about the gateway01 chassis is listed below
[root@gateway01 ~]# ovs-vsctl show
ad701a9e-a75c-486f-870d-1d58605d28df
Bridge br-ex
Port ens1f0
Interface ens1f0
Port patch-vbr_04a0be3427447000-local-network-to-br-int
Interface patch-vbr_04a0be3427447000-local-network-to-br-int
type: patch
options: {peer=patch-br-int-to-vbr_04a0be3427447000-local-network}
Port br-ex
Interface br-ex
type: internal
Bridge br-int
fail_mode: secure
datapath_type: system
Port ovn-node01-0
Interface ovn-node01-0
type: geneve
options: {csum="true", key=flow, remote_ip="172.31.1.21"}
Port patch-br-int-to-vbr_04a0be3427447000-local-network
Interface patch-br-int-to-vbr_04a0be3427447000-local-network
type: patch
options: {peer=patch-vbr_04a0be3427447000-local-network-to-br-int}
Port br-int
Interface br-int
type: internal
ovs_version: "2.17.7-e054917"
[root@gateway01 ~]# ovs-vsctl list openvswitch
ovs-vsctl: unknown table "openvswitch"
[root@gateway01 ~]# ovs-vsctl list open_vswitch
_uuid : ad701a9e-a75c-486f-870d-1d58605d28df
bridges : [03f31940-ceab-4cb5-a2a9-b25071509d2c, 6e0703d0-192c-4178-af00-c2aff8e7ea55]
cur_cfg : 17
datapath_types : [netdev, system]
datapaths : {system=f9d52158-c861-4cb4-8cfa-acfd26440534}
db_version : "8.3.1"
doca_initialized : false
doca_version : none
dpdk_initialized : false
dpdk_version : "MLNX_DPDK 22.11.1.4.2"
external_ids : {hostname=gateway01, ovn-bridge-mappings="physnet1:br-ex", ovn-encap-ip="172.31.5.23", ovn-encap-type=geneve, ovn-remote="tcp:10.185.11.71:6642", rundir="/var/run/openvswitch", system-id=gateway01}
iface_types : [bareudp, erspan, geneve, gre, gtpu, internal, ip6erspan, ip6gre, lisp, patch, stt, system, tap, vxlan]
manager_options : []
next_cfg : 17
other_config : {hw-offload="true", ovn-chassis-idx-gateway01="", vlan-limit="0"}
ovs_version : "2.17.7-e054917"
ssl : []
statistics : {}
system_type : almalinux
system_version : "9.2"
[root@gateway01 ~]# ethtool -i ens1f0
driver: mlx5_core
version: 23.04-1.1.3
firmware-version: 22.37.1014 (MT_0000000359)
expansion-rom-version:
bus-info: 0000:02:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
[root@gateway01 ~]# ethtool -k ens1f0 | grep offload
tcp-segmentation-offload: on
generic-segmentation-offload: on
generic-receive-offload: on
large-receive-offload: off
rx-vlan-offload: off [requested on]
tx-vlan-offload: on
l2-fwd-offload: off [fixed]
hw-tc-offload: on
esp-hw-offload: off [fixed]
esp-tx-csum-hw-offload: off [fixed]
rx-udp_tunnel-port-offload: on
tls-hw-tx-offload: off [fixed]
tls-hw-rx-offload: off [fixed]
macsec-hw-offload: off [fixed]
hsr-tag-ins-offload: off [fixed]
hsr-tag-rm-offload: off [fixed]
hsr-fwd-offload: off [fixed]
hsr-dup-offload: off [fixed]
10.199.100.10 -> 172.31.6.24 (snat(src=123.123.123.123))
1.With the iperf3 test, there seems to be no problem, all flows are unloaded normally, and the traffic is as expected.
#iperf3 test
[root@gateway02 ~]# iperf3 -s
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
Accepted connection from 123.123.123.123, port 56014
[ 5] local 172.31.6.24 port 5201 connected to 123.123.123.123 port 56022
[ ID] Interval Transfer Bitrate
[ 5] 0.00-1.00 sec 2.48 GBytes 21.3 Gbits/sec
[ 5] 1.00-2.00 sec 2.61 GBytes 22.4 Gbits/sec
[ 5] 2.00-3.00 sec 2.61 GBytes 22.4 Gbits/sec
[ 5] 3.00-4.00 sec 2.61 GBytes 22.4 Gbits/sec
[ 5] 4.00-5.00 sec 2.61 GBytes 22.5 Gbits/sec
[ 5] 5.00-6.00 sec 2.61 GBytes 22.5 Gbits/sec
[ 5] 6.00-7.00 sec 2.61 GBytes 22.5 Gbits/sec
[ 5] 7.00-8.00 sec 2.61 GBytes 22.5 Gbits/sec
[ 5] 8.00-9.00 sec 2.61 GBytes 22.4 Gbits/sec
[ 5] 9.00-10.00 sec 2.61 GBytes 22.4 Gbits/sec
[root@node01 ~]# ip netns exec sw1-vm1 iperf3 -c 172.31.6.24 -i 1 -t 1000
Connecting to host 172.31.6.24, port 5201
[ 5] local 10.199.100.10 port 56022 connected to 172.31.6.24 port 5201
[ ID] Interval Transfer Bitrate Retr Cwnd
[ 5] 0.00-1.00 sec 2.59 GBytes 22.2 Gbits/sec 59 1.00 MBytes
[ 5] 1.00-2.00 sec 2.61 GBytes 22.4 Gbits/sec 103 994 KBytes
[ 5] 2.00-3.00 sec 2.61 GBytes 22.4 Gbits/sec 59 937 KBytes
[ 5] 3.00-4.00 sec 2.61 GBytes 22.4 Gbits/sec 69 957 KBytes
[ 5] 4.00-5.00 sec 2.61 GBytes 22.5 Gbits/sec 29 1002 KBytes
[ 5] 5.00-6.00 sec 2.61 GBytes 22.4 Gbits/sec 40 1.04 MBytes
[ 5] 6.00-7.00 sec 2.61 GBytes 22.5 Gbits/sec 18 756 KBytes
#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:8546650, bytes:12084949738, used:0.980s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0x7d),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:8546646, bytes:12084948109, used:0.980s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0x7e),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:8546645, bytes:12084946695, used:0.980s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:1615469836, bytes:2384348244165, used:0.980s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:ct(zone=16,nat),recirc(0x7f)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0x7f),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0x80),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:159918, bytes:11197872, used:0.980s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:37541599, bytes:4654672765, used:0.980s, actions:2
#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:5ac63871-658b-4b25-8e56-217768575a16, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:25590231, bytes:36184573272, used:0.180s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
ufid:af46cfe3-bc62-4f74-ab05-d449c2e50bfa, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x7d),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:25590227, bytes:36184571643, used:0.180s, offloaded:yes, dp:tc, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
ufid:f219f29b-222c-4f58-a1cf-a6bcf43af76f, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x7e),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:25590227, bytes:36184571643, used:0.180s, offloaded:yes, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:1632513421, bytes:2409504570083, used:0.180s, offloaded:yes, dp:tc, actions:br-ex
ufid:a25c1957-775d-4fcf-91d4-06a2675e4a9c, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x21),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:ct(zone=16,nat),recirc(0x7f)
ufid:87c65051-3be0-40da-9f68-b8df8102abe6, skb_priority(0/0),skb_mark(0/0),ct_state(0x20/0x30),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x7f),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=10.199.100.0/255.255.255.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ufid:ee212541-e807-4efc-8119-1c0f4b0b9671, skb_priority(0/0),skb_mark(0/0),ct_state(0x28/0x39),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x80),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no),tcp(src=0/0,dst=0/0), packets:480973, bytes:33677870, used:0.180s, offloaded:yes, dp:tc, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),genev_sys_6081
ufid:01d285c1-2635-4fd8-8738-f8df0173163f, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:37862656, bytes:4694489941, used:0.180s, offloaded:yes, dp:tc, actions:ens1f0
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:51895, bytes:6175505, used:0.233s, dp:ovs, actions:drop
#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:29841401, bytes:42195727652, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x7d)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0x7d),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:29841397, bytes:42195726023, used:0.000s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x7e)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0x7e),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:29841397, bytes:42195726023, used:0.000s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:1636764592, bytes:2415779298479, used:0.000s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:ct(zone=16,nat),recirc(0x7f)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0x7f),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x80)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0x80),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:561852, bytes:39342064, used:0.000s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:37943535, bytes:4704521601, used:0.000s, actions:2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:51896, bytes:6175624, used:0.100s, actions:drop
#View the conntrack：
[root@gateway01 ~]$ ovs-appctl dpctl/dump-conntrack | wc -l
4
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=54346,dport=5201),reply=(src=172.31.6.24,dst=10.199.100.10,sport=5201,dport=54346),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=50604,dport=5201),reply=(src=172.31.6.24,dst=123.123.123.123,sport=5201,dport=50604),zone=16
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=50604,dport=5201),reply=(src=172.31.6.24,dst=10.199.100.10,sport=5201,dport=50604),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=54346,dport=5201),reply=(src=172.31.6.24,dst=123.123.123.123,sport=5201,dport=54346),zone=16
#View the traffic:
[root@gateway01 ~]# sar -n DEV 1
Linux 5.14.0-284.11.1.el9_2.x86_64 (gateway01) 08/04/2023 _x86_64_ (48 CPU)
11:56:54 PM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil
11:56:55 PM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM enp8s0f0 1.00 1.00 0.06 0.18 0.00 0.00 0.00 0.00
11:56:55 PM enp8s0f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM ens2f0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM enp8s0f2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM ens2f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM enp8s0f3 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM ens1f0 1394058.00 1394071.00 1973751.61 1897756.30 0.00 0.00 0.00 16.17
11:56:55 PM ens1f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM ovs-system 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM br-ex 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM br-int 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
11:56:55 PM genev_sys_6081 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
result:Use iperf3 to test that everything is OK.
2.Using dperf to test udp packets, it was found that some flows were not offloaded, and the traffic did not meet the expected value. The CPU occupied 100% of the ksoftirqd process
#dperf test
[root@node01 ~]# ip netns exec sw1-vm1 ./dperf.x -c dperf.conf
seconds 18 cpuUsage 7 7 7 7 7 7
pktRx 0 pktTx 5,000,000 bitsRx 0 bitsTx 4,240,000,000 dropTx 0
tcpRx 0 tcpTx 0 udpRx 0 udpTx 5,000,000
arpRx 0 arpTx 0 icmpRx 0 icmpTx 0
tosRx 0 otherRx 0 badRx 0
udpRt 5,000,000 udpDrop 0 tcpDrop 0
skOpen 0 skClose 0 skCon 5,000 skErr 0
ierrors 0 oerrors 0 imissed 0
seconds 19 cpuUsage 7 7 7 7 7 7
pktRx 0 pktTx 5,000,084 bitsRx 0 bitsTx 4,240,071,232 dropTx 0
tcpRx 0 tcpTx 0 udpRx 0 udpTx 5,000,084
arpRx 0 arpTx 0 icmpRx 0 icmpTx 0
tosRx 0 otherRx 0 badRx 0
udpRt 5,000,084 udpDrop 0 tcpDrop 0
skOpen 0 skClose 0 skCon 5,000 skErr 0
ierrors 0 oerrors 0 imissed 0
#View the traffic:
[root@gateway01 ~]# sar -n DEV 1
Linux 5.14.0-284.11.1.el9_2.x86_64 (gateway01) 08/05/2023 _x86_64_ (48 CPU)
12:07:23 AM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil
12:07:24 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM enp8s0f0 2.00 1.00 0.12 0.18 0.00 0.00 0.00 0.00
12:07:24 AM enp8s0f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM ens2f0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM enp8s0f2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM ens2f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM enp8s0f3 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM ens1f0 4920514.00 167186.00 807275.69 17959.43 0.00 0.00 1.00 6.61
12:07:24 AM ens1f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM ovs-system 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM br-ex 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM br-int 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM genev_sys_6081 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:24 AM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil
12:07:25 AM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM enp8s0f0 6.00 13.00 0.38 7.21 0.00 0.00 0.00 0.01
12:07:25 AM enp8s0f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM ens2f0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM enp8s0f2 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM ens2f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM enp8s0f3 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM ens1f0 5001917.00 170087.00 820623.04 18270.96 0.00 0.00 0.00 6.72
12:07:25 AM ens1f1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM ovs-system 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM br-ex 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM br-int 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
12:07:25 AM genev_sys_6081 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00
#View NIC traffic
[root@gateway01 ~]# ifconfig ens1f0
ens1f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether e8:eb:d3:3a:d7:f8 txqueuelen 1000 (Ethernet)
RX packets 8797857529 bytes 5322858965370 (4.8 TiB)
RX errors 0 dropped 1219084890 overruns 0 frame 0
TX packets 3279243046 bytes 4213255342903 (3.8 TiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@gateway01 ~]# ifconfig ens1f0
ens1f0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether e8:eb:d3:3a:d7:f8 txqueuelen 1000 (Ethernet)
RX packets 8806548992 bytes 5324319131049 (4.8 TiB)
RX errors 0 dropped 1227480644 overruns 0 frame 0
TX packets 3279538578 bytes 4213287851313 (3.8 TiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
#View process
[root@gateway01 ~]# top
top - 00:14:38 up 2 days, 1:49, 1 user, load average: 0.49, 0.29, 0.18
Tasks: 515 total, 2 running, 513 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.0 us, 0.0 sy, 0.0 ni, 97.9 id, 0.0 wa, 0.0 hi, 2.1 si, 0.0 st
MiB Mem : 128272.0 total, 125688.7 free, 2726.4 used, 818.1 buff/cache
MiB Swap: 4096.0 total, 4096.0 free, 0.0 used. 125545.6 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
166 root 20 0 0 0 0 R 100.0 0.0 22:53.12 ksoftirqd/29
33054 openvsw+ 10 -10 4350096 291584 25892 S 1.0 0.2 23:26.05 ovs-vswitchd
37613 root 20 0 10912 4540 3376 R 1.0 0.0 0:00.08 top
1 root 20 0 174004 18332 10924 S 0.0 0.0 0:06.86 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:00.12 kthreadd
3 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_gp
4 root 0 -20 0 0 0 I 0.0 0.0 0:00.00 rcu_par_gp
#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,ttl=63,frag=no), packets:16764463, bytes:1542330596, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:2491914653, bytes:3074457645572, used:0.640s, actions:1
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:5, bytes:350, used:0.730s, actions:2
#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:8ec2774d-8efe-4b14-a8b5-c53fef16e920, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,tos=0/0,ttl=63,frag=no),udp(src=0/0,dst=0/0), packets:17616859, bytes:1620751028, used:0.000s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
ufid:56740c84-edb0-4037-91de-7216907857ec, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x85),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:17616825, bytes:1620747900, used:0.000s, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
ufid:9ee0daa2-d3eb-4dfe-9231-15bbbac87ded, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x89),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=17,tos=0/0,ttl=0/0,frag=no),udp(src=0/0,dst=0/0), packets:17616828, bytes:1620748176, used:0.000s, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:2519113731, bytes:3079027090586, used:0.200s, offloaded:yes, dp:tc, actions:br-ex
ufid:32a63a03-7bc5-4f97-b418-5a593875f363, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:5, bytes:350, used:5.730s, offloaded:yes, dp:tc, actions:ens1f0
ufid:4232db89-e385-416f-bdfa-adf82ea5a5dc, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, offloaded:yes, dp:tc, actions:drop
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:0, bytes:0, used:never, dp:ovs, actions:drop
#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=17,ttl=63,frag=no), packets:17894519, bytes:1646295748, used:0.000s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0x85)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-rpl+trk),recirc_id(0x85),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:17894486, bytes:1646292712, used:0.000s, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-inv+trk),ct_mark(0/0x2),recirc_id(0x89),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=17,frag=no), packets:17894488, bytes:1646292896, used:0.000s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:2524233471, bytes:3079887206906, used:0.810s, actions:1
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:5, bytes:350, used:7.360s, actions:2
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, actions:drop
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:0, bytes:0, used:never, actions:drop
#View the conntrack：
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | wc -l
10000
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | less
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2391,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=2391),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3512,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3512),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3351,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3351),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5266,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5266),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5273,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5273),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3268,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=3268),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6768,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=6768),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5026,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5026),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3743,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3743),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6191,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6191),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4161,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=4161),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4525,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4525),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5846,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5846),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4898,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4898),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3361,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3361),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5677,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5677),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5285,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5285),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2536,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=2536),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6255,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6255),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2391,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=2391),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3512,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3512),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3351,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3351),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5266,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5266),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5273,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5273),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3268,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=3268),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6768,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=6768),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5026,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5026),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3743,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3743),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6191,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6191),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4161,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=4161),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4525,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4525),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5846,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5846),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=4898,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=4898),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=3361,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=3361),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5677,dport=80),reply=(src=172.31.6.24,dst=10.199.100.10,sport=80,dport=5677),zone=6
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=5285,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=5285),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=2536,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=2536),zone=16
udp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=6255,dport=80),reply=(src=172.31.6.24,dst=123.123.123.123,sport=80,dport=6255),zone=16
…………………………
#View vswitchd log
[root@gateway01 ~]# tail -f /var/log/openvswitch/ovs-vswitchd.log
2023-08-04T16:56:12.633Z|00051|bridge|INFO|bridge br-int: added interface br-int on port 65534
2023-08-04T16:56:12.633Z|00052|bridge|INFO|bridge br-ex: using datapath ID 0000e8ebd33ad7f8
2023-08-04T16:56:12.634Z|00053|connmgr|INFO|br-ex: added service controller "punix:/var/run/openvswitch/br-ex.mgmt"
2023-08-04T16:56:12.634Z|00054|bridge|INFO|bridge br-int: using datapath ID 0000e2027b129345
2023-08-04T16:56:12.634Z|00055|connmgr|INFO|br-int: added service controller "punix:/var/run/openvswitch/br-int.mgmt"
2023-08-04T16:56:12.634Z|00056|dpif|WARN|system@ovs-system: failed to query port patch-vbr_04a0be3427447000-local-network-to-br-int: Invalid argument
2023-08-04T16:56:12.636Z|00057|bridge|INFO|ovs-vswitchd (Open vSwitch) 2.17.7-e054917
2023-08-04T16:56:12.727Z|00058|netdev_offload_tc|INFO|added ingress qdisc to ovn-node01-0
2023-08-04T16:56:12.727Z|00059|netdev_offload|INFO|ovn-node01-0: Assigned flow API 'linux_tc'.
2023-08-04T16:56:12.727Z|00060|bridge|INFO|bridge br-int: added interface ovn-node01-0 on port 1
2023-08-04T16:56:22.639Z|00061|memory|INFO|285644 kB peak resident set size after 10.1 seconds
2023-08-04T16:56:22.639Z|00062|memory|INFO|handlers:48 idl-cells:413 ofconns:3 ports:6 revalidators:13 rules:1154 udpif keys:3
2023-08-04T16:56:22.755Z|00063|connmgr|INFO|br-int<->unix#1: 1148 flow_mods 10 s ago (1146 adds, 1 deletes, 1 modifications)
result:Using dperf to send udp packets indicates that there is a hardware offload problem on the gateway.There's nothing unusual about the vswitch log(The log level is info),I adjusted the log level to dbg,still no exception logs were found.
According to the flow table, the two flows are not offloaded.But I don't know why.
ufid:56740c84-edb0-4037-91de-7216907857ec, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x85),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:17616825, bytes:1620747900, used:0.000s, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0x89)
ufid:9ee0daa2-d3eb-4dfe-9231-15bbbac87ded, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0x89),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=17,tos=0/0,ttl=0/0,frag=no),udp(src=0/0,dst=0/0), packets:17616828, bytes:1620748176, used:0.000s, dp:tc, actions:ct_clear,ens1f0
3.Testing tcp packets with telnet found that some streams were not offloaded.
#telnet test
[root@node01 ~]# ip netns exec sw1-vm1 telnet 172.31.6.1 22
Trying 172.31.6.1...
Connected to 172.31.6.1.
Escape character is '^]'.
SSH-2.0-Comware-7.1.070
#View the flow of hardware offload in ovs:
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows type=offloaded
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:1, bytes:52, used:0.830s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:1, bytes:52, used:0.820s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:1, bytes:52, used:0.820s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:3201892454, bytes:3193734234632, used:1.270s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:0.820s, actions:ct(zone=16,nat),recirc(0xd2)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0xd2),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:0.820s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0xd3),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:1, bytes:73, used:0.820s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:153, bytes:23326, used:0.820s, actions:2
#View details
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows -m
ufid:679aa52d-f823-4bea-9a72-751ee9625071, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),skb_mark(0/0),ct_state(0/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:2, bytes:118, used:2.780s, offloaded:yes, dp:tc, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
ufid:52c65ea7-05f1-4f4c-bc3e-c2586fb06ead, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:0, bytes:0, used:never, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
ufid:25b73b7b-054f-4287-9592-b89bd8275e1c, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:1, bytes:52, used:3.750s, offloaded:yes, dp:tc, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
ufid:2d494a14-795b-469e-9854-df1d902c5b58, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:0, bytes:0, used:never, dp:tc, actions:ct_clear,ens1f0
ufid:1b87eaad-5b01-4b96-b46f-ad4bcc0507d0, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x20/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:52, used:3.750s, offloaded:yes, dp:tc, actions:ct_clear,ens1f0
ufid:664eb157-1281-450f-8aa0-9ab469591e34, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:3201892457, bytes:3193734235024, used:2.780s, offloaded:yes, dp:tc, actions:br-ex
ufid:25d2cb6b-d5be-4342-8a99-187d4e191122, skb_priority(0/0),skb_mark(0/0),ct_state(0/0x21),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:ct(zone=16,nat),recirc(0xd2)
ufid:66c9fa76-be97-49eb-8851-09acd554a600, skb_priority(0/0),skb_mark(0/0),ct_state(0x20/0x30),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd2),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=10.199.100.0/255.255.255.0,proto=6,tos=0/0,ttl=63,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ufid:be3e928d-e9c2-499f-a73b-a5251d27e861, skb_priority(0/0),skb_mark(0/0),ct_state(0x28/0x39),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd3),dp_hash(0/0),in_port(ens1f0),packet_type(ns=0/0,id=0/0),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no),tcp(src=0/0,dst=0/0), packets:1, bytes:73, used:3.750s, offloaded:yes, dp:tc, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),genev_sys_6081
ufid:17c5fac7-b52f-4d48-ad30-f10a9fe6f028, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:153, bytes:23326, used:3.750s, offloaded:yes, dp:tc, actions:ens1f0
ufid:4232db89-e385-416f-bdfa-adf82ea5a5dc, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(br-ex),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, offloaded:yes, dp:tc, actions:drop
ufid:34a47ff4-1e36-41cd-953e-909b51e5bf79, recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(ens1f0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:200, bytes:23800, used:0.276s, dp:ovs, actions:drop
#View the flow of fastpath：
[root@gateway01 ~]# ovs-appctl dpctl/dump-flows
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002/0x7fffffff}),flags(+key)),ct_state(-new-inv-trk),ct_mark(0/0x2),recirc_id(0),in_port(4),eth(src=ae:03:03:17:eb:df,dst=ae:cd:cd:04:d7:d4),eth_type(0x0800),ipv4(src=10.199.100.8/255.255.255.248,dst=172.31.6.0/255.255.254.0,proto=6,ttl=63,frag=no), packets:2, bytes:118, used:4.430s, actions:set(eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd0)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:0, bytes:0, used:never, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-rpl+trk),recirc_id(0xd0),in_port(4),eth(),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,frag=no), packets:1, bytes:52, used:5.390s, actions:ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(+new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:0, bytes:0, used:never, actions:ct_clear,2
tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,tp_dst=6081,geneve({len=4}),flags(+key)),ct_state(-new-inv+trk),ct_mark(0/0x2),recirc_id(0xd1),in_port(4),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(proto=6,frag=no), packets:1, bytes:52, used:5.390s, actions:ct_clear,2
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=e8:eb:d3:3a:d7:f8),eth_type(0x0800),ipv4(frag=no), packets:3201892459, bytes:3193734235205, used:1.370s, actions:1
ct_state(-new-trk),ct_mark(0/0x2),recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=123.123.123.123,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:5.390s, actions:ct(zone=16,nat),recirc(0xd2)
ct_state(-inv+trk),ct_mark(0/0x2),recirc_id(0xd2),in_port(2),eth(src=88:2a:5e:a9:30:ab,dst=ae:a9:a9:0c:6f:c8),eth_type(0x0800),ipv4(dst=10.199.100.0/255.255.255.0,proto=6,ttl=63,frag=no), packets:1, bytes:73, used:5.390s, actions:set(eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df)),set(ipv4(ttl=62)),ct(zone=6,nat),recirc(0xd3)
ct_state(-new+rpl-inv+trk),ct_mark(0/0x2),recirc_id(0xd3),in_port(2),eth(src=ae:cd:cd:04:d7:d4,dst=ae:03:03:17:eb:df),eth_type(0x0800),ipv4(src=172.31.6.0/255.255.254.0,dst=10.199.100.10,proto=6,tos=0/0x3,ttl=62,frag=no), packets:1, bytes:73, used:5.390s, actions:ct_clear,set(tunnel(tun_id=0x9,dst=172.31.1.21,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(csum|key))),set(eth(src=ae:95:95:65:27:a6,dst=00:00:19:91:00:10)),set(ipv4(ttl=61)),4
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(frag=no), packets:154, bytes:23430, used:1.560s, actions:2
recirc_id(0),in_port(1),eth(src=e8:eb:d3:3a:d7:f8,dst=01:80:c2:00:00:0e),eth_type(0x88cc), packets:0, bytes:0, used:never, actions:drop
recirc_id(0),in_port(2),eth(src=88:2a:5e:a9:30:ea,dst=01:80:c2:00:00:00),eth_type(0/0xffff), packets:200, bytes:23800, used:1.920s, actions:drop
#View the conntrack：
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=47190,dport=22),reply=(src=172.31.6.24,dst=10.199.100.10,sport=22,dport=47190),zone=6
tcp,orig=(src=10.199.100.10,dst=172.31.6.24,sport=47190,dport=22),reply=(src=172.31.6.24,dst=123.123.123.123,sport=22,dport=47190),zone=16
[root@gateway01 ~]# ovs-appctl dpctl/dump-conntrack | wc -l
2
result:Testing with telnet,I think hardware offload is normal.But there are two flow that are never used and not offloaded.
According to the flow table, the two flow are not offloaded.But these two flow do not seem to be used(used:never),I don't know if this access is a normal offload,Just from the flow table of type=offload the offload is normal.
ufid:52c65ea7-05f1-4f4c-bc3e-c2586fb06ead, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x29),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0xd0),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=00:00:00:00:00:00/00:00:00:00:00:00,dst=00:00:00:00:00:00/00:00:00:00:00:00),eth_type(0x0800),ipv4(src=10.199.100.0/255.255.255.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no), packets:0, bytes:0, used:never, dp:tc, actions:ct(commit,zone=6,nat(src)),ct(commit,zone=16,nat(src=123.123.123.123)),recirc(0xd1)
ufid:2d494a14-795b-469e-9854-df1d902c5b58, skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.1.21,dst=172.31.5.23,ttl=0/0,tp_dst=6081,geneve({class=0x102/0,type=0x80/0,len=4,0x10002/0}),flags(+key)),skb_mark(0/0),ct_state(0x21/0x31),ct_zone(0/0),ct_mark(0/0x2),ct_label(0/0),recirc_id(0xd1),dp_hash(0/0),in_port(genev_sys_6081),packet_type(ns=0/0,id=0/0),eth(src=ae:a9:a9:0c:6f:c8,dst=88:2a:5e:a9:30:ab),eth_type(0x0800),ipv4(src=0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=6,tos=0/0,ttl=0/0,frag=no),tcp(src=0/0,dst=0/0), packets:0, bytes:0, used:never, dp:tc, actions:ct_clear,ens1f0
I did not test the case of dperf sending tcp packets, I will add that after the subsequent tests. The question now is why are there some flows that cannot be offload in the second and third cases.What do I need to do to offload these flows.
I really hope to get your help. If you need any information, I will supplement it.The attachment is the result of the ovs-ofctl dump-flows br-int command
wanghuiheze commented
@marceloleitner Do you have any good suggestions or opinions,Thank you!
dceara commented
@wanghuiheze Sorry for the delayed reply but all flows that are not offloaded have ct_state(0x21/0x29). That translates to ct_state=+trk+new:
#define OVS_CS_F_NEW 0x01 /* Beginning of a new connection. */
#define OVS_CS_F_ESTABLISHED 0x02 /* Part of an existing connection. */
#define OVS_CS_F_RELATED 0x04 /* Related to an established
* connection. */
I think it's expected that they're not offloaded.
marceloleitner commented
Yup, sorry the delay as well.
I can confirm Dumitru's expectations: +trk+new are NOT offloadable.
dceara commented
Thanks for the confirmation @marceloleitner! Closing this for now.