stangri/source.openwrt.melmac.net

[pbr] issue: timeout waiting for wan gateway on USB-based WAN

p00rt opened this issue · 4 comments

p00rt commented

Describe the bug
To connect my router to the internet I either connect a smartphone to it and run USB tethering or connect to another network via WiFi. All connections are routed trough wireguard by default, but I want one device to be excluded from that. When I'm connected via wwan pbr works as expected (doesn't route that single computer trough the VPN), but when I switch over to USB and restart pbr, it doesn't come back up - instead it prints pbr 1.1.1-7 waiting for wan gateway... a bunch of times until it timeouts.

Your configs

  1. /etc/config/dhcp
config dnsmasq
	option domainneeded '1'
	option boguspriv '1'
	option filterwin2k '0'
	option localise_queries '1'
	option rebind_protection '1'
	option rebind_localhost '1'
	option local '/lan/'
	option domain 'lan'
	option expandhosts '1'
	option nonegcache '0'
	option authoritative '1'
	option readethers '1'
	option leasefile '/tmp/dhcp.leases'
	option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
	option nonwildcard '1'
	option localservice '1'
	option ednspacket_max '1232'

config dhcp 'lan'
	option interface 'lan'
	option start '100'
	option limit '150'
	option leasetime '12h'
	option dhcpv4 'server'
	option dhcpv6 'server'
	option ra 'server'
	option ra_slaac '1'
	list ra_flags 'managed-config'
	list ra_flags 'other-config'

config odhcpd 'odhcpd'
	option maindhcp '0'
	option leasefile '/tmp/hosts/odhcpd'
	option leasetrigger '/usr/sbin/odhcpd-update'
	option loglevel '4'
  1. /etc/config/network
config interface 'loopback'
	option device 'lo'
	option proto 'static'
	option ipaddr '127.0.0.1'
	option netmask '255.0.0.0'

config globals 'globals'
	option ula_prefix 'fd7f:6caf:b381::/48'

config device
	option name 'br-lan'
	option type 'bridge'
	list ports 'eth0.1'

config interface 'lan'
	option device 'br-lan'
	option proto 'static'
	option ipaddr '192.168.1.1'
	option netmask '255.255.255.0'
	option ip6assign '60'

config switch
	option name 'switch0'
	option reset '1'
	option enable_vlan '1'

config switch_vlan
	option device 'switch0'
	option vlan '1'
	option ports '1 2 3 4 0t'

config interface 'wwan'
	option proto 'dhcp'

config interface 'usb'
	option proto 'dhcp'
	option device 'usb0'

config interface 'ro_buh_wg_002'
	option proto 'wireguard'
	option private_key 'redacted'
	list addresses 'redacted'
	list addresses 'redacted'
	option auto '0'

config wireguard_ro_buh_wg_002
	option public_key 'Ekc3+qU88FuMfkEMyLlgRqDYv+WHJvUsfOMI/C0ydE4='
	list allowed_ips '0.0.0.0/0'
	option route_allowed_ips '1'
	option endpoint_host '146.70.124.194'
	option endpoint_port '51820'

config interface 'bg_sof_wg_001'
	option proto 'wireguard'
	option private_key 'redacted'
	list addresses 'redacted'
	list addresses 'redacted'
	option force_link '1'
	option auto '0'

config wireguard_bg_sof_wg_001
	option public_key 'J8KysHmHZWqtrVKKOppneDXSks/PDsB1XTlRHpwiABA='
	list allowed_ips '0.0.0.0/0'
	option route_allowed_ips '1'
	option endpoint_host '146.70.188.130'
	option endpoint_port '51820'

config interface 'hu_bud_wg_101'
	option proto 'wireguard'
	option private_key 'redacted'
	option force_link '1'
	list addresses 'redacted'
	list addresses 'redacted'

config wireguard_hu_bud_wg_101
	option public_key 'u+h0GmQJ8UBaMTi2BP9Ls6UUszcGC51y6vTmNr/y+AU='
	list allowed_ips '0.0.0.0/0'
	list allowed_ips '::0/0'
	option route_allowed_ips '1'
	option endpoint_host '146.70.196.194'
	option endpoint_port '51820'
  1. /etc/config/firewall
config defaults
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option synflood_protect '1'

config zone
	option name 'lan'
	option input 'ACCEPT'
	option output 'ACCEPT'
	option forward 'ACCEPT'
	list network 'lan'

config zone
	option name 'wan'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'usb'
	list network 'wwan'

config rule
	option name 'Allow-DHCP-Renew'
	option src 'wan'
	option proto 'udp'
	option dest_port '68'
	option target 'ACCEPT'
	option family 'ipv4'

config rule
	option name 'Allow-Ping'
	option src 'wan'
	option proto 'icmp'
	option icmp_type 'echo-request'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-IGMP'
	option src 'wan'
	option proto 'igmp'
	option family 'ipv4'
	option target 'ACCEPT'

config rule
	option name 'Allow-DHCPv6'
	option src 'wan'
	option proto 'udp'
	option dest_port '546'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-MLD'
	option src 'wan'
	option proto 'icmp'
	option src_ip 'fe80::/10'
	list icmp_type '130/0'
	list icmp_type '131/0'
	list icmp_type '132/0'
	list icmp_type '143/0'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Input'
	option src 'wan'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	list icmp_type 'router-solicitation'
	list icmp_type 'neighbour-solicitation'
	list icmp_type 'router-advertisement'
	list icmp_type 'neighbour-advertisement'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-ICMPv6-Forward'
	option src 'wan'
	option dest '*'
	option proto 'icmp'
	list icmp_type 'echo-request'
	list icmp_type 'echo-reply'
	list icmp_type 'destination-unreachable'
	list icmp_type 'packet-too-big'
	list icmp_type 'time-exceeded'
	list icmp_type 'bad-header'
	list icmp_type 'unknown-header-type'
	option limit '1000/sec'
	option family 'ipv6'
	option target 'ACCEPT'

config rule
	option name 'Allow-IPSec-ESP'
	option src 'wan'
	option dest 'lan'
	option proto 'esp'
	option target 'ACCEPT'

config rule
	option name 'Allow-ISAKMP'
	option src 'wan'
	option dest 'lan'
	option dest_port '500'
	option proto 'udp'
	option target 'ACCEPT'

config zone
	option name 'WGZONE'
	option input 'REJECT'
	option output 'ACCEPT'
	option forward 'REJECT'
	option masq '1'
	option mtu_fix '1'
	list network 'bg_sof_wg_001'
	list network 'hu_bud_wg_101'
	list network 'ro_buh_wg_002'

config forwarding
	option src 'lan'
	option dest 'WGZONE'

config rule
	option src 'lan'
	list src_mac 'A4:BB:6D:CF:AC:29'
	option dest 'wan'
	option target 'ACCEPT'

config include 'pbr'
	option fw4_compatible '1'
	option type 'script'
	option path '/usr/share/pbr/pbr.firewall.include'

config forwarding
	option src 'lan'
	option dest 'wan'
  1. /etc/config/pbr
config pbr 'config'
	option verbosity '2'
	option resolver_set 'none'
	option ipv6_enabled '0'
	list ignored_interface 'vpnserver'
	list ignored_interface 'wgserver'
	option boot_timeout '30'
	option rule_create_option 'add'
	option procd_reload_delay '1'
	option webui_show_ignore_target '0'
	list webui_supported_protocol 'all'
	list webui_supported_protocol 'tcp'
	list webui_supported_protocol 'udp'
	list webui_supported_protocol 'tcp udp'
	list webui_supported_protocol 'icmp'
	option enabled '1'
	option strict_enforcement '0'

#config policy
#	option interface 'wwan'
#	option name 'pracaWAN'
#	option src_addr 'A4:BB:6D:CF:AC:29'
#	option enabled '0'

config policy
	option interface 'usb'
	option name 'pracaUSB'
	option src_addr 'A4:BB:6D:CF:AC:29'
	option enabled '0'

To Reproduce

  1. Switch to USB WAN
  2. reload or restart pbr
  3. See error

Expected behavior
The service should start normally

Policy Routing run-time information

  1. Output of /etc/init.d/pbr reload with verbosity set to 2:
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
pbr 1.1.1-7 waiting for wan gateway...
ERROR: The pbr 1.1.1-7 service failed to discover WAN gateway!
  1. Output of /etc/init.d/pbr status:
============================================================
pbr - environment
pbr 1.1.1-7 running on OpenWrt 22.03.5. WAN (IPv4): wwan//0.0.0.0.
============================================================
Dnsmasq version 2.86  Copyright (c) 2000-2021 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP no-DHCPv6 no-Lua TFTP no-conntrack no-ipset no-auth no-cryptohash no-DNSSEC no-ID loop-detect inotify dumpfile
============================================================
pbr chains - policies
	chain pbr_forward { # handle 36
	}
	chain pbr_input { # handle 37
	}
	chain pbr_output { # handle 38
	}
	chain pbr_prerouting { # handle 39
	}
	chain pbr_postrouting { # handle 40
	}
============================================================
pbr chains - marking
============================================================
pbr nft sets
============================================================
IPv4 table 257 route: 
IPv4 table 257 rule(s):
30001:	from all fwmark 0x20000/0xff0000 lookup pbr_usb
stangri commented

When internet connection via cellphone/usb is active, could you please run the commands below and provide me with the output:

. /lib/functions/network.sh 
network_find_wan wan
echo "$wan"
ls -la "/sys/devices/virtual/net/usb0/tun_flags"
p00rt commented

Thanks for your incredibly quick reply! Here's the output

root@OpenWrt:~# . /lib/functions/network.sh 
root@OpenWrt:~# network_find_wan wan
root@OpenWrt:~# echo "$wan"
hu_bud_wg_101
root@OpenWrt:~# ls -la "/sys/devices/virtual/net/usb0/tun_flags"
ls: /sys/devices/virtual/net/usb0/tun_flags: No such file or directory

there is in fact no /sys/devices/virtual/net/usb0 directory at all.

perhaps the output from ubus call network.interface dump might be useful as well:

===== SECTION SKIPPED =======
{
	"interface": "hu_bud_wg_101",
	"up": true,
	"pending": false,
	"available": true,
	"autostart": true,
	"dynamic": false,
	"uptime": 176615,
	"l3_device": "hu_bud_wg_101",
	"proto": "wireguard",
	"updated": [
		"addresses",
		"routes"
	],
	"metric": 0,
	"dns_metric": 0,
	"delegation": true,
	"ipv4-address": [
		{
			"address": "redacted",
			"mask": 32
		}
	],
	"ipv6-address": [
		{
			"address": "redacted",
			"mask": 128
		}
	],
	"ipv6-prefix": [
		
	],
	"ipv6-prefix-assignment": [
		
	],
	"route": [
		{
			"target": "::",
			"mask": 0,
			"nexthop": "::",
			"source": "::/0"
		},
		{
			"target": "0.0.0.0",
			"mask": 0,
			"nexthop": "0.0.0.0",
			"source": "0.0.0.0/0"
		}
	],
	"dns-server": [
		
	],
	"dns-search": [
		
	],
	"neighbors": [
		
	],
	"inactive": {
		"ipv4-address": [
			
		],
		"ipv6-address": [
			
		],
		"route": [
			
		],
		"dns-server": [
			
		],
		"dns-search": [
			
		],
		"neighbors": [
			
		]
	},
	"data": {
		
	}
},
===== SECTION SKIPPED =======
{
	"interface": "usb",
	"up": true,
	"pending": false,
	"available": true,
	"autostart": true,
	"dynamic": false,
	"uptime": 176616,
	"l3_device": "usb0",
	"proto": "dhcp",
	"device": "usb0",
	"metric": 0,
	"dns_metric": 0,
	"delegation": true,
	"ipv4-address": [
		{
			"address": "192.168.42.107",
			"mask": 24
		}
	],
	"ipv6-address": [
		
	],
	"ipv6-prefix": [
		
	],
	"ipv6-prefix-assignment": [
		
	],
	"route": [
		{
			"target": "0.0.0.0",
			"mask": 0,
			"nexthop": "192.168.42.129",
			"source": "192.168.42.107/32"
		}
	],
	"dns-server": [
		"192.168.42.129"
	],
	"dns-search": [
		
	],
	"neighbors": [
		
	],
	"inactive": {
		"ipv4-address": [
			
		],
		"ipv6-address": [
			
		],
		"route": [
			
		],
		"dns-server": [
			
		],
		"dns-search": [
			
		],
		"neighbors": [
			
		]
	},
	"data": {
		"dhcpserver": "192.168.42.129",
		"hostname": "OpenWrt",
		"leasetime": 3599
	}
},
===== SECTION SKIPPED =======
stangri commented

In the newer versions available from my repo, you can set the procd_wan_interface option to usb and it will use the usb as the wan interface. You'd have to script setting this option when you switch to phone as your gate to the internet and remove it when you switch back to wwan. You can also try renaming usb to wanusb and pbr may discover it as a wan interface then.

Outside of that, there's no way really for pbr to discover that usb is your wan interface.

stangri commented

Closing due to lack of replies.