[client] Add experimental support for userspace routing

client/Dockerfile-rootless

@@ -9,6 +9,7 @@ USER netbird:netbird
 
 ENV NB_FOREGROUND_MODE=true
 ENV NB_USE_NETSTACK_MODE=true
+ENV NB_ENABLE_NETSTACK_LOCAL_FORWARDING=true
 ENV NB_CONFIG=config.json
 ENV NB_DAEMON_ADDR=unix://netbird.sock
 ENV NB_DISABLE_DNS=true

client/cmd/trace.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/client/proto"
+)
+
+var traceCmd = &cobra.Command{
+	Use:   "trace <direction> <source-ip> <dest-ip>",
+	Short: "Trace a packet through the firewall",
+	Example: `
+  netbird debug trace in 192.168.1.10 10.10.0.2 -p tcp --sport 12345 --dport 443 --syn --ack
+  netbird debug trace out 10.10.0.1 8.8.8.8 -p udp  --dport 53
+  netbird debug trace in 10.10.0.2 10.10.0.1 -p icmp --type 8 --code 0
+  netbird debug trace in 100.64.1.1 self -p tcp --dport 80`,
+	Args: cobra.ExactArgs(3),
+	RunE: tracePacket,
+}
+
+func init() {
+	debugCmd.AddCommand(traceCmd)
+
+	traceCmd.Flags().StringP("protocol", "p", "tcp", "Protocol (tcp/udp/icmp)")
+	traceCmd.Flags().Uint16("sport", 0, "Source port")
+	traceCmd.Flags().Uint16("dport", 0, "Destination port")
+	traceCmd.Flags().Uint8("icmp-type", 0, "ICMP type")
+	traceCmd.Flags().Uint8("icmp-code", 0, "ICMP code")
+	traceCmd.Flags().Bool("syn", false, "TCP SYN flag")
+	traceCmd.Flags().Bool("ack", false, "TCP ACK flag")
+	traceCmd.Flags().Bool("fin", false, "TCP FIN flag")
+	traceCmd.Flags().Bool("rst", false, "TCP RST flag")
+	traceCmd.Flags().Bool("psh", false, "TCP PSH flag")
+	traceCmd.Flags().Bool("urg", false, "TCP URG flag")
+}
+
+func tracePacket(cmd *cobra.Command, args []string) error {
+	direction := strings.ToLower(args[0])
+	if direction != "in" && direction != "out" {
+		return fmt.Errorf("invalid direction: use 'in' or 'out'")
+	}
+
+	protocol := cmd.Flag("protocol").Value.String()
+	if protocol != "tcp" && protocol != "udp" && protocol != "icmp" {
+		return fmt.Errorf("invalid protocol: use tcp/udp/icmp")
+	}
+
+	sport, err := cmd.Flags().GetUint16("sport")
+	if err != nil {
+		return fmt.Errorf("invalid source port: %v", err)
+	}
+	dport, err := cmd.Flags().GetUint16("dport")
+	if err != nil {
+		return fmt.Errorf("invalid destination port: %v", err)
+	}
+
+	// For TCP/UDP, generate random ephemeral port (49152-65535) if not specified
+	if protocol != "icmp" {
+		if sport == 0 {
+			sport = uint16(rand.Intn(16383) + 49152)
+		}
+		if dport == 0 {
+			dport = uint16(rand.Intn(16383) + 49152)
+		}
+	}
+
+	var tcpFlags *proto.TCPFlags
+	if protocol == "tcp" {
+		syn, _ := cmd.Flags().GetBool("syn")
+		ack, _ := cmd.Flags().GetBool("ack")
+		fin, _ := cmd.Flags().GetBool("fin")
+		rst, _ := cmd.Flags().GetBool("rst")
+		psh, _ := cmd.Flags().GetBool("psh")
+		urg, _ := cmd.Flags().GetBool("urg")
+
+		tcpFlags = &proto.TCPFlags{
+			Syn: syn,
+			Ack: ack,
+			Fin: fin,
+			Rst: rst,
+			Psh: psh,
+			Urg: urg,
+		}
+	}
+
+	icmpType, _ := cmd.Flags().GetUint32("icmp-type")
+	icmpCode, _ := cmd.Flags().GetUint32("icmp-code")
+
+	conn, err := getClient(cmd)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := proto.NewDaemonServiceClient(conn)
+	resp, err := client.TracePacket(cmd.Context(), &proto.TracePacketRequest{
+		SourceIp:        args[1],
+		DestinationIp:   args[2],
+		Protocol:        protocol,
+		SourcePort:      uint32(sport),
+		DestinationPort: uint32(dport),
+		Direction:       direction,
+		TcpFlags:        tcpFlags,
+		IcmpType:        &icmpType,
+		IcmpCode:        &icmpCode,
+	})
+	if err != nil {
+		return fmt.Errorf("trace failed: %v", status.Convert(err).Message())
+	}
+
+	printTrace(cmd, args[1], args[2], protocol, sport, dport, resp)
+	return nil
+}
+
+func printTrace(cmd *cobra.Command, src, dst, proto string, sport, dport uint16, resp *proto.TracePacketResponse) {
+	cmd.Printf("Packet trace %s:%d -> %s:%d (%s)\n\n", src, sport, dst, dport, strings.ToUpper(proto))
+
+	for _, stage := range resp.Stages {
+		if stage.ForwardingDetails != nil {
+			cmd.Printf("%s: %s [%s]\n", stage.Name, stage.Message, *stage.ForwardingDetails)
+		} else {
+			cmd.Printf("%s: %s\n", stage.Name, stage.Message)
+		}
+	}
+
+	disposition := map[bool]string{
+		true:  "\033[32mALLOWED\033[0m", // Green
+		false: "\033[31mDENIED\033[0m",  // Red
+	}[resp.FinalDisposition]
+
+	cmd.Printf("\nFinal disposition: %s\n", disposition)
+}

client/firewall/create.go

@@ -14,13 +14,13 @@ import (
 )
 
 // NewFirewall creates a firewall manager instance
-func NewFirewall(iface IFaceMapper, _ *statemanager.Manager) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, _ *statemanager.Manager, disableServerRoutes bool) (firewall.Manager, error) {
 	if !iface.IsUserspaceBind() {
 		return nil, fmt.Errorf("not implemented for this OS: %s", runtime.GOOS)
 	}
 
 	// use userspace packet filtering firewall
-	fm, err := uspfilter.Create(iface)
+	fm, err := uspfilter.Create(iface, disableServerRoutes)
 	if err != nil {
 		return nil, err
 	}

client/firewall/create_linux.go

@@ -33,12 +33,12 @@ const SKIP_NFTABLES_ENV = "NB_SKIP_NFTABLES_CHECK"
 // FWType is the type for the firewall type
 type FWType int
 
-func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager, disableServerRoutes bool) (firewall.Manager, error) {
 	// on the linux system we try to user nftables or iptables
 	// in any case, because we need to allow netbird interface traffic
 	// so we use AllowNetbird traffic from these firewall managers
 	// for the userspace packet filtering firewall
-	fm, err := createNativeFirewall(iface, stateManager)
+	fm, err := createNativeFirewall(iface, stateManager, disableServerRoutes)
 
 	if !iface.IsUserspaceBind() {
 		return fm, err
@@ -47,10 +47,10 @@ func NewFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewal
 	if err != nil {
 		log.Warnf("failed to create native firewall: %v. Proceeding with userspace", err)
 	}
-	return createUserspaceFirewall(iface, fm)
+	return createUserspaceFirewall(iface, fm, disableServerRoutes)
 }
 
-func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager) (firewall.Manager, error) {
+func createNativeFirewall(iface IFaceMapper, stateManager *statemanager.Manager, routes bool) (firewall.Manager, error) {
 	fm, err := createFW(iface)
 	if err != nil {
 		return nil, fmt.Errorf("create firewall: %s", err)
@@ -77,12 +77,12 @@ func createFW(iface IFaceMapper) (firewall.Manager, error) {
 	}
 }
 
-func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager) (firewall.Manager, error) {
+func createUserspaceFirewall(iface IFaceMapper, fm firewall.Manager, disableServerRoutes bool) (firewall.Manager, error) {
 	var errUsp error
 	if fm != nil {
-		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm)
+		fm, errUsp = uspfilter.CreateWithNativeFirewall(iface, fm, disableServerRoutes)
 	} else {
-		fm, errUsp = uspfilter.Create(iface)
+		fm, errUsp = uspfilter.Create(iface, disableServerRoutes)
 	}
 
 	if errUsp != nil {

client/firewall/iface.go

@@ -1,6 +1,8 @@
 package firewall
 
 import (
+	wgdevice "golang.zx2c4.com/wireguard/device"
+
 	"github.com/netbirdio/netbird/client/iface/device"
 )
 
@@ -10,4 +12,6 @@ type IFaceMapper interface {
 	Address() device.WGAddress
 	IsUserspaceBind() bool
 	SetFilter(device.PacketFilter) error
+	GetDevice() *device.FilteredDevice
+	GetWGDevice() *wgdevice.Device
 }

client/firewall/iptables/manager_linux.go

@@ -213,6 +213,11 @@ func (m *Manager) AllowNetbird() error {
 // Flush doesn't need to be implemented for this manager
 func (m *Manager) Flush() error { return nil }
 
+// SetLogLevel sets the log level for the firewall manager
+func (m *Manager) SetLogLevel(log.Level) {
+	// not supported
+}
+
 func getConntrackEstablished() []string {
 	return []string{"-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}
 }

client/firewall/iptables/router_linux.go

@@ -135,7 +135,16 @@ func (r *router) AddRouteFiltering(
 	}
 
 	rule := genRouteFilteringRuleSpec(params)
-	if err := r.iptablesClient.Append(tableFilter, chainRTFWD, rule...); err != nil {
+	// Insert DROP rules at the beginning, append ACCEPT rules at the end
+	var err error
+	if action == firewall.ActionDrop {
+		// after the established rule
+		err = r.iptablesClient.Insert(tableFilter, chainRTFWD, 2, rule...)
+	} else {
+		err = r.iptablesClient.Append(tableFilter, chainRTFWD, rule...)
+	}
+
+	if err != nil {
 		return nil, fmt.Errorf("add route rule: %v", err)
 	}
 

client/firewall/manager/firewall.go

@@ -99,6 +99,8 @@ type Manager interface {
 
 	// Flush the changes to firewall controller
 	Flush() error
+
+	SetLogLevel(log.Level)
 }
 
 func GenKey(format string, pair RouterPair) string {

client/firewall/nftables/manager_linux.go

@@ -318,6 +318,11 @@ func (m *Manager) cleanupNetbirdTables() error {
 	return nil
 }
 
+// SetLogLevel sets the log level for the firewall manager
+func (m *Manager) SetLogLevel(log.Level) {
+	// not supported
+}
+
 // Flush rule/chain/set operations from the buffer
 //
 // Method also get all rules after flush and refreshes handle values in the rulesets

client/firewall/nftables/manager_linux_test.go

@@ -107,7 +107,7 @@ func TestNftablesManager(t *testing.T) {
 			Kind: expr.VerdictAccept,
 		},
 	}
-	require.ElementsMatch(t, rules[0].Exprs, expectedExprs1, "expected the same expressions")
+	compareExprsIgnoringCounters(t, rules[0].Exprs, expectedExprs1)
 
 	ipToAdd, _ := netip.AddrFromSlice(ip)
 	add := ipToAdd.Unmap()
@@ -307,3 +307,18 @@ func TestNftablesManagerCompatibilityWithIptables(t *testing.T) {
 	stdout, stderr = runIptablesSave(t)
 	verifyIptablesOutput(t, stdout, stderr)
 }
+
+func compareExprsIgnoringCounters(t *testing.T, got, want []expr.Any) {
+	t.Helper()
+	require.Equal(t, len(got), len(want), "expression count mismatch")
+
+	for i := range got {
+		if _, isCounter := got[i].(*expr.Counter); isCounter {
+			_, wantIsCounter := want[i].(*expr.Counter)
+			require.True(t, wantIsCounter, "expected Counter at index %d", i)
+			continue
+		}
+
+		require.Equal(t, got[i], want[i], "expression mismatch at index %d", i)
+	}
+}

client/firewall/nftables/router_linux.go

@@ -233,7 +233,13 @@ func (r *router) AddRouteFiltering(
 		UserData: []byte(ruleKey),
 	}
 
-	rule = r.conn.AddRule(rule)
+	// Insert DROP rules at the beginning, append ACCEPT rules at the end
+	if action == firewall.ActionDrop {
+		// TODO: Insert after the established rule
+		rule = r.conn.InsertRule(rule)
+	} else {
+		rule = r.conn.AddRule(rule)
+	}
 
 	log.Tracef("Adding route rule %s", spew.Sdump(rule))
 	if err := r.conn.Flush(); err != nil {

client/firewall/uspfilter/allow_netbird.go

@@ -3,6 +3,11 @@
 package uspfilter
 
 import (
+	"context"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
 	"github.com/netbirdio/netbird/client/firewall/uspfilter/conntrack"
 	"github.com/netbirdio/netbird/client/internal/statemanager"
 )
@@ -17,17 +22,29 @@ func (m *Manager) Reset(stateManager *statemanager.Manager) error {
 
 	if m.udpTracker != nil {
 		m.udpTracker.Close()
-		m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout)
+		m.udpTracker = conntrack.NewUDPTracker(conntrack.DefaultUDPTimeout, m.logger)
 	}
 
 	if m.icmpTracker != nil {
 		m.icmpTracker.Close()
-		m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout)
+		m.icmpTracker = conntrack.NewICMPTracker(conntrack.DefaultICMPTimeout, m.logger)
 	}
 
 	if m.tcpTracker != nil {
 		m.tcpTracker.Close()
-		m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout)
+		m.tcpTracker = conntrack.NewTCPTracker(conntrack.DefaultTCPTimeout, m.logger)
+	}
+
+	if m.forwarder != nil {
+		m.forwarder.Stop()
+	}
+
+	if m.logger != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		if err := m.logger.Stop(ctx); err != nil {
+			log.Errorf("failed to shutdown logger: %v", err)
+		}
 	}
 
 	if m.nativeFirewall != nil {