refactor(state): per-instance parameter tuning

cmd/run.go

@@ -8,6 +8,8 @@ import (
 
 import _ "net/http/pprof" // remove in stable version of nylon
 
+var opts state.NylonOptions
+
 // runCmd represents the run command
 var runCmd = &cobra.Command{
 	Use:   "run",
@@ -23,7 +25,7 @@ var runCmd = &cobra.Command{
 			isVerbose = true
 		}
 
-		core.Bootstrap(centralPath, nodePath, logPath, isVerbose)
+		core.Bootstrap(centralPath, nodePath, logPath, isVerbose, opts)
 	},
 	GroupID: "ny",
 }
@@ -32,13 +34,13 @@ func init() {
 	rootCmd.AddCommand(runCmd)
 
 	runCmd.Flags().BoolP("verbose", "v", false, "Verbose output")
-	runCmd.Flags().BoolVarP(&state.DBG_log_probe, "dbg-probe", "p", false, "Write probes to console")
-	runCmd.Flags().BoolVarP(&state.DBG_log_wireguard, "dbg-wg", "w", false, "Outputs wireguard logs to the console")
-	runCmd.Flags().BoolVarP(&state.DBG_log_repo_updates, "dbg-repo", "", false, "Outputs repo updates to the console")
-	runCmd.Flags().BoolVarP(&state.DBG_debug, "dbg-perf", "", false, "Enables performance debugging server on port 6060")
-	runCmd.Flags().BoolVarP(&state.DBG_trace, "dbg-trace", "", false, "Enables trace to trace.out")
-	runCmd.Flags().BoolVarP(&state.DBG_trace_tc, "dbg-trace-tc", "", false, "Enables logging of packet routing")
-	runCmd.Flags().BoolVarP(&state.DBG_log_json, "json", "j", false, "Enables structued json logging")
+	runCmd.Flags().BoolVarP(&opts.DBG_log_probe, "dbg-probe", "p", false, "Write probes to console")
+	runCmd.Flags().BoolVarP(&opts.DBG_log_wireguard, "dbg-wg", "w", false, "Outputs wireguard logs to the console")
+	runCmd.Flags().BoolVarP(&opts.DBG_log_repo_updates, "dbg-repo", "", false, "Outputs repo updates to the console")
+	runCmd.Flags().BoolVarP(&opts.DBG_debug, "dbg-perf", "", false, "Enables performance debugging server on port 6060")
+	runCmd.Flags().BoolVarP(&opts.DBG_trace, "dbg-trace", "", false, "Enables trace to trace.out")
+	runCmd.Flags().BoolVarP(&opts.DBG_trace_tc, "dbg-trace-tc", "", false, "Enables logging of packet routing")
+	runCmd.Flags().BoolVarP(&opts.DBG_log_json, "json", "j", false, "Enables structued json logging")
 	runCmd.Flags().StringP("config", "c", DefaultConfigPath, "Path to the config file")
 	runCmd.Flags().StringP("node", "n", DefaultNodeConfigPath, "Path to the node config file")
 	runCmd.Flags().StringP("log", "l", "", "Path to the log file (overrides config)")

core/entrypoint.go

@@ -12,8 +12,8 @@ import (
 	"github.com/goccy/go-yaml"
 )
 
-func setupDebugging() {
-	if state.DBG_trace {
+func setupDebugging(opts state.NylonOptions) {
+	if opts.DBG_trace {
 		f, err := os.Create("trace.out")
 		if err != nil {
 			log.Fatal(err)
@@ -25,7 +25,7 @@ func setupDebugging() {
 		}
 		log.Println("Started tracing")
 	}
-	if state.DBG_debug {
+	if opts.DBG_debug {
 		go func() {
 			log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
 		}()
@@ -96,8 +96,8 @@ func readNodeConfig(nodePath string) (*state.LocalCfg, error) {
 }
 
 // Bootstrap provides startup logic in a real environment
-func Bootstrap(centralPath, nodePath, logPath string, verbose bool) {
-	setupDebugging()
+func Bootstrap(centralPath, nodePath, logPath string, verbose bool, opts state.NylonOptions) {
+	setupDebugging(opts)
 	level := slog.LevelInfo
 	if verbose {
 		level = slog.LevelDebug
@@ -124,7 +124,7 @@ func Bootstrap(centralPath, nodePath, logPath string, verbose bool) {
 	if err != nil {
 		panic(err)
 	}
-	n, err := NewNylon(*centralCfg, *nodeCfg, level, centralPath, nil)
+	n, err := NewNylon(*centralCfg, *nodeCfg, level, centralPath, nil, opts, nil)
 	if err != nil {
 		panic(err)
 	}

core/ipc_handler.go

@@ -129,7 +129,7 @@ func handleStatus(n *Nylon, req *protocol.StatusRequest) *protocol.IpcResponse {
 				PublicKey:       keyString(n.LocalCfg.Key.Pubkey()),
 				ListenPort:      listenPort,
 				ConfigTimestamp: n.CentralCfg.Timestamp,
-				TraceEnabled:    state.DBG_trace_tc,
+				TraceEnabled:    n.DBG_trace_tc,
 				Advertised:      buildAdvertisements(n),
 				Seqnos:          buildSeqnos(n),
 				Stats: &protocol.NodeStats{
@@ -470,7 +470,7 @@ func handleIPCReload(n *Nylon, req *protocol.ReloadRequest) *protocol.IpcRespons
 }
 
 func handleTrace(n *Nylon, rw *bufio.ReadWriter) error {
-	if !state.DBG_trace_tc {
+	if !n.DBG_trace_tc {
 		if err := writeResponse(rw, errResponse("tracing not enabled; restart with --dbg-trace-tc")); err != nil {
 			return err
 		}

core/nylon.go

@@ -29,6 +29,10 @@ import (
 type Nylon struct {
 	Trace *NylonTrace
 
+	// tunables and options
+	state.RouterTunables
+	state.NylonOptions
+
 	// state
 	state.ConfigState
 	RouterState   *state.RouterState
@@ -73,13 +77,20 @@ type AppliedSystemState struct {
 	Peers   map[state.NodeId]state.NyPublicKey
 }
 
-func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any) (*Nylon, error) {
+func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, configPath string, aux map[string]any, opts state.NylonOptions, tunables *state.RouterTunables) (*Nylon, error) {
 	ctx, cancel := context.WithCancelCause(context.Background())
 
 	dispatch := make(chan func() error, 128)
 
+	var rt state.RouterTunables
+	if tunables != nil {
+		rt = *tunables
+	} else {
+		rt = state.DefaultRouterTunables()
+	}
+
 	handlers := make([]slog.Handler, 0)
-	if state.DBG_log_json {
+	if opts.DBG_log_json {
 		handlers = append(handlers,
 			slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
 				Level: logLevel,
@@ -120,7 +131,9 @@ func NewNylon(ccfg state.CentralCfg, ncfg state.LocalCfg, logLevel slog.Level, c
 	}
 
 	n := &Nylon{
-		Trace: &NylonTrace{},
+		Trace:          &NylonTrace{},
+		RouterTunables: rt,
+		NylonOptions:   opts,
 		ConfigState: state.ConfigState{
 			CentralCfg: ccfg,
 			LocalCfg:   ncfg,
@@ -172,7 +185,7 @@ func (n *Nylon) Init() error {
 
 	n.RepeatTask(func() error {
 		return nylonGc(n)
-	}, state.GcDelay)
+	}, n.GcDelay)
 
 	// wireguard configuration
 	err = n.initWireGuard()
@@ -183,14 +196,14 @@ func (n *Nylon) Init() error {
 	// endpoint probing
 	n.RepeatTask(func() error {
 		return n.probeLinks(true)
-	}, state.ProbeDelay)
+	}, n.ProbeDelay)
 	n.RepeatTask(func() error {
 		// refresh dynamic endpoints
 		for _, neigh := range n.RouterState.Neighbours {
 			for _, ep := range neigh.Eps {
 				if nep, ok := ep.(*state.NylonEndpoint); ok {
 					go func() {
-						_, err := nep.DynEP.Refresh()
+						_, err := nep.DynEP.Refresh(n.EndpointResolveExpiry)
 						if err != nil {
 							n.Log.Debug("failed to resolve endpoint", "ep", nep.DynEP.Value, "err", err.Error())
 						}
@@ -199,13 +212,13 @@ func (n *Nylon) Init() error {
 			}
 		}
 		return nil
-	}, state.EndpointResolveDelay)
+	}, n.EndpointResolveDelay)
 	n.RepeatTask(func() error {
 		return n.probeLinks(false)
-	}, state.ProbeRecoveryDelay)
+	}, n.ProbeRecoveryDelay)
 	n.RepeatTask(func() error {
 		return n.probeNew()
-	}, state.ProbeDiscoveryDelay)
+	}, n.ProbeDiscoveryDelay)
 
 	n.startAdvertisedPrefixHealth()
 
@@ -219,7 +232,7 @@ func (n *Nylon) Init() error {
 		for _, repo := range n.CentralCfg.Dist.Repos {
 			n.Log.Info("config source", "repo", repo)
 		}
-		n.RepeatTask(func() error { return checkForConfigUpdates(n) }, state.CentralUpdateDelay)
+		n.RepeatTask(func() error { return checkForConfigUpdates(n) }, n.CentralUpdateDelay)
 	}
 	return nil
 }

core/nylon_apply.go

@@ -70,7 +70,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 			continue
 		}
 		// configure existing neighbours
-		reconcileConfiguredEndpoints(neigh, cfg.Endpoints)
+		reconcileConfiguredEndpoints(neigh, cfg.Endpoints, &n.RouterTunables)
 		neighs = append(neighs, neigh)
 		delete(desired, neigh.Id)
 	}
@@ -89,7 +89,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 			Eps:    make([]state.Endpoint, 0, len(cfg.Endpoints)),
 		}
 		for _, ep := range cfg.Endpoints {
-			stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, nil))
+			stNeigh.Eps = append(stNeigh.Eps, state.NewEndpoint(ep, false, nil, &n.RouterTunables))
 		}
 		neighs = append(neighs, stNeigh)
 	}
@@ -107,7 +107,7 @@ func (n *Nylon) reconcileRouterState(next *state.CentralCfg) error {
 	return nil
 }
 
-func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.DynamicEndpoint) {
+func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.DynamicEndpoint, t *state.RouterTunables) {
 	desiredByValue := make(map[string]*state.DynamicEndpoint, len(desired))
 	for _, ep := range desired {
 		desiredByValue[ep.Value] = ep
@@ -131,7 +131,7 @@ func reconcileConfiguredEndpoints(neigh *state.Neighbour, desired []*state.Dynam
 		if _, ok := seen[ep.Value]; ok {
 			continue
 		}
-		eps = append(eps, state.NewEndpoint(ep, false, nil))
+		eps = append(eps, state.NewEndpoint(ep, false, nil, t))
 	}
 	neigh.Eps = eps
 }
@@ -164,7 +164,7 @@ func (n *Nylon) reconcileAdvertisedPrefixes(next *state.CentralCfg) {
 	for prefix, desired := range desiredLocal {
 		if _, ok := currentLocal[prefix]; !ok {
 			n.Log.Debug("starting prefix healthcheck", "prefix", prefix)
-			desired.Start(n.Log)
+			desired.Start(n.Log, &n.RouterTunables)
 		}
 		n.RouterState.Advertised[prefix] = state.Advertisement{
 			NodeId:   n.LocalCfg.Id,
@@ -180,7 +180,7 @@ func (n *Nylon) reconcileAdvertisedPrefixes(next *state.CentralCfg) {
 func (n *Nylon) startAdvertisedPrefixHealth() {
 	for _, ph := range n.GetNode(n.LocalCfg.Id).Prefixes {
 		n.Log.Debug("starting prefix healthcheck", "prefix", ph.GetPrefix())
-		ph.Start(n.Log)
+		ph.Start(n.Log, &n.RouterTunables)
 	}
 }
 

core/nylon_distribution.go

@@ -90,7 +90,7 @@ func checkForConfigUpdates(n *Nylon) error {
 					return err
 				}
 				if config.Timestamp <= currentTimestamp {
-					if state.DBG_log_repo_updates {
+					if n.DBG_log_repo_updates {
 						n.Log.Debug(fmt.Sprintf("found old update bundle at %s, skipping", repo))
 					}
 					return nil

core/nylon_endpoints.go

@@ -103,7 +103,7 @@ func handleProbePing(n *Nylon, node state.NodeId, wgEndpoint conn.Endpoint) {
 				}
 				dep.Renew()
 
-				if state.DBG_log_probe {
+				if n.DBG_log_probe {
 					n.Log.Debug("probe from", "addr", ap.String())
 				}
 				return
@@ -113,7 +113,7 @@ func handleProbePing(n *Nylon, node state.NodeId, wgEndpoint conn.Endpoint) {
 	// create a new link if we dont have a link
 	for _, neigh := range n.RouterState.Neighbours {
 		if neigh.Id == node {
-			newEp := state.NewEndpoint(state.NewDynamicEndpoint(wgEndpoint.DstIPPort().String()), true, wgEndpoint)
+			newEp := state.NewEndpoint(state.NewDynamicEndpoint(wgEndpoint.DstIPPort().String()), true, wgEndpoint, &n.RouterTunables)
 			newEp.Renew()
 			neigh.Eps = append(neigh.Eps, newEp)
 			// push route update to improve convergence time
@@ -135,7 +135,7 @@ func handleProbePong(n *Nylon, node state.NodeId, token uint64, ep conn.Endpoint
 					health := linkHealth.Value()
 					latency := time.Since(health.TimeSent)
 					// we have a link
-					if state.DBG_log_probe {
+					if n.DBG_log_probe {
 						n.Log.Debug("probe back", "peer", node, "ping", latency)
 					}
 					dpLink.Renew()
@@ -194,7 +194,7 @@ func (n *Nylon) probeNew() error {
 			})
 			if idx == -1 {
 				// add the link to the neighbour
-				dpl := state.NewEndpoint(ep, false, nil)
+				dpl := state.NewEndpoint(ep, false, nil, &n.RouterTunables)
 				neigh.Eps = append(neigh.Eps, dpl)
 				err := n.Probe(peer, dpl, false)
 				if err != nil {

core/nylon_passive.go

@@ -9,7 +9,7 @@ import (
 func (n *Nylon) initPassiveClient() error {
 	n.RepeatTask(func() error {
 		return scanPassivePeers(n)
-	}, state.ProbeDelay)
+	}, n.ProbeDelay)
 	return nil
 }
 
@@ -39,7 +39,7 @@ func scanPassivePeers(n *Nylon) error {
 
 			// TODO: we could make this expire after a longer period of time, like 24h. However, this would require our passive client to wait for the full route propagation time after 24 hours. (Might cause unexpected interruptions)
 
-			recentlyUpdated := time.Since(peer.LastReceivedPacket()) < state.ClientDeadThreshold
+			recentlyUpdated := time.Since(peer.LastReceivedPacket()) < n.ClientDeadThreshold
 			if n.IsClient(*nid) {
 				// we have a passive client
 				for _, newPrefix := range ncfg.Prefixes {

core/nylon_tc.go

@@ -20,7 +20,7 @@ const (
 func (n *Nylon) InstallTC() {
 	t := n.Trace
 
-	if state.DBG_trace_tc {
+	if n.DBG_trace_tc {
 		n.Device.InstallFilter(func(dev *device.Device, packet *device.TCElement) (device.TCAction, error) {
 			if packet.Validate() { // make sure it's an IP packet
 				peer := packet.FromPeer
@@ -59,7 +59,7 @@ func (n *Nylon) InstallTC() {
 					return device.TcDrop, nil
 				}
 				packet.ToPeer = entry.Peer
-				if state.DBG_trace_tc {
+				if n.DBG_trace_tc {
 					t.Submit(fmt.Sprintf("Fwd packet: %v -> %v, via %s\n", packet.GetSrc(), packet.GetDst(), entry.Nh))
 				}
 				return device.TcForward, nil
@@ -75,7 +75,7 @@ func (n *Nylon) InstallTC() {
 					return device.TcDrop, nil
 				}
 				packet.ToPeer = entry.Peer
-				if state.DBG_trace_tc {
+				if n.DBG_trace_tc {
 					t.Submit(fmt.Sprintf("Fwd packet: %v -> %v, via %s\n", packet.GetSrc(), packet.GetDst(), entry.Nh))
 				}
 				return device.TcForward, nil
@@ -93,7 +93,7 @@ func (n *Nylon) InstallTC() {
 					packet.DecrementTTL()
 				}
 				if ttl == 0 {
-					if state.DBG_trace_tc {
+					if n.DBG_trace_tc {
 						t.Submit(fmt.Sprintf("TTL Expired: %v -> %v\n", packet.GetSrc(), packet.GetDst()))
 					}
 					return device.TcBounce, nil
@@ -110,7 +110,7 @@ func (n *Nylon) InstallTC() {
 		entry, ok := n.router.ExitTable.Load().Lookup(packet.GetDst())
 		// we should only accept packets destined to us, but not our passive clients
 		if ok && entry.Nh == n.LocalCfg.Id {
-			if state.DBG_trace_tc {
+			if n.DBG_trace_tc {
 				t.Submit(fmt.Sprintf("Exit: %v -> %v\n", packet.GetSrc(), packet.GetDst()))
 			}
 			//dev.Log.Verbosef("BounceCur packet: %v -> %v", packet.GetSrc(), packet.GetDst())

core/nylon_wireguard.go

@@ -92,7 +92,7 @@ listen_port=%d
 	// init wireguard related tasks
 	n.RepeatTask(func() error {
 		return n.UpdateWireGuard()
-	}, state.ProbeDelay)
+	}, n.ProbeDelay)
 
 	return nil
 }