diff --git a/control/cmd/control/main.go b/control/cmd/control/main.go index 6f3766517b..1d2764b038 100644 --- a/control/cmd/control/main.go +++ b/control/cmd/control/main.go @@ -222,7 +222,7 @@ func realMain(ctx context.Context) error { SCIONNetworkMetrics: metrics.SCIONNetworkMetrics, SCIONPacketConnMetrics: metrics.SCIONPacketConnMetrics, MTU: topo.MTU(), - Topology: cpInfoProvider{topo: topo}, + Topology: adaptTopology(topo), } quicStack, err := nc.QUICStack() if err != nil { @@ -945,29 +945,24 @@ func (h *healther) GetCAHealth(ctx context.Context) (api.CAHealthStatus, bool) { return api.Unavailable, false } -type cpInfoProvider struct { - topo *topology.Loader -} - -func (c cpInfoProvider) LocalIA(_ context.Context) (addr.IA, error) { - return c.topo.IA(), nil -} - -func (c cpInfoProvider) PortRange(_ context.Context) (uint16, uint16, error) { - start, end := c.topo.PortRange() - return start, end, nil -} - -func (c cpInfoProvider) Interfaces(_ context.Context) (map[uint16]netip.AddrPort, error) { - ifMap := c.topo.InterfaceInfoMap() - ifsToUDP := make(map[uint16]netip.AddrPort, len(ifMap)) - for i, v := range ifMap { - if i > (1<<16)-1 { - return nil, serrors.New("invalid interface id", "id", i) - } - ifsToUDP[uint16(i)] = v.InternalAddr +func adaptTopology(topo *topology.Loader) snet.Topology { + start, end := topo.PortRange() + return snet.Topology{ + LocalIA: topo.IA(), + PortRange: snet.TopologyPortRange{ + Start: start, + End: end, + }, + Interface: func(ifID uint16) (netip.AddrPort, bool) { + // XXX(lukedirtwalker): The amount of conversiones between + // netip.AddrPort and *net.UDPAddr is a bit too high... + a := topo.UnderlayNextHop(ifID) + if a == nil { + return netip.AddrPort{}, false + } + return a.AddrPort(), true + }, } - return ifsToUDP, nil } func getCAHealth( diff --git a/gateway/gateway.go b/gateway/gateway.go index c6395be076..2f2233dfd6 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -240,10 +240,16 @@ func (g *Gateway) Run(ctx context.Context) error { // ********************************************* // Initialize base SCION network information: IA // ********************************************* - localIA, err := g.Daemon.LocalIA(context.Background()) + topoReloader, err := daemon.NewReloadingTopology(ctx, g.Daemon) if err != nil { - return serrors.Wrap("unable to learn local ISD-AS number", err) + return serrors.Wrap("loading topology", err) } + topo := topoReloader.Topology() + go func() { + defer log.HandlePanic() + topoReloader.Run(ctx, 10*time.Second) + }() + localIA := topo.LocalIA logger.Info("Learned local IA from SCION Daemon", "ia", localIA) // ************************************************************************* @@ -299,7 +305,7 @@ func (g *Gateway) Run(ctx context.Context) error { ProbesSendErrors: probesSendErrors, SCMPErrors: g.Metrics.SCMPErrors, SCIONPacketConnMetrics: g.Metrics.SCIONPacketConnMetrics, - Topology: g.Daemon, + Topology: topo, }, PathUpdateInterval: PathUpdateInterval(ctx), PathFetchTimeout: 0, // using default for now @@ -409,7 +415,7 @@ func (g *Gateway) Run(ctx context.Context) error { // scionNetworkNoSCMP is the network for the QUIC server connection. Because SCMP errors // will cause the server's accepts to fail, we ignore SCMP. scionNetworkNoSCMP := &snet.SCIONNetwork{ - Topology: g.Daemon, + Topology: topo, // Discard all SCMP propagation, to avoid accept/read errors on the // QUIC server/client. SCMPHandler: snet.SCMPPropagationStopper{ @@ -472,7 +478,7 @@ func (g *Gateway) Run(ctx context.Context) error { // scionNetwork is the network for all SCION connections, with the exception of the QUIC server // and client connection. scionNetwork := &snet.SCIONNetwork{ - Topology: g.Daemon, + Topology: topo, SCMPHandler: snet.DefaultSCMPHandler{ RevocationHandler: revocationHandler, SCMPErrors: g.Metrics.SCMPErrors, diff --git a/pkg/daemon/topology.go b/pkg/daemon/topology.go new file mode 100644 index 0000000000..bc208033e0 --- /dev/null +++ b/pkg/daemon/topology.go @@ -0,0 +1,117 @@ +// Copyright 2024 Anapaya Systems +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "net/netip" + "sync" + "time" + + "github.com/scionproto/scion/pkg/log" + "github.com/scionproto/scion/pkg/private/serrors" + "github.com/scionproto/scion/pkg/snet" +) + +// LoadTopology loads the local topology from the given connector. The topology +// information is loaded once and does not update automatically. +func LoadTopology(ctx context.Context, conn Connector) (snet.Topology, error) { + ia, err := conn.LocalIA(ctx) + if err != nil { + return snet.Topology{}, serrors.Wrap("loading local ISD-AS", err) + } + start, end, err := conn.PortRange(ctx) + if err != nil { + return snet.Topology{}, serrors.Wrap("loading port range", err) + } + interfaces, err := conn.Interfaces(ctx) + if err != nil { + return snet.Topology{}, serrors.Wrap("loading interfaces", err) + } + + return snet.Topology{ + LocalIA: ia, + PortRange: snet.TopologyPortRange{ + Start: start, + End: end, + }, + Interface: func(ifID uint16) (netip.AddrPort, bool) { + a, ok := interfaces[ifID] + return a, ok + }, + }, nil +} + +// ReloadingTopology is a topology that reloads the interface information +// periodically. It is safe for concurrent use. +type ReloadingTopology struct { + conn Connector + baseTopology snet.Topology + interfaces sync.Map +} + +// NewReloadingTopology creates a new ReloadingTopology that reloads the +// interface information periodically. The Run method must be called for +// interface information to be populated. +func NewReloadingTopology(ctx context.Context, conn Connector) (*ReloadingTopology, error) { + topo, err := LoadTopology(ctx, conn) + if err != nil { + return nil, err + } + return &ReloadingTopology{ + conn: conn, + baseTopology: topo, + }, nil +} + +func (t *ReloadingTopology) Topology() snet.Topology { + base := t.baseTopology + return snet.Topology{ + LocalIA: base.LocalIA, + PortRange: base.PortRange, + Interface: func(ifID uint16) (netip.AddrPort, bool) { + a, ok := t.interfaces.Load(ifID) + if !ok { + return netip.AddrPort{}, false + } + return a.(netip.AddrPort), true + }, + } +} + +func (t *ReloadingTopology) Run(ctx context.Context, period time.Duration) { + ticker := time.NewTicker(period) + defer ticker.Stop() + + reload := func() { + intfs, err := t.conn.Interfaces(ctx) + if err != nil { + log.FromCtx(ctx).Error("Failed to reload interfaces", "err", err) + } + for ifID, addr := range intfs { + t.interfaces.Store(ifID, addr) + } + } + + reload() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + reload() + } + } +} diff --git a/pkg/snet/conn.go b/pkg/snet/conn.go index 3ec657f299..b2635dcbc6 100644 --- a/pkg/snet/conn.go +++ b/pkg/snet/conn.go @@ -16,7 +16,6 @@ package snet import ( - "context" "net" "time" @@ -65,21 +64,13 @@ func NewCookedConn( options ...ConnOption, ) (*Conn, error) { o := apply(options) - localIA, err := topo.LocalIA(context.Background()) - if err != nil { - return nil, err - } local := &UDPAddr{ - IA: localIA, + IA: topo.LocalIA, Host: pconn.LocalAddr().(*net.UDPAddr), } if local.Host == nil || local.Host.IP.IsUnspecified() { return nil, serrors.New("nil or unspecified address is not supported.") } - start, end, err := topo.PortRange(context.Background()) - if err != nil { - return nil, err - } return &Conn{ conn: pconn, local: local, @@ -89,8 +80,8 @@ func NewCookedConn( buffer: make([]byte, common.SupportedMTU), local: local, remote: o.remote, - dispatchedPortStart: start, - dispatchedPortEnd: end, + dispatchedPortStart: topo.PortRange.Start, + dispatchedPortEnd: topo.PortRange.End, }, scionConnReader: scionConnReader{ conn: pconn, diff --git a/pkg/snet/packet_conn.go b/pkg/snet/packet_conn.go index dd848a6799..9352a2c734 100644 --- a/pkg/snet/packet_conn.go +++ b/pkg/snet/packet_conn.go @@ -15,7 +15,6 @@ package snet import ( - "context" "net" "syscall" "time" @@ -343,16 +342,7 @@ func (c *SCIONPacketConn) lastHop(p *Packet) (*net.UDPAddr, error) { } func (c *SCIONPacketConn) ifIDToAddr(ifID uint16) (*net.UDPAddr, error) { - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - intfs, err := c.Topology.Interfaces(ctx) - if err != nil { - return nil, serrors.Wrap( - "resolving interfaces address (fetching interfaces)", err, - "interface", ifID, - ) - } - addrPort, ok := intfs[ifID] + addrPort, ok := c.Topology.Interface(ifID) if !ok { return nil, serrors.New("interface number not found", "interface", ifID) } diff --git a/pkg/snet/snet.go b/pkg/snet/snet.go index fe2d46a9f9..189a8c046f 100644 --- a/pkg/snet/snet.go +++ b/pkg/snet/snet.go @@ -49,11 +49,22 @@ import ( "github.com/scionproto/scion/pkg/private/serrors" ) -// Topology provides local-IA topology information -type Topology interface { - LocalIA(ctx context.Context) (addr.IA, error) - PortRange(ctx context.Context) (uint16, uint16, error) - Interfaces(ctx context.Context) (map[uint16]netip.AddrPort, error) +// Topology provides information about the topology of the local ISD-AS. +type Topology struct { + // LocalIA is local ISD-AS. + LocalIA addr.IA + // PortRange is the directly dispatched port range. Start and End are + // inclusive. + PortRange TopologyPortRange + // Interface provides information about a local interface. If the interface + // is not present, the second return value must be false. + Interface func(uint16) (netip.AddrPort, bool) +} + +// TopologyPortRange is the range of ports that are directly dispatched to the +// application. The range is inclusive. +type TopologyPortRange struct { + Start, End uint16 } var _ Network = (*SCIONNetwork)(nil) @@ -68,7 +79,8 @@ type SCIONNetworkMetrics struct { // SCIONNetwork is the SCION networking context. type SCIONNetwork struct { // Topology provides local AS information, needed to handle sockets and - // traffic. + // traffic. Note that the Interfaces method might be called once per packet, + // so an efficient implementation is strongly recommended. Topology Topology // ReplyPather is used to create reply paths when reading packets on Conn // (that implements net.Conn). If unset, the default reply pather is used, @@ -91,10 +103,7 @@ func (n *SCIONNetwork) OpenRaw(ctx context.Context, addr *net.UDPAddr) (PacketCo if addr == nil || addr.IP.IsUnspecified() { return nil, serrors.New("nil or unspecified address is not supported") } - start, end, err := n.Topology.PortRange(ctx) - if err != nil { - return nil, err - } + start, end := n.Topology.PortRange.Start, n.Topology.PortRange.End if addr.Port == 0 { pconn, err = listenUDPRange(addr, start, end) } else { diff --git a/private/app/path/path.go b/private/app/path/path.go index 937ca0a580..5a61b8bb64 100644 --- a/private/app/path/path.go +++ b/private/app/path/path.go @@ -83,6 +83,10 @@ func Choose( if err != nil { return nil, serrors.Wrap("fetching paths", err) } + topo, err := daemon.LoadTopology(ctx, conn) + if err != nil { + return nil, serrors.Wrap("loading topology", err) + } if o.epic { // Only use paths that support EPIC and intra-AS (empty) paths. epicPaths := []snet.Path{} @@ -102,7 +106,7 @@ func Choose( paths = epicPaths } if o.probeCfg != nil { - paths, err = filterUnhealthy(ctx, paths, remote, conn, o.probeCfg, o.epic) + paths, err = filterUnhealthy(ctx, paths, remote, topo, o.probeCfg, o.epic) if err != nil { return nil, serrors.Wrap("probing paths", err) } @@ -121,7 +125,7 @@ func filterUnhealthy( ctx context.Context, paths []snet.Path, remote addr.IA, - sd daemon.Connector, + topo snet.Topology, cfg *ProbeConfig, epic bool, ) ([]snet.Path, error) { @@ -144,7 +148,7 @@ func filterUnhealthy( LocalIA: cfg.LocalIA, LocalIP: cfg.LocalIP, SCIONPacketConnMetrics: cfg.SCIONPacketConnMetrics, - Topology: sd, + Topology: topo, }.GetStatuses(subCtx, nonEmptyPaths, pathprobe.WithEPIC(epic)) if err != nil { return nil, serrors.Wrap("probing paths", err) diff --git a/scion-pki/certs/renew.go b/scion-pki/certs/renew.go index 2b28c5de72..a074777fe9 100644 --- a/scion-pki/certs/renew.go +++ b/scion-pki/certs/renew.go @@ -739,9 +739,13 @@ func (r *renewer) requestRemote( IA: r.LocalIA, Host: &net.UDPAddr{IP: localIP}, } + topo, err := daemon.LoadTopology(ctx, r.Daemon) + if err != nil { + return nil, serrors.Wrap("loading topology", err) + } sn := &snet.SCIONNetwork{ - Topology: r.Daemon, + Topology: topo, SCMPHandler: snet.SCMPPropagationStopper{ Handler: snet.DefaultSCMPHandler{ RevocationHandler: daemon.RevHandler{Connector: r.Daemon}, diff --git a/scion/cmd/scion/ping.go b/scion/cmd/scion/ping.go index 85645eccef..1db9204da6 100644 --- a/scion/cmd/scion/ping.go +++ b/scion/cmd/scion/ping.go @@ -151,11 +151,12 @@ On other errors, ping will exit with code 2. } defer sd.Close() - info, err := app.QueryASInfo(traceCtx, sd) + topo, err := daemon.LoadTopology(ctx, sd) if err != nil { - return err + return serrors.Wrap("loading topology", err) } - span.SetTag("src.isd_as", info.IA) + + span.SetTag("src.isd_as", topo.LocalIA) opts := []path.Option{ path.WithInteractive(flags.interactive), @@ -166,7 +167,7 @@ On other errors, ping will exit with code 2. } if flags.healthyOnly { opts = append(opts, path.WithProbing(&path.ProbeConfig{ - LocalIA: info.IA, + LocalIA: topo.LocalIA, LocalIP: localIP, })) } @@ -212,7 +213,7 @@ On other errors, ping will exit with code 2. panic("Invalid Local IP address") } local := addr.Addr{ - IA: info.IA, + IA: topo.LocalIA, Host: addr.HostIP(asNetipAddr), } pldSize := int(flags.size) @@ -266,7 +267,7 @@ On other errors, ping will exit with code 2. } stats, err := ping.Run(ctx, ping.Config{ - Topology: sd, + Topology: topo, Attempts: count, Interval: flags.interval, Timeout: flags.timeout, diff --git a/scion/cmd/scion/traceroute.go b/scion/cmd/scion/traceroute.go index 51777d9c39..ca28d49249 100644 --- a/scion/cmd/scion/traceroute.go +++ b/scion/cmd/scion/traceroute.go @@ -125,11 +125,11 @@ On other errors, traceroute will exit with code 2. return serrors.Wrap("connecting to SCION Daemon", err) } defer sd.Close() - info, err := app.QueryASInfo(traceCtx, sd) + topo, err := daemon.LoadTopology(ctx, sd) if err != nil { - return err + return serrors.Wrap("loading topology", err) } - span.SetTag("src.isd_as", info.IA) + span.SetTag("src.isd_as", topo.LocalIA) path, err := path.Choose(traceCtx, sd, remote.IA, path.WithInteractive(flags.interactive), path.WithRefresh(flags.refresh), @@ -180,14 +180,14 @@ On other errors, traceroute will exit with code 2. panic("Invalid Local IP address") } local := addr.Addr{ - IA: info.IA, + IA: topo.LocalIA, Host: addr.HostIP(asNetipAddr), } ctx = app.WithSignal(traceCtx, os.Interrupt, syscall.SIGTERM) var stats traceroute.Stats var updates []traceroute.Update cfg := traceroute.Config{ - Topology: sd, + Topology: topo, Remote: remote, NextHop: nextHop, MTU: path.Metadata().MTU, diff --git a/scion/showpaths/showpaths.go b/scion/showpaths/showpaths.go index b0c301d378..e80ccbc1dd 100644 --- a/scion/showpaths/showpaths.go +++ b/scion/showpaths/showpaths.go @@ -309,10 +309,11 @@ func Run(ctx context.Context, dst addr.IA, cfg Config) (*Result, error) { return nil, serrors.Wrap("connecting to the SCION Daemon", err, "addr", cfg.Daemon) } defer sdConn.Close() - localIA, err := sdConn.LocalIA(ctx) + topo, err := daemon.LoadTopology(ctx, sdConn) if err != nil { - return nil, serrors.Wrap("determining local ISD-AS", err) + return nil, serrors.Wrap("loading topology", err) } + localIA := topo.LocalIA if dst == localIA { return &Result{ LocalIA: localIA, @@ -355,7 +356,7 @@ func Run(ctx context.Context, dst addr.IA, cfg Config) (*Result, error) { DstIA: dst, LocalIA: localIA, LocalIP: cfg.Local, - Topology: sdConn, + Topology: topo, }.GetStatuses(ctx, p, pathprobe.WithEPIC(cfg.Epic)) if err != nil { return nil, serrors.Wrap("getting statuses", err) diff --git a/tools/end2end/main.go b/tools/end2end/main.go index 55b5a6ac24..03a0268f2d 100644 --- a/tools/end2end/main.go +++ b/tools/end2end/main.go @@ -134,13 +134,21 @@ func (s server) run() { sdConn := integration.SDConn() defer sdConn.Close() + + loadCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + topo, err := daemon.LoadTopology(loadCtx, sdConn) + if err != nil { + integration.LogFatal("Error loading topology", "err", err) + } + sn := &snet.SCIONNetwork{ SCMPHandler: snet.DefaultSCMPHandler{ RevocationHandler: daemon.RevHandler{Connector: sdConn}, SCMPErrors: scmpErrorsCounter, }, PacketConnMetrics: scionPacketConnMetrics, - Topology: sdConn, + Topology: topo, } conn, err := sn.Listen(context.Background(), "udp", integration.Local.Host) if err != nil { @@ -233,13 +241,21 @@ func (c *client) run() int { defer integration.Done(integration.Local.IA, remote.IA) c.sdConn = integration.SDConn() defer c.sdConn.Close() + + loadCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + topo, err := daemon.LoadTopology(loadCtx, c.sdConn) + if err != nil { + integration.LogFatal("Error loading topology", "err", err) + } + c.network = &snet.SCIONNetwork{ SCMPHandler: snet.DefaultSCMPHandler{ RevocationHandler: daemon.RevHandler{Connector: c.sdConn}, SCMPErrors: scmpErrorsCounter, }, PacketConnMetrics: scionPacketConnMetrics, - Topology: c.sdConn, + Topology: topo, } log.Info("Send", "local", fmt.Sprintf("%v,[%v] -> %v,[%v]",