implement tableflip for graceful upgrades:

- add tableflip option to config.yml
- implement tableflip in conn.go Listen()
- add secondary prometheus listener so using tableflip will rotate and keep both online
- implement conngroup for checking if all connections have died out

README.md

@@ -21,6 +21,7 @@ fork from [haveachin/infrared](https://github.com/haveachin/infrared)
 - Status packet caching
 - Bandwith usage tracking for proxy configs through prometheus
 - Use redis to get proxy configs ([lhridder/infrapi](https://github.com/lhridder/infrapi))
+- Live upgrades using [tableflip](https://github.com/cloudflare/tableflip)
 
 ## Command-Line Flags
 
@@ -60,13 +61,17 @@ genericPing:
   version: infrared
   description: There is no proxy associated with this domain. Please check your configuration.
   iconPath: 
+tableflip:
+  enabled: false
+  pidfile: infrared.pid
 ```
 Values can be left out if they don't deviate from the default, an empty config.yml is still required for startup.
 ### Fields
 - `receiveProxyProtocol` whether to allow for inbound proxyProtocol connections.
 - prometheus:
   - `enabled` whether to enable to builtin prometheus exporter or not.
   - `bind` on what port/address to have the prometheus exporter listen on.
+  - `bind2` what secondary port should be used when using tableflip.
 - api:
   - `nabled` if the json http api should be enabled.
   - `bind` on what port/address to have the api listen on.
@@ -84,6 +89,9 @@ Values can be left out if they don't deviate from the default, an empty config.y
   - `host` what redis server to connect to when caching geoip and username lookups.
   - `DB` what redis db should be used on the redis server.
   - `pass` what password should be used when logging into the redis server.
+- tableflip:
+  - `enabled` whether or not tableflip should be used.
+  - `pidfile` where the PID file used for tableflip is located.
 - `rejoinMessage` what text response should be sent when a player needs to rejoin to verify they're not a bot.
 - `underAttack` if the instance should permanently be in attack mode.
 - `debug` if debug logs should be enabled.
@@ -229,6 +237,36 @@ redis:
 * Increasing `net.core.somaxconn` in sysctl to for example 50000 (default is 4096). Can be done with `sysctl net.core.somaxconn=50000`.
 * Increasing the `ulimit` to for example 500000 (default is 1024). Can be done with `ulimit -n 500000` when running in a terminal or `LimitNOFILE=500000` in a systemd service file.
 
+## Tableflip
+[Tableflip](https://github.com/cloudflare/tableflip) allows for the golang application to be upgraded live by swapping the binary and creating a new process without killing off existing connections.\
+#### Systemd
+To use this feature running infrared under systemd is required, here an example of how the .service file should look:
+Upgrades can then be triggered with `systemctl reload infrared`.
+```text
+[Unit]
+Description=Infrared
+After=network-online.target
+
+[Service]
+User=root
+Group=root
+WorkingDirectory=/srv/infrared
+ExecStart=/srv/infrared/infrared
+ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/srv/infrared/infrared.pid
+LimitNOFILE=500000
+LimitNPROC=500000
+
+[Install]
+WantedBy=multi-user.target
+```
+#### Configuration
+```yaml
+tableflip:
+  enabled: true
+  pidfile: infrared.pid
+```
+
 ## API
 ### Route examples
 GET `/proxies` will return
@@ -272,3 +310,4 @@ GET `/` will return 200(OK)
 - [MMDB geoip library for golang](https://github.com/oschwald/geoip2-golang)
 - [Govalidator](https://github.com/asaskevich/govalidator)
 - [Mux router](https://github.com/gorilla/mux)
+- [Tableflip](https://github.com/cloudflare/tableflip)

cmd/infrared/main.go

@@ -1,11 +1,16 @@
 package main
 
 import (
+	"errors"
 	"flag"
+	"fmt"
+	"github.com/cloudflare/tableflip"
 	"github.com/haveachin/infrared"
 	"github.com/haveachin/infrared/api"
 	"log"
 	"os"
+	"os/signal"
+	"syscall"
 )
 
 const (
@@ -107,6 +112,40 @@ func main() {
 		}
 	}()
 
+	if infrared.Config.Tableflip.Enabled {
+		log.Println("Starting tableflip upgrade listener")
+
+		if _, err := os.Stat(infrared.Config.Tableflip.PIDfile); errors.Is(err, os.ErrNotExist) {
+			pid := fmt.Sprint(os.Getpid())
+			bb := []byte(pid)
+			err := os.WriteFile(infrared.Config.Tableflip.PIDfile, bb, os.ModePerm)
+			if err != nil {
+				log.Printf("Failed to set up PIDfile: %s", err)
+				return
+			}
+		}
+
+		var err error
+		infrared.Upg, err = tableflip.New(tableflip.Options{
+			PIDFile: infrared.Config.Tableflip.PIDfile,
+		})
+		if err != nil {
+			log.Printf("Failed to set up Tableflip Upgrader: %s", err)
+			return
+		}
+
+		go func() {
+			sig := make(chan os.Signal, 1)
+			signal.Notify(sig, syscall.SIGHUP)
+			for range sig {
+				err := infrared.Upg.Upgrade()
+				if err != nil {
+					log.Println("upgrade failed:", err)
+				}
+			}
+		}()
+	}
+
 	if infrared.Config.Api.Enabled && !infrared.Config.UseRedisConfig {
 		go api.ListenAndServe(configPath, infrared.Config.Api.Bind)
 	}
@@ -163,5 +202,15 @@ func main() {
 		log.Fatal("Gateway exited; error: ", err)
 	}
 
-	gateway.KeepProcessActive()
+	if infrared.Config.Tableflip.Enabled {
+		if err := infrared.Upg.Ready(); err != nil {
+			panic(err)
+		}
+		<-infrared.Upg.Exit()
+
+		gateway.WaitConnGroup()
+		log.Println("Shutting down infrared")
+	} else {
+		gateway.KeepProcessActive()
+	}
 }

config.go

@@ -56,6 +56,7 @@ type Redis struct {
 type Service struct {
 	Enabled bool   `yaml:"enabled"`
 	Bind    string `yaml:"bind"`
+	Bind2   string `yaml:"bind2"`
 }
 
 type GenericPing struct {
@@ -70,6 +71,11 @@ type GeoIP struct {
 	EnableIprisk bool   `yaml:"enableIprisk"`
 }
 
+type Tableflip struct {
+	Enabled bool   `yaml:"enabled"`
+	PIDfile string `yaml:"pidfile"`
+}
+
 type GlobalConfig struct {
 	Debug                bool   `yaml:"debug"`
 	ReceiveProxyProtocol bool   `yaml:"receiveProxyProtocol"`
@@ -85,6 +91,7 @@ type GlobalConfig struct {
 	Prometheus           Service
 	GeoIP                GeoIP
 	GenericPing          GenericPing
+	Tableflip            Tableflip
 }
 
 type redisEvent struct {
@@ -130,6 +137,10 @@ var DefaultConfig = GlobalConfig{
 		Description: "There is no proxy associated with this domain. Please check your configuration.",
 		IconPath:    "",
 	},
+	Tableflip: Tableflip{
+		Enabled: false,
+		PIDfile: "",
+	},
 }
 
 func (cfg *ProxyConfig) Dialer() (*Dialer, error) {

conn.go

@@ -36,6 +36,11 @@ type Listener struct {
 }
 
 func Listen(addr string) (Listener, error) {
+	if Config.Tableflip.Enabled {
+		l, err := Upg.Listen("tcp", addr)
+		return Listener{Listener: l}, err
+	}
+
 	l, err := net.Listen("tcp", addr)
 	return Listener{Listener: l}, err
 }

gateway.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"github.com/Lukaesebrot/mojango"
 	"github.com/asaskevich/govalidator"
+	"github.com/cloudflare/tableflip"
 	"github.com/go-redis/redis/v8"
 	"github.com/gofrs/uuid"
 	"github.com/haveachin/infrared/protocol"
@@ -50,13 +51,15 @@ var (
 		Help: "The total number of used bytes of bandwith per proxy",
 	}, []string{"host"})
 	ctx = context.Background()
+	Upg *tableflip.Upgrader
 )
 
 type Gateway struct {
 	listeners            sync.Map
 	Proxies              sync.Map
 	closed               chan bool
 	wg                   sync.WaitGroup
+	conngroup            sync.WaitGroup
 	ReceiveProxyProtocol bool
 	underAttack          bool
 	connections          int
@@ -143,7 +146,28 @@ func (gateway *Gateway) EnablePrometheus(bind string) error {
 		defer gateway.wg.Done()
 
 		http.Handle("/metrics", promhttp.Handler())
-		http.ListenAndServe(bind, nil)
+
+		if Config.Tableflip.Enabled {
+			var listen net.Listener
+			var err error
+			listen, err = net.Listen("tcp", bind)
+			if err != nil {
+				if strings.Contains(err.Error(), "bind: address already in use") {
+					log.Printf("Starting secondary prometheus listener on %s", Config.Prometheus.Bind2)
+					listen, err = net.Listen("tcp", Config.Prometheus.Bind2)
+					if err != nil {
+						log.Printf("Failed to open secondary prometheus listener: %s", err)
+						return
+					}
+				} else {
+					log.Printf("Failed to open new prometheus listener: %s", err)
+					return
+				}
+			}
+			http.Serve(listen, nil)
+		} else {
+			http.ListenAndServe(bind, nil)
+		}
 	}()
 
 	log.Println("Enabling Prometheus metrics endpoint on", bind)
@@ -168,6 +192,10 @@ func (gateway *Gateway) KeepProcessActive() {
 	gateway.wg.Wait()
 }
 
+func (gateway *Gateway) WaitConnGroup() {
+	gateway.conngroup.Wait()
+}
+
 // Close closes all listeners
 func (gateway *Gateway) Close() {
 	gateway.listeners.Range(func(k, v interface{}) bool {
@@ -278,6 +306,8 @@ func (gateway *Gateway) listenAndServe(listener Listener, addr string) error {
 		}
 
 		go func() {
+			gateway.conngroup.Add(1)
+			defer gateway.conngroup.Done()
 			if Config.Debug {
 				log.Printf("[>] Incoming %s on listener %s", conn.RemoteAddr(), addr)
 			}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Lukaesebrot/mojango v0.0.0-20200623100037-76cbec69139f
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
+	github.com/cloudflare/tableflip v1.2.3
 	github.com/containerd/containerd v1.4.3 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v20.10.3+incompatible
@@ -27,6 +28,6 @@ require (
 	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
 	google.golang.org/grpc v1.35.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.0.3 // indirect
 )

go.sum

@@ -41,6 +41,8 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/tableflip v1.2.3 h1:8I+B99QnnEWPHOY3fWipwVKxS70LGgUsslG7CSfmHMw=
+github.com/cloudflare/tableflip v1.2.3/go.mod h1:P4gRehmV6Z2bY5ao5ml9Pd8u6kuEnlB37pUFMmv7j2E=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -440,6 +442,7 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=