Restore Graceful Restarting & Socket Activation (#7274)

* Prevent deadlock in indexer initialisation during graceful restart

* Move from gracehttp to our own service to add graceful ssh

* Add timeout for start of indexers and make hammer time configurable

* Fix issue with re-initialization in indexer during tests

* move the code to detect use of closed to graceful

* Handle logs gracefully - add a pid suffix just before restart

* Move to using a cond and a holder for indexers

* use time.Since

* Add some comments and attribution

* update modules.txt

* Use zero to disable timeout

* Move RestartProcess to its own file

* Add cleanup routine

Author: zeripath

cmd/web.go

@@ -75,17 +75,13 @@ func runLetsEncrypt(listenAddr, domain, directory, email string, m http.Handler)
 	}
 	go func() {
 		log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
-		var err = http.ListenAndServe(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler))) // all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
+		// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
+		var err = runHTTP(setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))
 		if err != nil {
 			log.Fatal("Failed to start the Let's Encrypt handler on port %s: %v", setting.PortToRedirect, err)
 		}
 	}()
-	server := &http.Server{
-		Addr:      listenAddr,
-		Handler:   m,
-		TLSConfig: certManager.TLSConfig(),
-	}
-	return server.ListenAndServeTLS("", "")
+	return runHTTPSWithTLSConfig(listenAddr, certManager.TLSConfig(), context2.ClearHandler(m))
 }
 
 func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
@@ -101,12 +97,21 @@ func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func runWeb(ctx *cli.Context) error {
+	if os.Getppid() > 1 && len(os.Getenv("LISTEN_FDS")) > 0 {
+		log.Info("Restarting Gitea on PID: %d from parent PID: %d", os.Getpid(), os.Getppid())
+	} else {
+		log.Info("Starting Gitea on PID: %d", os.Getpid())
+	}
+
+	// Set pid file setting
 	if ctx.IsSet("pid") {
 		setting.CustomPID = ctx.String("pid")
 	}
 
+	// Perform global initialization
 	routers.GlobalInit()
 
+	// Set up Macaron
 	m := routes.NewMacaron()
 	routes.RegisterRoutes(m)
 
@@ -164,6 +169,7 @@ func runWeb(ctx *cli.Context) error {
 	var err error
 	switch setting.Protocol {
 	case setting.HTTP:
+		NoHTTPRedirector()
 		err = runHTTP(listenAddr, context2.ClearHandler(m))
 	case setting.HTTPS:
 		if setting.EnableLetsEncrypt {
@@ -172,9 +178,15 @@ func runWeb(ctx *cli.Context) error {
 		}
 		if setting.RedirectOtherPort {
 			go runHTTPRedirector()
+		} else {
+			NoHTTPRedirector()
 		}
 		err = runHTTPS(listenAddr, setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
 	case setting.FCGI:
+		NoHTTPRedirector()
+		// FCGI listeners are provided as stdin - this is orthogonal to the LISTEN_FDS approach
+		// in graceful and systemD
+		NoMainListener()
 		var listener net.Listener
 		listener, err = net.Listen("tcp", listenAddr)
 		if err != nil {
@@ -187,6 +199,10 @@ func runWeb(ctx *cli.Context) error {
 		}()
 		err = fcgi.Serve(listener, context2.ClearHandler(m))
 	case setting.UnixSocket:
+		// This could potentially be inherited using LISTEN_FDS but currently
+		// these cannot be inherited
+		NoHTTPRedirector()
+		NoMainListener()
 		if err := os.Remove(listenAddr); err != nil && !os.IsNotExist(err) {
 			log.Fatal("Failed to remove unix socket directory %s: %v", listenAddr, err)
 		}
@@ -207,8 +223,9 @@ func runWeb(ctx *cli.Context) error {
 	}
 
 	if err != nil {
-		log.Fatal("Failed to start server: %v", err)
+		log.Critical("Failed to start server: %v", err)
 	}
-
+	log.Info("HTTP Listener: %s Closed", listenAddr)
+	log.Close()
 	return nil
 }

cmd/web_graceful.go

@@ -10,36 +10,28 @@ import (
 	"crypto/tls"
 	"net/http"
 
-	"code.gitea.io/gitea/modules/log"
-
-	"github.com/facebookgo/grace/gracehttp"
+	"code.gitea.io/gitea/modules/graceful"
 )
 
 func runHTTP(listenAddr string, m http.Handler) error {
-	return gracehttp.Serve(&http.Server{
-		Addr:    listenAddr,
-		Handler: m,
-	})
+	return graceful.HTTPListenAndServe("tcp", listenAddr, m)
 }
 
 func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
-	config := &tls.Config{
-		MinVersion: tls.VersionTLS10,
-	}
-	if config.NextProtos == nil {
-		config.NextProtos = []string{"http/1.1"}
-	}
-
-	config.Certificates = make([]tls.Certificate, 1)
-	var err error
-	config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
-	if err != nil {
-		log.Fatal("Failed to load https cert file %s: %v", listenAddr, err)
-	}
-
-	return gracehttp.Serve(&http.Server{
-		Addr:      listenAddr,
-		Handler:   m,
-		TLSConfig: config,
-	})
+	return graceful.HTTPListenAndServeTLS("tcp", listenAddr, certFile, keyFile, m)
+}
+
+func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
+	return graceful.HTTPListenAndServeTLSConfig("tcp", listenAddr, tlsConfig, m)
+}
+
+// NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
+func NoHTTPRedirector() {
+	graceful.InformCleanup()
+}
+
+// NoMainListener tells our cleanup routine that we will not be using a possibly provided listener
+// for our main HTTP/HTTPS service
+func NoMainListener() {
+	graceful.InformCleanup()
 }

cmd/web_windows.go

@@ -7,6 +7,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"net/http"
 )
 
@@ -17,3 +18,20 @@ func runHTTP(listenAddr string, m http.Handler) error {
 func runHTTPS(listenAddr, certFile, keyFile string, m http.Handler) error {
 	return http.ListenAndServeTLS(listenAddr, certFile, keyFile, m)
 }
+
+func runHTTPSWithTLSConfig(listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
+	server := &http.Server{
+		Addr:      listenAddr,
+		Handler:   m,
+		TLSConfig: tlsConfig,
+	}
+	return server.ListenAndServeTLS("", "")
+}
+
+// NoHTTPRedirector is a no-op on Windows
+func NoHTTPRedirector() {
+}
+
+// NoMainListener is a no-op on Windows
+func NoMainListener() {
+}

custom/conf/app.ini.sample

@@ -244,6 +244,12 @@ LFS_CONTENT_PATH = data/lfs
 LFS_JWT_SECRET =
 ; LFS authentication validity period (in time.Duration), pushes taking longer than this may fail.
 LFS_HTTP_AUTH_EXPIRY = 20m
+; Allow graceful restarts using SIGHUP to fork
+ALLOW_GRACEFUL_RESTARTS = true
+; After a restart the parent will finish ongoing requests before
+; shutting down. Force shutdown if this process takes longer than this delay.
+; set to a negative value to disable
+GRACEFUL_HAMMER_TIME = 60s
 ; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
 STATIC_CACHE_TIME = 6h
 
@@ -299,6 +305,9 @@ ISSUE_INDEXER_QUEUE_DIR = indexers/issues.queue
 ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
 ; Batch queue number, default is 20
 ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
+; Timeout the indexer if it takes longer than this to start.
+; Set to zero to disable timeout.
+STARTUP_TIMEOUT=30s
 
 ; repo indexer by default disabled, since it uses a lot of disk space
 REPO_INDEXER_ENABLED = false

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -157,6 +157,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LETSENCRYPT_ACCEPTTOS`: **false**: This is an explicit check that you accept the terms of service for Let's Encrypt.
 - `LETSENCRYPT_DIRECTORY`: **https**: Directory that Letsencrypt will use to cache information such as certs and private keys.
 - `LETSENCRYPT_EMAIL`: **email@example.com**: Email used by Letsencrypt to notify about problems with issued certificates. (No default)
+- `ALLOW_GRACEFUL_RESTARTS`: **true**: Perform a graceful restart on SIGHUP
+- `GRACEFUL_HAMMER_TIME`: **60s**: After a restart the parent process will stop accepting new connections and will allow requests to finish before stopping. Shutdown will be forced if it takes longer than this time.
 
 ## Database (`database`)
 
@@ -189,6 +191,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
+- `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.
 
 ## Admin (`admin`)
 - `DEFAULT_EMAIL_NOTIFICATIONS`: **enabled**: Default configuration for email notifications for users (user configurable). Options: enabled, onmention, disabled

go.mod

@@ -32,13 +32,8 @@ require (
 	github.com/emirpasic/gods v1.12.0
 	github.com/etcd-io/bbolt v1.3.2 // indirect
 	github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a
-	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
-	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
-	github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f
-	github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 // indirect
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/gliderlabs/ssh v0.2.2
 	github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect

go.sum

@@ -142,20 +142,10 @@ github.com/etcd-io/bbolt v1.3.2 h1:RLRQ0TKLX7DlBRXAJHvbmXL17Q3KNnTBtZ9B6Qo+/Y0=
 github.com/etcd-io/bbolt v1.3.2/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a h1:M1bRpaZAn4GSsqu3hdK2R8H0AH9O6vqCTCbm2oAFGfE=
 github.com/ethantkoenig/rupture v0.0.0-20180203182544-0a76f03a811a/go.mod h1:MkKY/CB98aVE4VxO63X5vTQKUgcn+3XP15LMASe3lYs=
-github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
-github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
-github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
-github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
-github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f h1:0mlfEUWnUDVZnqWEVHGerL5bKYDKMEmT/Qk/W/3nGuo=
-github.com/facebookgo/grace v0.0.0-20160926231715-5729e484473f/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
-github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2 h1:3Zvf9wRhl1cOhckN1oRGWPOkIhOketmEcrQ4TeFAoR4=
-github.com/facebookgo/httpdown v0.0.0-20160323221027-a3b1354551a2/go.mod h1:TUV/fX3XrTtBQb5+ttSUJzcFgLNpILONFTKmBuk5RSw=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
-github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 h1:0YtRCqIZs2+Tz49QuH6cJVw/IFqzo39gEqZ0iYLxD2M=
-github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4/go.mod h1:vsJz7uE339KUCpBXx3JAJzSRH7Uk4iGGyJzR529qDIA=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=

models/repo_indexer.go

@@ -8,10 +8,12 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -70,9 +72,30 @@ func InitRepoIndexer() {
 	if !setting.Indexer.RepoIndexerEnabled {
 		return
 	}
+	waitChannel := make(chan time.Duration)
 	repoIndexerOperationQueue = make(chan repoIndexerOperation, setting.Indexer.UpdateQueueLength)
-	indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
-	go processRepoIndexerOperationQueue()
+	go func() {
+		start := time.Now()
+		log.Info("Initializing Repository Indexer")
+		indexer.InitRepoIndexer(populateRepoIndexerAsynchronously)
+		go processRepoIndexerOperationQueue()
+		waitChannel <- time.Since(start)
+	}()
+	if setting.Indexer.StartupTimeout > 0 {
+		go func() {
+			timeout := setting.Indexer.StartupTimeout
+			if graceful.IsChild && setting.GracefulHammerTime > 0 {
+				timeout += setting.GracefulHammerTime
+			}
+			select {
+			case duration := <-waitChannel:
+				log.Info("Repository Indexer Initialization took %v", duration)
+			case <-time.After(timeout):
+				log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
+			}
+		}()
+
+	}
 }
 
 // populateRepoIndexerAsynchronously asynchronously populates the repo indexer

modules/graceful/cleanup.go

@@ -0,0 +1,38 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package graceful
+
+import "sync"
+
+var cleanupWaitGroup sync.WaitGroup
+
+func init() {
+	cleanupWaitGroup = sync.WaitGroup{}
+
+	// There are three places that could inherit sockets:
+	//
+	// * HTTP or HTTPS main listener
+	// * HTTP redirection fallback
+	// * SSH
+	//
+	// If you add an additional place you must increment this number
+	// and add a function to call InformCleanup if it's not going to be used
+	cleanupWaitGroup.Add(3)
+
+	// Wait till we're done getting all of the listeners and then close
+	// the unused ones
+	go func() {
+		cleanupWaitGroup.Wait()
+		// Ignore the error here there's not much we can do with it
+		// They're logged in the CloseProvidedListeners function
+		_ = CloseProvidedListeners()
+	}()
+}
+
+// InformCleanup tells the cleanup wait group that we have either taken a listener
+// or will not be taking a listener
+func InformCleanup() {
+	cleanupWaitGroup.Done()
+}