lib/ur: Implement crash (panic) reporting (fixes #959) (#5702)

* lib/ur: Implement crash (panic) reporting (fixes #959)

This implements a simple crash reporting method. It piggybacks on the
panic log files created by the monitor process, picking these up and
uploading them from the usage reporting routine.

A new config value points to the crash receiver base URL, which defaults
to "https://crash.syncthing.net/newcrash" (following the pattern of
"https://data.syncthing.net/newdata" for usage reports, but allowing us
to separate the service as required).

cmd/syncthing/monitor.go

@@ -8,6 +8,7 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"io"
 	"os"
 	"os/exec"
@@ -33,6 +34,8 @@ const (
 	loopThreshold         = 60 * time.Second
 	logFileAutoCloseDelay = 5 * time.Second
 	logFileMaxOpenTime    = time.Minute
+	panicUploadMaxWait    = 30 * time.Second
+	panicUploadNoticeWait = 10 * time.Second
 )
 
 func monitorMain(runtimeOptions RuntimeOptions) {
@@ -72,6 +75,8 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 	childEnv := childEnv()
 	first := true
 	for {
+		maybeReportPanics()
+
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
 			os.Exit(exitError)
@@ -173,6 +178,13 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
 	var panicFd *os.File
+	defer func() {
+		if panicFd != nil {
+			_ = panicFd.Close()
+			maybeReportPanics()
+		}
+	}()
+
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
@@ -430,3 +442,39 @@ func childEnv() []string {
 	env = append(env, "STMONITORED=yes")
 	return env
 }
+
+// maybeReportPanics tries to figure out if crash reporting is on or off,
+// and reports any panics it can find if it's enabled. We spend at most
+// panicUploadMaxWait uploading panics...
+func maybeReportPanics() {
+	// Try to get a config to see if/where panics should be reported.
+	cfg, err := loadOrDefaultConfig()
+	if err != nil {
+		l.Warnln("Couldn't load config; not reporting crash")
+		return
+	}
+
+	// Bail if we're not supposed to report panics.
+	opts := cfg.Options()
+	if !opts.CREnabled {
+		return
+	}
+
+	// Set up a timeout on the whole operation.
+	ctx, cancel := context.WithTimeout(context.Background(), panicUploadMaxWait)
+	defer cancel()
+
+	// Print a notice if the upload takes a long time.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(panicUploadNoticeWait):
+			l.Warnln("Uploading crash reports is taking a while, please wait...")
+		}
+	}()
+
+	// Report the panics.
+	dir := locations.GetBaseDir(locations.ConfigBaseDir)
+	uploadPanicLogs(ctx, opts.CRURL, dir)
+}