lib/events: Become a service (fixes #5372) (#5373)

Here the event Logger is rewritten as a service with a main loop instead
of mutexes. This loop has a select with essentially two legs: incoming
events, and subscription changes. When both are possible select will
chose one randomly, thus ensuring that in practice unsubscribes will
happen timely and not block the system.

lib/events/events_test.go

@@ -9,6 +9,7 @@ package events
 import (
 	"encoding/json"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 )
@@ -28,6 +29,9 @@ func TestNewLogger(t *testing.T) {
 
 func TestSubscriber(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
+
 	s := l.Subscribe(0)
 	defer l.Unsubscribe(s)
 	if s == nil {
@@ -37,6 +41,9 @@ func TestSubscriber(t *testing.T) {
 
 func TestTimeout(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
+
 	s := l.Subscribe(0)
 	defer l.Unsubscribe(s)
 	_, err := s.Poll(timeout)
@@ -47,6 +54,8 @@ func TestTimeout(t *testing.T) {
 
 func TestEventBeforeSubscribe(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	l.Log(DeviceConnected, "foo")
 	s := l.Subscribe(0)
@@ -60,6 +69,8 @@ func TestEventBeforeSubscribe(t *testing.T) {
 
 func TestEventAfterSubscribe(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	defer l.Unsubscribe(s)
@@ -85,6 +96,8 @@ func TestEventAfterSubscribe(t *testing.T) {
 
 func TestEventAfterSubscribeIgnoreMask(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(DeviceDisconnected)
 	defer l.Unsubscribe(s)
@@ -98,6 +111,8 @@ func TestEventAfterSubscribeIgnoreMask(t *testing.T) {
 
 func TestBufferOverflow(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	defer l.Unsubscribe(s)
@@ -121,6 +136,8 @@ func TestBufferOverflow(t *testing.T) {
 
 func TestUnsubscribe(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	l.Log(DeviceConnected, "foo")
@@ -141,6 +158,8 @@ func TestUnsubscribe(t *testing.T) {
 
 func TestGlobalIDs(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	defer l.Unsubscribe(s)
@@ -171,6 +190,8 @@ func TestGlobalIDs(t *testing.T) {
 
 func TestSubscriptionIDs(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(DeviceConnected)
 	defer l.Unsubscribe(s)
@@ -211,6 +232,8 @@ func TestSubscriptionIDs(t *testing.T) {
 
 func TestBufferedSub(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	defer l.Unsubscribe(s)
@@ -240,6 +263,8 @@ func TestBufferedSub(t *testing.T) {
 
 func BenchmarkBufferedSub(b *testing.B) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
 	defer l.Unsubscribe(s)
@@ -294,6 +319,8 @@ func BenchmarkBufferedSub(b *testing.B) {
 
 func TestSinceUsesSubscriptionId(t *testing.T) {
 	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
 
 	s := l.Subscribe(DeviceConnected)
 	defer l.Unsubscribe(s)
@@ -339,3 +366,93 @@ func TestUnmarshalEvent(t *testing.T) {
 		t.Fatal("Failed to unmarshal event:", err)
 	}
 }
+
+func TestUnsubscribeContention(t *testing.T) {
+	// Check that we can unsubscribe without blocking the whole system.
+
+	const (
+		listeners = 50
+		senders   = 1000
+	)
+
+	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
+
+	// Start listeners. These will poll until the stop channel is closed,
+	// then exit and unsubscribe.
+
+	stopListeners := make(chan struct{})
+	var listenerWg sync.WaitGroup
+	listenerWg.Add(listeners)
+	for i := 0; i < listeners; i++ {
+		go func() {
+			defer listenerWg.Done()
+
+			s := l.Subscribe(AllEvents)
+			defer l.Unsubscribe(s)
+
+			for {
+				select {
+				case <-s.C():
+
+				case <-stopListeners:
+					return
+				}
+			}
+		}()
+	}
+
+	// Start senders. These send pointless events until the stop channel is
+	// closed.
+
+	stopSenders := make(chan struct{})
+	defer close(stopSenders)
+	var senderWg sync.WaitGroup
+	senderWg.Add(senders)
+	for i := 0; i < senders; i++ {
+		go func() {
+			defer senderWg.Done()
+
+			t := time.NewTicker(time.Millisecond)
+
+			for {
+				select {
+				case <-t.C:
+					l.Log(StateChanged, nil)
+
+				case <-stopSenders:
+					return
+				}
+			}
+		}()
+	}
+
+	// Give everything time to start up.
+
+	time.Sleep(time.Second)
+
+	// Stop the listeners and wait for them to exit. This should happen in a
+	// reasonable time frame.
+
+	t0 := time.Now()
+	close(stopListeners)
+	listenerWg.Wait()
+	if d := time.Since(t0); d > time.Minute {
+		t.Error("It should not take", d, "to unsubscribe from an event stream")
+	}
+}
+
+func BenchmarkLogEvent(b *testing.B) {
+	l := NewLogger()
+	defer l.Stop()
+	go l.Serve()
+
+	s := l.Subscribe(AllEvents)
+	defer l.Unsubscribe(s)
+	NewBufferedSubscription(s, 1) // runs in the background
+
+	for i := 0; i < b.N; i++ {
+		l.Log(StateChanged, nil)
+	}
+}