silences: avoid deadlock

Calling gossip.GossipBroadcast() will cause a deadlock if
there is a currently executing OnBroadcast* function.

See prometheus#982

silence/silence.go

@@ -370,7 +370,9 @@ func (s *Silences) setSilence(sil *pb.Silence) error {
 	}
 
 	s.st.Merge(st)
-	s.gossip.GossipBroadcast(st)
+	// setSilence() is called with s.mtx locked, which can produce
+	// a deadlock if we call GossipBroadcast from here.
+	go s.gossip.GossipBroadcast(st)
 
 	return nil
 }

silence/silence_test.go

@@ -197,19 +197,24 @@ func TestSilencesSetSilence(t *testing.T) {
 		},
 	}
 
-	var called bool
+	done := make(chan bool)
 	s.gossip = &mockGossip{
 		broadcast: func(d mesh.GossipData) {
 			data, ok := d.(*gossipData)
 			require.True(t, ok, "gossip data of unknown type")
 			require.Equal(t, want, data, "unexpected gossip broadcast data")
-
-			called = true
+			close(done)
 		},
 	}
 	require.NoError(t, s.setSilence(sil))
-	require.True(t, called, "GossipBroadcast was not called")
 	require.Equal(t, want.data, s.st.data, "Unexpected silence state")
+
+	// GossipBroadcast is called in a gorountine.
+	select {
+	case <-done:
+	case <-time.After(1 * time.Second):
+		t.Fatal("GossipBroadcast was not called")
+	}
 }
 
 func TestSilenceSet(t *testing.T) {