SL-17434 Crash clearing LLEventPumps

We actively use event pumps's connections in threads, make sure nothing
modifies list of connections during reset.
And in case this doesn't fix the issue list affected pump before it
crashes to have a better idea of what is going on.

indra/llcommon/llevents.cpp

@@ -152,12 +152,21 @@ void LLEventPumps::clear()
     }
 }
 
-void LLEventPumps::reset()
+void LLEventPumps::reset(bool log_pumps)
 {
     // Reset every known LLEventPump instance. Leave it up to each instance to
     // decide what to do with the reset() call.
+    if (log_pumps)
+    {
+        LL_INFOS() << "Resetting " << (S32)mPumpMap.size() << " pumps" << LL_ENDL;
+    }
+
     for (PumpMap::value_type& pair : mPumpMap)
     {
+        if (log_pumps)
+        {
+            LL_INFOS() << "Resetting pump " << pair.first << LL_ENDL;
+        }
         pair.second->reset();
     }
 }
@@ -314,9 +323,11 @@ std::string LLEventPump::inventName(const std::string& pfx)
 
 void LLEventPump::clear()
 {
+    LLMutexLock lock(&mConnectionListMutex);
     // Destroy the original LLStandardSignal instance, replacing it with a
     // whole new one.
     mSignal = std::make_shared<LLStandardSignal>();
+
     mConnections.clear();
 }
 
@@ -324,6 +335,7 @@ void LLEventPump::reset()
 {
     // Resetting mSignal is supposed to disconnect everything on its own
     // But due to crash on 'reset' added explicit cleanup to get more data
+    LLMutexLock lock(&mConnectionListMutex);
     ConnectionMap::const_iterator iter = mConnections.begin();
     ConnectionMap::const_iterator end = mConnections.end();
     while (iter!=end)
@@ -348,6 +360,8 @@ LLBoundListener LLEventPump::listen_impl(const std::string& name, const LLEventL
         return LLBoundListener();
     }
 
+    LLMutexLock lock(&mConnectionListMutex);
+
     float nodePosition = 1.0;
 
     // if the supplied name is empty we are not interested in the ordering mechanism 
@@ -507,8 +521,9 @@ LLBoundListener LLEventPump::listen_impl(const std::string& name, const LLEventL
     return bound;
 }
 
-LLBoundListener LLEventPump::getListener(const std::string& name) const
+LLBoundListener LLEventPump::getListener(const std::string& name)
 {
+    LLMutexLock lock(&mConnectionListMutex);
     ConnectionMap::const_iterator found = mConnections.find(name);
     if (found != mConnections.end())
     {
@@ -520,6 +535,7 @@ LLBoundListener LLEventPump::getListener(const std::string& name) const
 
 void LLEventPump::stopListening(const std::string& name)
 {
+    LLMutexLock lock(&mConnectionListMutex);
     ConnectionMap::iterator found = mConnections.find(name);
     if (found != mConnections.end())
     {

indra/llcommon/llevents.h

@@ -293,7 +293,7 @@ public:
      * Reset all known LLEventPump instances
      * workaround for DEV-35406 crash on shutdown
      */
-    void reset();
+    void reset(bool log_pumps = false);
 
 private:
     friend class LLEventPump;
@@ -519,7 +519,7 @@ public:
 
     /// Get the LLBoundListener associated with the passed name (dummy
     /// LLBoundListener if not found)
-    virtual LLBoundListener getListener(const std::string& name) const;
+    virtual LLBoundListener getListener(const std::string& name);
     /**
      * Instantiate one of these to block an existing connection:
      * @code
@@ -562,6 +562,7 @@ private:
     LLHandle<LLEventPumps> mRegistry;
 
     std::string mName;
+    LLMutex mConnectionListMutex;
 
 protected:
     virtual LLBoundListener listen_impl(const std::string& name, const LLEventListener&,

indra/newview/llappviewer.cpp

@@ -1736,7 +1736,7 @@ bool LLAppViewer::cleanup()
     LLNotifications::instance().clear();
 
 	// workaround for DEV-35406 crash on shutdown
-	LLEventPumps::instance().reset();
+	LLEventPumps::instance().reset(true);
 
 	//dump scene loading monitor results
 	if (LLSceneMonitor::instanceExists())