livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix for SH-2845, SH-2846, SH-2847, SH-2851: curl crashes and out-of-memory crashes.

meow-7.2.2
Xiaohong Bao 2012-01-12 16:36:56 -07:00
parent 837e38d819
commit f082de03ff
15 changed files with 475 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
indra/llmessage/llcurl.cpp
View File

@ -72,10 +72,12 @@
static const U32 EASY_HANDLE_POOL_SIZE = 5;
static const S32 MULTI_PERFORM_CALL_REPEAT = 5;
static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds
static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation
static const S32 MAX_ACTIVE_REQUEST_COUNT = 100;
static
static const F32 DEFAULT_MULTI_IDLE_TIME = 120.0f ; //seconds
static const S32 MAX_NUM_OF_HANDLES = 256 ; //max number of handles, (multi handles and easy handles combined).
// DEBUG //
S32 gCurlEasyCount = 0;
S32 gCurlMultiCount = 0;
@ -87,6 +89,8 @@ std::vector<LLMutex*> LLCurl::sSSLMutex;
std::string LLCurl::sCAPath;
std::string LLCurl::sCAFile;
LLCurlThread* LLCurl::sCurlThread = NULL ;
LLMutex* LLCurl::sHandleMutexp = NULL ;
S32 LLCurl::sTotalHandles = 0 ;
void check_curl_code(CURLcode code)
{
@ -230,7 +234,7 @@ CURL* LLCurl::Easy::allocEasyHandle()
if (sFreeHandles.empty())
{
ret = curl_easy_init();
ret = LLCurl::newEasyHandle();
}
else
{
@ -250,16 +254,27 @@ CURL* LLCurl::Easy::allocEasyHandle()
//static
void LLCurl::Easy::releaseEasyHandle(CURL* handle)
{
static const S32 MAX_NUM_FREE_HANDLES = 32 ;
if (!handle)
{
llerrs << "handle cannot be NULL!" << llendl;
return ; //handle allocation failed.
//llerrs << "handle cannot be NULL!" << llendl;
}
LLMutexLock lock(sHandleMutexp) ;
if (sActiveHandles.find(handle) != sActiveHandles.end())
{
sActiveHandles.erase(handle);
sFreeHandles.insert(handle);
if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES)
{
sFreeHandles.insert(handle);
}
else
{
LLCurl::deleteEasyHandle(handle) ;
}
}
else
{
@ -517,8 +532,7 @@ void LLCurl::Easy::prepRequest(const std::string& url,
}
////////////////////////////////////////////////////////////////////////////
LLMutex* LLCurl::Multi::sMultiInitMutexp = NULL ;
LLCurl::Multi::Multi()
LLCurl::Multi::Multi(F32 idle_time_out)
: mQueued(0),
mErrorCount(0),
mState(STATE_READY),
@ -527,28 +541,47 @@ LLCurl::Multi::Multi()
mDeletionMutexp(NULL),
mEasyMutexp(NULL)
{
mCurlMultiHandle = initMulti();
mCurlMultiHandle = LLCurl::newMultiHandle();
if (!mCurlMultiHandle)
{
llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl;
mCurlMultiHandle = initMulti();
mCurlMultiHandle = LLCurl::newMultiHandle();
}
llassert_always(mCurlMultiHandle);
if(LLCurl::getCurlThread()->getThreaded())
//llassert_always(mCurlMultiHandle);
if(mCurlMultiHandle)
{
mMutexp = new LLMutex(NULL) ;
mDeletionMutexp = new LLMutex(NULL) ;
mEasyMutexp = new LLMutex(NULL) ;
}
LLCurl::getCurlThread()->addMulti(this) ;
if(LLCurl::getCurlThread()->getThreaded())
{
mMutexp = new LLMutex(NULL) ;
mDeletionMutexp = new LLMutex(NULL) ;
mEasyMutexp = new LLMutex(NULL) ;
}
LLCurl::getCurlThread()->addMulti(this) ;
++gCurlMultiCount;
mIdleTimeOut = idle_time_out ;
if(mIdleTimeOut < DEFAULT_MULTI_IDLE_TIME)
{
mIdleTimeOut = DEFAULT_MULTI_IDLE_TIME ;
}
++gCurlMultiCount;
}
}
LLCurl::Multi::~Multi()
{
cleanup() ;
}
void LLCurl::Multi::cleanup()
{
if(!mCurlMultiHandle)
{
return ; //nothing to clean.
}
// Clean up active
for(easy_active_list_t::iterator iter = mEasyActiveList.begin();
iter != mEasyActiveList.end(); ++iter)
@ -564,7 +597,8 @@ LLCurl::Multi::~Multi()
for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer());
mEasyFreeList.clear();
check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle));
check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle));
mCurlMultiHandle = NULL ;
delete mMutexp ;
mMutexp = NULL ;
@ -572,15 +606,13 @@ LLCurl::Multi::~Multi()
mDeletionMutexp = NULL ;
delete mEasyMutexp ;
mEasyMutexp = NULL ;
mQueued = 0 ;
mState = STATE_COMPLETED;
--gCurlMultiCount;
}
CURLM* LLCurl::Multi::initMulti()
{
LLMutexLock lock(sMultiInitMutexp) ;
return curl_multi_init() ;
return ;
}
void LLCurl::Multi::lock()
@ -604,6 +636,7 @@ void LLCurl::Multi::markDead()
LLMutexLock lock(mDeletionMutexp) ;
mDead = TRUE ;
LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ;
}
void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state)
@ -630,6 +663,11 @@ bool LLCurl::Multi::isCompleted()
bool LLCurl::Multi::waitToComplete()
{
if(!isValid())
{
return true ;
}
if(!mMutexp) //not threaded
{
doPerform() ;
@ -639,7 +677,7 @@ bool LLCurl::Multi::waitToComplete()
bool completed = (STATE_COMPLETED == mState) ;
if(!completed)
{
LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ;
LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ;
}
return completed;
@ -689,7 +727,12 @@ bool LLCurl::Multi::doPerform()
}
mQueued = q;
setState(STATE_COMPLETED) ;
setState(STATE_COMPLETED) ;
mIdleTimer.reset() ;
}
else if(mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it.
{
dead = true ;
}
return dead ;
@ -697,6 +740,11 @@ bool LLCurl::Multi::doPerform()
S32 LLCurl::Multi::process()
{
if(!isValid())
{
return 0 ;
}
waitToComplete() ;
if (getState() != STATE_COMPLETED)
@ -849,7 +897,11 @@ bool LLCurlThread::CurlRequest::processRequest()
if(mMulti)
{
completed = mCurlThread->doMultiPerform(mMulti) ;
setPriority(LLQueuedThread::PRIORITY_LOW) ;
if(!completed)
{
setPriority(LLQueuedThread::PRIORITY_LOW) ;
}
}
return completed ;
@ -857,24 +909,26 @@ bool LLCurlThread::CurlRequest::processRequest()
void LLCurlThread::CurlRequest::finishRequest(bool completed)
{
mCurlThread->deleteMulti(mMulti) ;
if(mMulti->isDead())
{
mCurlThread->deleteMulti(mMulti) ;
}
else
{
mMulti->cleanup() ; //being idle too long, remove the request.
}
mMulti = NULL ;
}
LLCurlThread::LLCurlThread(bool threaded) :
LLQueuedThread("curlthread", threaded)
{
if(!LLCurl::Multi::sMultiInitMutexp)
{
LLCurl::Multi::sMultiInitMutexp = new LLMutex(NULL) ;
}
}
//virtual
LLCurlThread::~LLCurlThread()
{
delete LLCurl::Multi::sMultiInitMutexp ;
LLCurl::Multi::sMultiInitMutexp = NULL ;
}
S32 LLCurlThread::update(F32 max_time_ms)
@ -896,7 +950,19 @@ void LLCurlThread::addMulti(LLCurl::Multi* multi)
void LLCurlThread::killMulti(LLCurl::Multi* multi)
{
multi->markDead() ;
if(!multi)
{
return ;
}
if(multi->isValid())
{
multi->markDead() ;
}
else
{
deleteMulti(multi) ;
}
}
//private
@ -942,7 +1008,14 @@ LLCurlRequest::~LLCurlRequest()
void LLCurlRequest::addMulti()
{
LLCurl::Multi* multi = new LLCurl::Multi();
if(!multi->isValid())
{
LLCurl::getCurlThread()->killMulti(multi) ;
mActiveMulti = NULL ;
mActiveRequestCount = 0 ;
return;
}
mMultiSet.insert(multi);
mActiveMulti = multi;
mActiveRequestCount = 0;
@ -1066,6 +1139,19 @@ S32 LLCurlRequest::process()
{
curlmulti_set_t::iterator curiter = iter++;
LLCurl::Multi* multi = *curiter;
if(!multi->isValid())
{
if(multi == mActiveMulti)
{
mActiveMulti = NULL ;
mActiveRequestCount = 0 ;
}
mMultiSet.erase(curiter) ;
LLCurl::getCurlThread()->killMulti(multi) ;
continue ;
}
S32 tres = multi->process();
res += tres;
if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0)
@ -1086,6 +1172,19 @@ S32 LLCurlRequest::getQueued()
{
curlmulti_set_t::iterator curiter = iter++;
LLCurl::Multi* multi = *curiter;
if(!multi->isValid())
{
if(multi == mActiveMulti)
{
mActiveMulti = NULL ;
mActiveRequestCount = 0 ;
}
LLCurl::getCurlThread()->killMulti(multi);
mMultiSet.erase(curiter) ;
continue ;
}
queued += multi->mQueued;
if (multi->getState() != LLCurl::Multi::STATE_READY)
{
@ -1105,13 +1204,22 @@ LLCurlEasyRequest::LLCurlEasyRequest()
{
mMulti = new LLCurl::Multi();
mEasy = mMulti->allocEasy();
if (mEasy)
if(mMulti->isValid())
{
mEasy->setErrorBuffer();
mEasy->setCA();
// Set proxy settings if configured to do so.
LLProxy::getInstance()->applyProxySettings(mEasy);
mEasy = mMulti->allocEasy();
if (mEasy)
{
mEasy->setErrorBuffer();
mEasy->setCA();
// Set proxy settings if configured to do so.
LLProxy::getInstance()->applyProxySettings(mEasy);
}
}
else
{
LLCurl::getCurlThread()->killMulti(mMulti) ;
mEasy = NULL ;
mMulti = NULL ;
}
}
@ -1122,7 +1230,7 @@ LLCurlEasyRequest::~LLCurlEasyRequest()
void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(option, value);
}
@ -1130,7 +1238,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value)
void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setoptString(option, value);
}
@ -1138,7 +1246,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value
void LLCurlEasyRequest::setPost(char* postdata, S32 size)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_POST, 1);
mEasy->setopt(CURLOPT_POSTFIELDS, postdata);
@ -1148,7 +1256,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size)
void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER
@ -1157,7 +1265,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u
void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_WRITEDATA, userdata);
@ -1166,7 +1274,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use
void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_READDATA, userdata);
@ -1175,7 +1283,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd
void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback);
mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata);
@ -1184,7 +1292,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void*
void LLCurlEasyRequest::slist_append(const char* str)
{
if (mEasy)
if (isValid() && mEasy)
{
mEasy->slist_append(str);
}
@ -1195,7 +1303,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url)
llassert_always(!mRequestSent);
mRequestSent = true;
lldebugs << url << llendl;
if (mEasy)
if (isValid() && mEasy)
{
mEasy->setHeaders();
mEasy->setoptString(CURLOPT_URL, url);
@ -1207,7 +1315,7 @@ void LLCurlEasyRequest::requestComplete()
{
llassert_always(mRequestSent);
mRequestSent = false;
if (mEasy)
if (isValid() && mEasy)
{
mMulti->removeEasy(mEasy);
}
@ -1216,6 +1324,10 @@ void LLCurlEasyRequest::requestComplete()
// Usage: Call getRestult until it returns false (no more messages)
bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info)
{
if(!isValid())
{
return false ;
}
if (!mMulti->isCompleted())
{ //we're busy, try again later
return false;
@ -1280,7 +1392,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info)
std::string LLCurlEasyRequest::getErrorString()
{
return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string();
}
////////////////////////////////////////////////////////////////////////////
@ -1328,6 +1440,7 @@ void LLCurl::initClass(bool multi_threaded)
sCurlThread = new LLCurlThread(multi_threaded) ;
if(multi_threaded)
{
sHandleMutexp = new LLMutex(NULL) ;
Easy::sHandleMutexp = new LLMutex(NULL) ;
}
}
@ -1354,7 +1467,7 @@ void LLCurl::cleanupClass()
for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter)
{
CURL* curl = *iter;
curl_easy_cleanup(curl);
LLCurl::deleteEasyHandle(curl);
}
Easy::sFreeHandles.clear();
@ -1362,9 +1475,77 @@ void LLCurl::cleanupClass()
delete Easy::sHandleMutexp ;
Easy::sHandleMutexp = NULL ;
delete sHandleMutexp ;
sHandleMutexp = NULL ;
llassert(Easy::sActiveHandles.empty());
}
//static
CURLM* LLCurl::newMultiHandle()
{
LLMutexLock lock(sHandleMutexp) ;
if(sTotalHandles + 1 > MAX_NUM_OF_HANDLES)
{
llwarns << "no more handles available." << llendl ;
return NULL ; //failed
}
sTotalHandles++;
CURLM* ret = curl_multi_init() ;
if(!ret)
{
llwarns << "curl_multi_init failed." << llendl ;
}
return ret ;
}
//static
CURLMcode LLCurl::deleteMultiHandle(CURLM* handle)
{
if(handle)
{
LLMutexLock lock(sHandleMutexp) ;
sTotalHandles-- ;
return curl_multi_cleanup(handle) ;
}
return CURLM_OK ;
}
//static
CURL* LLCurl::newEasyHandle()
{
LLMutexLock lock(sHandleMutexp) ;
if(sTotalHandles + 1 > MAX_NUM_OF_HANDLES)
{
llwarns << "no more handles available." << llendl ;
return NULL ; //failed
}
sTotalHandles++;
CURL* ret = curl_easy_init() ;
if(!ret)
{
llwarns << "curl_easy_init failed." << llendl ;
}
return ret ;
}
//static
void LLCurl::deleteEasyHandle(CURL* handle)
{
if(handle)
{
LLMutexLock lock(sHandleMutexp) ;
curl_easy_cleanup(handle) ;
sTotalHandles-- ;
}
}
const unsigned int LLCurl::MAX_REDIRECTS = 5;
// Provide access to LLCurl free functions outside of llcurl.cpp without polluting the global namespace.

22
indra/llmessage/llcurl.h
View File

@ -43,6 +43,7 @@
#include "llsd.h"
#include "llthread.h"
#include "llqueuedthread.h"
#include "llframetimer.h"
class LLMutex;
class LLCurlThread;
@ -182,11 +183,20 @@ public:
static unsigned long ssl_thread_id(void);
static LLCurlThread* getCurlThread() { return sCurlThread ;}
static CURLM* newMultiHandle() ;
static CURLMcode deleteMultiHandle(CURLM* handle) ;
static CURL* newEasyHandle() ;
static void deleteEasyHandle(CURL* handle) ;
private:
static std::string sCAPath;
static std::string sCAFile;
static const unsigned int MAX_REDIRECTS;
static LLCurlThread* sCurlThread;
static LLMutex* sHandleMutexp ;
static S32 sTotalHandles ;
};
class LLCurl::Easy
@ -277,7 +287,7 @@ public:
STATE_COMPLETED=2
} ePerformState;
Multi();
Multi(F32 idle_time_out = 0.f);
LLCurl::Easy* allocEasy();
bool addEasy(LLCurl::Easy* easy);
@ -288,7 +298,10 @@ public:
void setState(ePerformState state) ;
ePerformState getState() ;
bool isCompleted() ;
bool isValid() {return mCurlMultiHandle != NULL ;}
bool isDead() {return mDead;}
bool waitToComplete() ;
@ -299,9 +312,9 @@ public:
S32 mQueued;
S32 mErrorCount;
static CURLM* initMulti() ;
private:
void easyFree(LLCurl::Easy*);
void cleanup() ;
CURLM* mCurlMultiHandle;
@ -319,8 +332,8 @@ private:
LLMutex* mMutexp ;
LLMutex* mDeletionMutexp ;
LLMutex* mEasyMutexp ;
static LLMutex* sMultiInitMutexp ;
LLFrameTimer mIdleTimer ;
F32 mIdleTimeOut;
};
class LLCurlThread : public LLQueuedThread
@ -414,6 +427,7 @@ public:
std::string getErrorString();
bool isCompleted() {return mMulti->isCompleted() ;}
bool wait() { return mMulti->waitToComplete(); }
bool isValid() {return mMulti && mMulti->isValid(); }
LLCurl::Easy* getEasy() const { return mEasy; }

10
indra/llmessage/llhttpassetstorage.cpp
View File

@ -232,7 +232,8 @@ LLSD LLHTTPAssetRequest::getFullDetails() const
void LLHTTPAssetRequest::setupCurlHandle()
{
// *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC
mCurlHandle = curl_easy_init();
mCurlHandle = LLCurl::newEasyHandle();
llassert_always(mCurlHandle != NULL) ;
// Apply proxy settings if configured to do so
LLProxy::getInstance()->applyProxySettings(mCurlHandle);
@ -278,7 +279,7 @@ void LLHTTPAssetRequest::setupCurlHandle()
void LLHTTPAssetRequest::cleanupCurlHandle()
{
curl_easy_cleanup(mCurlHandle);
LLCurl::deleteEasyHandle(mCurlHandle);
if (mAssetStoragep)
{
// Terminating a request. Thus upload or download is no longer pending.
@ -429,12 +430,13 @@ void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& l
// curl_global_init moved to LLCurl::initClass()
mCurlMultiHandle = curl_multi_init();
mCurlMultiHandle = LLCurl::newMultiHandle() ;
llassert_always(mCurlMultiHandle != NULL) ;
}
LLHTTPAssetStorage::~LLHTTPAssetStorage()
{
curl_multi_cleanup(mCurlMultiHandle);
LLCurl::deleteMultiHandle(mCurlMultiHandle);
mCurlMultiHandle = NULL;
// curl_global_cleanup moved to LLCurl::initClass()

12
indra/llmessage/llhttpclient.cpp
View File

@ -228,6 +228,12 @@ static void request(
LLPumpIO::chain_t chain;
LLURLRequest* req = new LLURLRequest(method, url);
if(!req->isValid())//failed
{
delete req ;
return ;
}
req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req);
@ -423,7 +429,9 @@ static LLSD blocking_request(
{
lldebugs << "blockingRequest of " << url << llendl;
char curl_error_buffer[CURL_ERROR_SIZE] = "\0";
CURL* curlp = curl_easy_init();
CURL* curlp = LLCurl::newEasyHandle();
llassert_always(curlp != NULL) ;
LLHTTPBuffer http_buffer;
std::string body_str;
@ -517,7 +525,7 @@ static LLSD blocking_request(
}
// * Cleanup
curl_easy_cleanup(curlp);
LLCurl::deleteEasyHandle(curlp);
return response;
}

6
indra/llmessage/lliopipe.cpp
View File

@ -75,6 +75,12 @@ LLIOPipe::~LLIOPipe()
//lldebugs << "destroying LLIOPipe" << llendl;
}
//virtual
bool LLIOPipe::isValid()
{
return true ;
}
// static
std::string LLIOPipe::lookupStatusString(EStatus status)
{

2
indra/llmessage/lliopipe.h
View File

@ -231,6 +231,8 @@ public:
*/
virtual ~LLIOPipe();
virtual bool isValid() ;
protected:
/**
* @brief Base Constructor.

50
indra/llmessage/llpumpio.cpp
View File

@ -195,7 +195,7 @@ bool LLPumpIO::prime(apr_pool_t* pool)
return ((pool == NULL) ? false : true);
}
bool LLPumpIO::addChain(const chain_t& chain, F32 timeout)
bool LLPumpIO::addChain(const chain_t& chain, F32 timeout, bool has_curl_request)
{
LLMemType m1(LLMemType::MTYPE_IO_PUMP);
if(chain.empty()) return false;
@ -204,6 +204,7 @@ bool LLPumpIO::addChain(const chain_t& chain, F32 timeout)
LLScopedLock lock(mChainsMutex);
#endif
LLChainInfo info;
info.mHasCurlRequest = has_curl_request;
info.setTimeoutSeconds(timeout);
info.mData = LLIOPipe::buffer_ptr_t(new LLBufferArray);
LLLinkInfo link;
@ -440,6 +441,15 @@ void LLPumpIO::pump()
static LLFastTimer::DeclareTimer FTM_PUMP_IO("Pump IO");
LLPumpIO::current_chain_t LLPumpIO::removeRunningChain(LLPumpIO::current_chain_t& run_chain)
{
std::for_each(
(*run_chain).mDescriptors.begin(),
(*run_chain).mDescriptors.end(),
ll_delete_apr_pollset_fd_client_data());
return mRunningChains.erase(run_chain);
}
//timeout is in microseconds
void LLPumpIO::pump(const S32& poll_timeout)
{
@ -585,10 +595,16 @@ void LLPumpIO::pump(const S32& poll_timeout)
// << (*run_chain).mChainLinks[0].mPipe
// << " because we reached the end." << llendl;
#endif
run_chain = mRunningChains.erase(run_chain);
run_chain = removeRunningChain(run_chain);
continue;
}
}
else if(isChainExpired(*run_chain))
{
run_chain = removeRunningChain(run_chain);
continue;
}
PUMP_DEBUG;
if((*run_chain).mLock)
{
@ -696,11 +712,7 @@ void LLPumpIO::pump(const S32& poll_timeout)
PUMP_DEBUG;
// This chain is done. Clean up any allocated memory and
// erase the chain info.
std::for_each(
(*run_chain).mDescriptors.begin(),
(*run_chain).mDescriptors.end(),
ll_delete_apr_pollset_fd_client_data());
run_chain = mRunningChains.erase(run_chain);
run_chain = removeRunningChain(run_chain);
// *NOTE: may not always need to rebuild the pollset.
mRebuildPollset = true;
@ -1095,6 +1107,24 @@ void LLPumpIO::processChain(LLChainInfo& chain)
PUMP_DEBUG;
}
bool LLPumpIO::isChainExpired(LLChainInfo& chain)
{
if(!chain.mHasCurlRequest)
{
return false ;
}
for(links_t::iterator iter = chain.mChainLinks.begin(); iter != chain.mChainLinks.end(); ++iter)
{
if(!(*iter).mPipe->isValid())
{
return true ;
}
}
return false ;
}
bool LLPumpIO::handleChainError(
LLChainInfo& chain,
LLIOPipe::EStatus error)
@ -1136,6 +1166,9 @@ bool LLPumpIO::handleChainError(
#endif
keep_going = false;
break;
case LLIOPipe::STATUS_EXPIRED:
keep_going = false;
break ;
default:
if(LLIOPipe::isSuccess(error))
{
@ -1157,7 +1190,8 @@ bool LLPumpIO::handleChainError(
LLPumpIO::LLChainInfo::LLChainInfo() :
mInit(false),
mLock(0),
mEOS(false)
mEOS(false),
mHasCurlRequest(false)
{
LLMemType m1(LLMemType::MTYPE_IO_PUMP);
mTimer.setTimerExpirySec(DEFAULT_CHAIN_EXPIRY_SECS);

12
indra/llmessage/llpumpio.h
View File

@ -113,7 +113,7 @@ public:
* expire. Pass in 0.0f to never expire.
* @return Returns true if anything was added to the pump.
*/
bool addChain(const chain_t& chain, F32 timeout);
bool addChain(const chain_t& chain, F32 timeout, bool has_curl_request = false);
/**
* @brief Struct to associate a pipe with it's buffer io indexes.
@ -356,12 +356,13 @@ protected:
// basic member data
bool mInit;
bool mEOS;
bool mHasCurlRequest;
S32 mLock;
LLFrameTimer mTimer;
links_t::iterator mHead;
links_t mChainLinks;
LLIOPipe::buffer_ptr_t mData;
bool mEOS;
LLIOPipe::buffer_ptr_t mData;
LLSD mContext;
// tracking inside the pump
@ -402,7 +403,7 @@ protected:
protected:
void initialize(apr_pool_t* pool);
void cleanup();
current_chain_t removeRunningChain(current_chain_t& chain) ;
/**
* @brief Given the internal state of the chains, rebuild the pollset
* @see setConditional()
@ -429,6 +430,9 @@ protected:
*/
bool handleChainError(LLChainInfo& chain, LLIOPipe::EStatus error);
//if the chain is expired, remove it
bool isChainExpired(LLChainInfo& chain) ;
public:
/**
* @brief Return number of running chains.

3
indra/llmessage/llsdrpcclient.h
View File

@ -239,6 +239,8 @@ public:
LLSDRPCClientFactory(const std::string& fixed_url) : mURL(fixed_url) {}
virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const
{
llerrs << "Can not call this." << llendl ;
lldebugs << "LLSDRPCClientFactory::build" << llendl;
LLIOPipe::ptr_t service(new Client);
chain.push_back(service);
@ -282,6 +284,7 @@ public:
LLXMLSDRPCClientFactory(const std::string& fixed_url) : mURL(fixed_url) {}
virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const
{
llerrs << "who calls this?" << llendl ;
lldebugs << "LLXMLSDRPCClientFactory::build" << llendl;
LLIOPipe::ptr_t service(new Client);
chain.push_back(service);

18
indra/llmessage/llurlrequest.cpp
View File

@ -83,6 +83,12 @@ LLURLRequestDetail::LLURLRequestDetail() :
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
mCurlRequest = new LLCurlEasyRequest();
if(!mCurlRequest->isValid()) //failed.
{
delete mCurlRequest ;
mCurlRequest = NULL ;
}
}
LLURLRequestDetail::~LLURLRequestDetail()
@ -250,12 +256,24 @@ void LLURLRequest::allowCookies()
mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, "");
}
//virtual
bool LLURLRequest::isValid()
{
return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid();
}
// virtual
LLIOPipe::EStatus LLURLRequest::handleError(
LLIOPipe::EStatus status,
LLPumpIO* pump)
{
LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST);
if(!isValid())
{
return STATUS_EXPIRED ;
}
if(mCompletionCallback && pump)
{
LLURLRequestComplete* complete = NULL;

2
indra/llmessage/llurlrequest.h
View File

@ -188,6 +188,8 @@ public:
*/
void allowCookies();
/*virtual*/ bool isValid() ;
public:
/**
* @brief Give this pipe a chance to handle a generated error

131
indra/newview/llmeshrepository.cpp
View File

@ -498,9 +498,11 @@ void LLMeshRepoThread::run()
LODRequest req = mLODReqQ.front();
mLODReqQ.pop();
mMutex->unlock();
if (fetchMeshLOD(req.mMeshParams, req.mLOD))
if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit
{
count++;
mMutex->lock();
mLODReqQ.push(req) ;
mMutex->unlock();
}
}
}
@ -512,9 +514,11 @@ void LLMeshRepoThread::run()
HeaderRequest req = mHeaderReqQ.front();
mHeaderReqQ.pop();
mMutex->unlock();
if (fetchMeshHeader(req.mMeshParams))
if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit
{
count++;
mMutex->lock();
mHeaderReqQ.push(req) ;
mMutex->unlock();
}
}
}
@ -658,6 +662,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)
return false;
}
bool ret = true ;
U32 header_size = mMeshHeaderSize[mesh_id];
if (header_size > 0)
@ -673,7 +678,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)
//check VFS for mesh skin info
LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH);
if (file.getSize() >= offset+size)
{
{
LLMeshRepository::sCacheBytesRead += size;
file.seek(offset);
U8* buffer = new U8[size];
@ -689,7 +694,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)
if (!zero)
{ //attempt to parse
if (skinInfoReceived(mesh_id, buffer, size))
{
{
delete[] buffer;
return true;
}
@ -704,11 +709,14 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)
std::string http_url = constructUrl(mesh_id);
if (!http_url.empty())
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size,
{
ret = mCurlRequest->getByteRange(http_url, headers, offset, size,
new LLMeshSkinInfoResponder(mesh_id, offset, size));
if(ret)
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
}
}
}
}
@ -718,7 +726,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)
}
//early out was not hit, effectively fetched
return true;
return ret;
}
bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)
@ -732,7 +740,8 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)
}
U32 header_size = mMeshHeaderSize[mesh_id];
bool ret = true ;
if (header_size > 0)
{
S32 version = mMeshHeader[mesh_id]["version"].asInteger();
@ -748,6 +757,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)
if (file.getSize() >= offset+size)
{
LLMeshRepository::sCacheBytesRead += size;
file.seek(offset);
U8* buffer = new U8[size];
file.read(buffer, size);
@ -777,11 +787,14 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)
std::string http_url = constructUrl(mesh_id);
if (!http_url.empty())
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
mCurlRequest->getByteRange(http_url, headers, offset, size,
{
ret = mCurlRequest->getByteRange(http_url, headers, offset, size,
new LLMeshDecompositionResponder(mesh_id, offset, size));
if(ret)
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
}
}
}
}
@ -791,7 +804,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)
}
//early out was not hit, effectively fetched
return true;
return ret;
}
bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)
@ -805,6 +818,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)
}
U32 header_size = mMeshHeaderSize[mesh_id];
bool ret = true ;
if (header_size > 0)
{
@ -850,11 +864,15 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)
std::string http_url = constructUrl(mesh_id);
if (!http_url.empty())
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
mCurlRequest->getByteRange(http_url, headers, offset, size,
{
ret = mCurlRequest->getByteRange(http_url, headers, offset, size,
new LLMeshPhysicsShapeResponder(mesh_id, offset, size));
if(ret)
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
}
}
}
else
@ -868,13 +886,12 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)
}
//early out was not hit, effectively fetched
return true;
return ret;
}
bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params)
//return false if failed to get header
bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count)
{
bool retval = false;
{
//look for mesh in asset in vfs
LLVFile file(gVFS, mesh_params.getSculptID(), LLAssetType::AT_MESH);
@ -889,36 +906,40 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params)
file.read(buffer, bytes);
if (headerReceived(mesh_params, buffer, bytes))
{ //did not do an HTTP request, return false
return false;
return true;
}
}
}
//either cache entry doesn't exist or is corrupt, request header from simulator
//either cache entry doesn't exist or is corrupt, request header from simulator
bool retval = true ;
std::vector<std::string> headers;
headers.push_back("Accept: application/octet-stream");
std::string http_url = constructUrl(mesh_params.getSculptID());
if (!http_url.empty())
{
++sActiveHeaderRequests;
retval = true;
//grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits
//within the first 4KB
//NOTE -- this will break of headers ever exceed 4KB
LLMeshRepository::sHTTPRequestCount++;
mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params));
//NOTE -- this will break of headers ever exceed 4KB
retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params));
if(retval)
{
++sActiveHeaderRequests;
LLMeshRepository::sHTTPRequestCount++;
}
count++;
}
return retval;
}
bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod)
//return false if failed to get mesh lod.
bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count)
{ //protected by mMutex
mHeaderMutex->lock();
bool retval = false;
bool retval = true;
LLUUID mesh_id = mesh_params.getSculptID();
@ -955,7 +976,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod)
if (lodReceived(mesh_params, lod, buffer, size))
{
delete[] buffer;
return false;
return true;
}
}
@ -968,12 +989,16 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod)
std::string http_url = constructUrl(mesh_id);
if (!http_url.empty())
{
++sActiveLODRequests;
retval = true;
LLMeshRepository::sHTTPRequestCount++;
mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size,
{
retval = mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size,
new LLMeshLODResponder(mesh_params, lod, offset, size));
if(retval)
{
++sActiveLODRequests;
LLMeshRepository::sHTTPRequestCount++;
}
count++;
}
else
{
@ -1540,8 +1565,17 @@ void LLMeshUploadThread::doWholeModelUpload()
LLSD body = full_model_data["asset_resources"];
dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num));
LLCurlRequest::headers_t headers;
mCurlRequest->post(mWholeModelUploadURL, headers, body,
new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle), mMeshUploadTimeOut);
{
LLCurl::ResponderPtr responder = new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle) ;
while(!mCurlRequest->post(mWholeModelUploadURL, headers, body, responder, mMeshUploadTimeOut))
{
//sleep for 10ms to prevent eating a whole core
apr_sleep(10000);
}
}
do
{
mCurlRequest->process();
@ -1571,8 +1605,15 @@ void LLMeshUploadThread::requestWholeModelFee()
mPendingUploads++;
LLCurlRequest::headers_t headers;
mCurlRequest->post(mWholeModelFeeCapability, headers, model_data,
new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle), mMeshUploadTimeOut);
{
LLCurl::ResponderPtr responder = new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle) ;
while(!mCurlRequest->post(mWholeModelFeeCapability, headers, model_data, responder, mMeshUploadTimeOut))
{
//sleep for 10ms to prevent eating a whole core
apr_sleep(10000);
}
}
do
{

4
indra/newview/llmeshrepository.h
View File

@ -323,8 +323,8 @@ public:
virtual void run();
void loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod);
bool fetchMeshHeader(const LLVolumeParams& mesh_params);
bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod);
bool fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count);
bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count);
bool headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size);
bool lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size);
bool skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size);

19
indra/newview/llxmlrpctransaction.cpp
View File

@ -305,6 +305,15 @@ void LLXMLRPCTransaction::Impl::init(XMLRPC_REQUEST request, bool useGzip)
{
mCurlRequest = new LLCurlEasyRequest();
}
if(!mCurlRequest->isValid())
{
llwarns << "mCurlRequest is invalid." << llendl ;
delete mCurlRequest ;
mCurlRequest = NULL ;
return ;
}
mErrorCert = NULL;
// mCurlRequest->setopt(CURLOPT_VERBOSE, 1); // useful for debugging
@ -357,10 +366,20 @@ LLXMLRPCTransaction::Impl::~Impl()
}
delete mCurlRequest;
mCurlRequest = NULL ;
}
bool LLXMLRPCTransaction::Impl::process()
{
if(!mCurlRequest || !mCurlRequest->isValid())
{
llwarns << "transaction failed." << llendl ;
delete mCurlRequest ;
mCurlRequest = NULL ;
return true ; //failed, quit.
}
switch(mStatus)
{
case LLXMLRPCTransaction::StatusComplete:

23
indra/viewer_components/updater/llupdatedownloader.cpp
View File

@ -39,7 +39,7 @@
#include "llsdserialize.h"
#include "llthread.h"
#include "llupdaterservice.h"
#include "llcurl.h"
class LLUpdateDownloader::Implementation:
public LLThread
@ -198,13 +198,19 @@ LLUpdateDownloader::Implementation::Implementation(LLUpdateDownloader::Client &
LLUpdateDownloader::Implementation::~Implementation()
{
if(isDownloading()) {
if(isDownloading())
{
cancel();
shutdown();
} else {
}
else
{
; // No op.
}
if(mCurl) curl_easy_cleanup(mCurl);
if(mCurl)
{
LLCurl::deleteEasyHandle(mCurl);
}
}
@ -406,9 +412,12 @@ void LLUpdateDownloader::Implementation::run(void)
void LLUpdateDownloader::Implementation::initializeCurlGet(std::string const & url, bool processHeader)
{
if(mCurl == 0) {
mCurl = curl_easy_init();
} else {
if(mCurl == 0)
{
mCurl = LLCurl::newEasyHandle();
}
else
{
curl_easy_reset(mCurl);
}