livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAINT-7634: Move StatsAccumulator into llcommon, collect data sent and error codes from core.

master
Rider Linden 2017-07-28 14:07:25 -07:00
parent b46697fcc1
commit 57d5744f2c
14 changed files with 370 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
indra/llcommon/CMakeLists.txt
View File

@ -226,6 +226,7 @@ set(llcommon_HEADER_FILES
llstring.h
llstringtable.h
llstaticstringtable.h
llstatsaccumulator.h
llsys.h
llthread.h
llthreadlocalstorage.h

120
indra/llcommon/llstatsaccumulator.h Normal file
View File

@ -0,0 +1,120 @@
/**
* @file llstatsaccumulator.h
* @brief Class for accumulating statistics.
*
* $LicenseInfo:firstyear=2002&license=viewerlgpl$
* Second Life Viewer Source Code
* Copyright (C) 2010, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License only.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
* $/LicenseInfo$
*/
#ifndef LL_STATS_ACCUMULATOR_H
#define LL_STATS_ACCUMULATOR_H
#include "llsd.h"
class LLStatsAccumulator
{
public:
inline LLStatsAccumulator()
{
reset();
}
inline void push(F32 val)
{
if (mCountOfNextUpdatesToIgnore > 0)
{
mCountOfNextUpdatesToIgnore--;
return;
}
mCount++;
mSum += val;
mSumOfSquares += val * val;
if (mCount == 1 || val > mMaxValue)
{
mMaxValue = val;
}
if (mCount == 1 || val < mMinValue)
{
mMinValue = val;
}
}
inline F32 getSum() const
{
return mSum;
}
inline F32 getMean() const
{
return (mCount == 0) ? 0.f : ((F32)mSum) / mCount;
}
inline F32 getMinValue() const
{
return mMinValue;
}
inline F32 getMaxValue() const
{
return mMaxValue;
}
inline F32 getStdDev() const
{
const F32 mean = getMean();
return (mCount < 2) ? 0.f : sqrt(llmax(0.f, mSumOfSquares / mCount - (mean * mean)));
}
inline U32 getCount() const
{
return mCount;
}
inline void reset()
{
mCount = 0;
mSum = mSumOfSquares = 0.f;
mMinValue = 0.0f;
mMaxValue = 0.0f;
mCountOfNextUpdatesToIgnore = 0;
}
inline LLSD asLLSD() const
{
LLSD data;
data["mean"] = getMean();
data["std_dev"] = getStdDev();
data["count"] = (S32)mCount;
data["min"] = getMinValue();
data["max"] = getMaxValue();
return data;
}
private:
S32 mCount;
F32 mSum;
F32 mSumOfSquares;
F32 mMinValue;
F32 mMaxValue;
U32 mCountOfNextUpdatesToIgnore;
};
#endif

2
indra/llcorehttp/CMakeLists.txt
View File

@ -30,6 +30,7 @@ set(llcorehttp_SOURCE_FILES
httpoptions.cpp
httprequest.cpp
httpresponse.cpp
httpstats.cpp
_httplibcurl.cpp
_httpopcancel.cpp
_httpoperation.cpp
@ -57,6 +58,7 @@ set(llcorehttp_HEADER_FILES
httpoptions.h
httprequest.h
httpresponse.h
httpstats.h
_httpinternal.h
_httplibcurl.h
_httpopcancel.h

6
indra/llcorehttp/_httpoprequest.cpp
View File

@ -47,6 +47,8 @@
#include "llhttpconstants.h"
#include "llproxy.h"
#include "httpstats.h"
// *DEBUG: "[curl:bugs] #1420" problem and testing.
//
// A pipelining problem, https://sourceforge.net/p/curl/bugs/1420/,
@ -805,6 +807,7 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void
}
const size_t req_size(size * nmemb);
const size_t write_size(op->mReplyBody->append(static_cast<char *>(data), req_size));
HTTPStats::instance().recordDataDown(write_size);
return write_size;
}
@ -833,7 +836,8 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void
const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos));
const size_t read_size(op->mReqBody->read(op->mCurlBodyPos, static_cast<char *>(data), do_size));
op->mCurlBodyPos += read_size;
HTTPStats::instance().recordDataUp(read_size);
op->mCurlBodyPos += read_size;
return read_size;
}

3
indra/llcorehttp/_httppolicy.cpp
View File

@ -34,6 +34,7 @@
#include "_httppolicyclass.h"
#include "lltimer.h"
#include "httpstats.h"
namespace
{
@ -448,6 +449,8 @@ bool HttpPolicy::stageAfterCompletion(const HttpOpRequest::ptr_t &op)
}
op->stageFromActive(mService);
HTTPStats::instance().recordResultCode(op->mStatus.getStatus());
return false; // not active
}

49
indra/llcorehttp/_httpservice.cpp
View File

@ -38,7 +38,8 @@
#include "lltimer.h"
#include "llthread.h"
#include "llexception.h"
#include "llmemory.h"
namespace
{
@ -293,22 +294,42 @@ void HttpService::threadRun(LLCoreInt::HttpThread * thread)
ELoopSpeed loop(REQUEST_SLEEP);
while (! mExitRequested)
{
loop = processRequestQueue(loop);
try
{
loop = processRequestQueue(loop);
// Process ready queue issuing new requests as needed
ELoopSpeed new_loop = mPolicy->processReadyQueue();
loop = (std::min)(loop, new_loop);
// Process ready queue issuing new requests as needed
ELoopSpeed new_loop = mPolicy->processReadyQueue();
loop = (std::min)(loop, new_loop);
// Give libcurl some cycles
new_loop = mTransport->processTransport();
loop = (std::min)(loop, new_loop);
// Give libcurl some cycles
new_loop = mTransport->processTransport();
loop = (std::min)(loop, new_loop);
// Determine whether to spin, sleep briefly or sleep for next request
if (REQUEST_SLEEP != loop)
{
ms_sleep(HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS);
}
}
// Determine whether to spin, sleep briefly or sleep for next request
if (REQUEST_SLEEP != loop)
{
ms_sleep(HTTP_SERVICE_LOOP_SLEEP_NORMAL_MS);
}
}
catch (const LLContinueError&)
{
LOG_UNHANDLED_EXCEPTION("");
}
catch (std::bad_alloc)
{
LLMemory::logMemoryInfo(TRUE);
//output possible call stacks to log file.
LLError::LLCallStacks::print();
LL_ERRS() << "Bad memory allocation in HttpService::threadRun()!" << LL_ENDL;
}
catch (...)
{
CRASH_ON_UNHANDLED_EXCEPTION("");
}
}
shutdown();
sState = STOPPED;

22
indra/llcorehttp/bufferarray.cpp
View File

@ -25,6 +25,8 @@
*/
#include "bufferarray.h"
#include "llexception.h"
#include "llmemory.h"
// BufferArray is a list of chunks, each a BufferArray::Block, of contiguous
@ -140,8 +142,22 @@ size_t BufferArray::append(const void * src, size_t len)
{
mBlocks.reserve(mBlocks.size() + 5);
}
Block * block = Block::alloc(BLOCK_ALLOC_SIZE);
memcpy(block->mData, c_src, copy_len);
Block * block;
try
{
block = Block::alloc(BLOCK_ALLOC_SIZE);
}
catch (std::bad_alloc)
{
LLMemory::logMemoryInfo(TRUE);
//output possible call stacks to log file.
LLError::LLCallStacks::print();
LL_WARNS() << "Bad memory allocation in thrown by Block::alloc in read!" << LL_ENDL;
break;
}
memcpy(block->mData, c_src, copy_len);
block->mUsed = copy_len;
llassert_always(block->mUsed <= block->mAlloced);
mBlocks.push_back(block);
@ -149,7 +165,7 @@ size_t BufferArray::append(const void * src, size_t len)
c_src += copy_len;
len -= copy_len;
}
return ret;
return ret - len;
}

12
indra/llcorehttp/httprequest.cpp
View File

@ -37,7 +37,7 @@
#include "_httpopsetget.h"
#include "lltimer.h"
#include "httpstats.h"
namespace
{
@ -52,6 +52,7 @@ namespace LLCore
// ====================================
// HttpRequest Implementation
// ====================================
HttpRequest::Statistics HttpRequest::mStatistics;
HttpRequest::HttpRequest()
@ -62,6 +63,12 @@ HttpRequest::HttpRequest()
mRequestQueue->addRef();
mReplyQueue.reset( new HttpReplyQueue() );
++mStatistics.mCurrentRequests;
++mStatistics.mTotalRequests;
LL_WARNS("HTTPRequest") << "New HttpRequest created (outstanding: " << mStatistics.mCurrentRequests << " total: " << mStatistics.mTotalRequests << ")" << LL_ENDL;
}
@ -74,6 +81,9 @@ HttpRequest::~HttpRequest()
}
mReplyQueue.reset();
--mStatistics.mCurrentRequests;
LL_WARNS("HTTPRequest") << "HttpRequest destroyed (outstanding: " << mStatistics.mCurrentRequests << " total: " << mStatistics.mTotalRequests << ")" << LL_ENDL;
}

15
indra/llcorehttp/httprequest.h
View File

@ -680,7 +680,20 @@ private:
/// @}
// End Global State
// ====================================
struct Statistics
{
Statistics():
mTotalRequests(0),
mCurrentRequests(0)
{}
S32 mTotalRequests;
S32 mCurrentRequests;
};
static Statistics mStatistics;
}; // end class HttpRequest

82
indra/llcorehttp/httpstats.cpp Normal file
View File

@ -0,0 +1,82 @@
/**
* @file llviewerstats.cpp
* @brief LLViewerStats class implementation
*
* $LicenseInfo:firstyear=2002&license=viewerlgpl$
* Second Life Viewer Source Code
* Copyright (C) 2010, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License only.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
* $/LicenseInfo$
*/
#include "httpstats.h"
#include "llerror.h"
namespace LLCore
{
HTTPStats::HTTPStats()
{
resetStats();
}
HTTPStats::~HTTPStats()
{
}
void HTTPStats::resetStats()
{
mResutCodes.clear();
mDataDown.reset();
mDataUp.reset();
}
void HTTPStats::recordResultCode(S32 code)
{
std::map<S32, S32>::iterator it;
it = mResutCodes.find(code);
if (it == mResutCodes.end())
mResutCodes[code] = 1;
else
(*it).second = (*it).second + 1;
}
void HTTPStats::dumpStats()
{
std::stringstream out;
out << "HTTPCore Stats" << std::endl;
out << "Bytes Sent: " << mDataUp.getSum() << std::endl;
out << "Bytes Recv: " << mDataDown.getSum() << std::endl;
out << "Result Codes:" << std::endl << "--- -----" << std::endl;
for (std::map<S32, S32>::iterator it = mResutCodes.begin(); it != mResutCodes.end(); ++it)
{
out << (*it).first << " " << (*it).second << std::endl;
}
LL_WARNS("HTTP Core") << out.str() << LL_ENDL;
}
}

70
indra/llcorehttp/httpstats.h Normal file
View File

@ -0,0 +1,70 @@
/**
* @file llviewerim_peningtats.h
* @brief LLViewerStats class header file
*
* $LicenseInfo:firstyear=2002&license=viewerlgpl$
* Second Life Viewer Source Code
* Copyright (C) 2010, Linden Research, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License only.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*
* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
* $/LicenseInfo$
*/
#ifndef LL_LLVIEWERSTATS_H
#define LL_LLVIEWERSTATS_H
#include "lltracerecording.h"
#include "lltrace.h"
#include "llstatsaccumulator.h"
#include "llsingleton.h"
#include "llsd.h"
namespace LLCore
{
class HTTPStats : public LLSingleton<HTTPStats>
{
LLSINGLETON(HTTPStats);
virtual ~HTTPStats();
public:
void resetStats();
typedef LLStatsAccumulator StatsAccumulator;
void recordDataDown(size_t bytes)
{
mDataDown.push(bytes);
}
void recordDataUp(size_t bytes)
{
mDataUp.push(bytes);
}
void recordResultCode(S32 code);
void dumpStats();
private:
StatsAccumulator mDataDown;
StatsAccumulator mDataUp;
std::map<S32, S32> mResutCodes;
};
}
#endif // LL_LLVIEWERSTATS_H

3
indra/newview/llappcorehttp.cpp
View File

@ -39,6 +39,7 @@
#include <curl/curl.h>
#include "llcorehttputil.h"
#include "httpstats.h"
// Here is where we begin to get our connection usage under control.
// This establishes llcorehttp policy classes that, among other
@ -313,6 +314,8 @@ void LLAppCoreHttp::requestStop()
void LLAppCoreHttp::cleanup()
{
LLCore::HTTPStats::instance().dumpStats();
if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle)
{
// Should have been started already...

4
indra/newview/llfasttimerview.cpp
View File

@ -796,13 +796,13 @@ LLSD LLFastTimerView::analyzePerformanceLogDefault(std::istream& is)
for(stats_map_t::iterator it = time_stats.begin(); it != time_stats.end(); ++it)
{
std::string label = it->first;
ret[label]["TotalTime"] = time_stats[label].mSum;
ret[label]["TotalTime"] = time_stats[label].getSum();
ret[label]["MeanTime"] = time_stats[label].getMean();
ret[label]["MaxTime"] = time_stats[label].getMaxValue();
ret[label]["MinTime"] = time_stats[label].getMinValue();
ret[label]["StdDevTime"] = time_stats[label].getStdDev();
ret[label]["Samples"] = sample_stats[label].mSum;
ret[label]["Samples"] = sample_stats[label].getSum();
ret[label]["MaxSamples"] = sample_stats[label].getMaxValue();
ret[label]["MinSamples"] = sample_stats[label].getMinValue();
ret[label]["StdDevSamples"] = sample_stats[label].getStdDev();

86
indra/newview/llviewerstats.h
View File

@ -30,7 +30,7 @@
#include "lltextureinfo.h"
#include "lltracerecording.h"
#include "lltrace.h"
#include "llstatsaccumulator.h"
enum ESimStatID
{
@ -256,89 +256,9 @@ public:
void addToMessage(LLSD &body);
struct StatsAccumulator
{
S32 mCount;
F32 mSum;
F32 mSumOfSquares;
F32 mMinValue;
F32 mMaxValue;
U32 mCountOfNextUpdatesToIgnore;
typedef LLStatsAccumulator StatsAccumulator;
inline StatsAccumulator()
{
reset();
}
inline void push( F32 val )
{
if ( mCountOfNextUpdatesToIgnore > 0 )
{
mCountOfNextUpdatesToIgnore--;
return;
}
mCount++;
mSum += val;
mSumOfSquares += val * val;
if (mCount == 1 || val > mMaxValue)
{
mMaxValue = val;
}
if (mCount == 1 || val < mMinValue)
{
mMinValue = val;
}
}
inline F32 getMean() const
{
return (mCount == 0) ? 0.f : ((F32)mSum)/mCount;
}
inline F32 getMinValue() const
{
return mMinValue;
}
inline F32 getMaxValue() const
{
return mMaxValue;
}
inline F32 getStdDev() const
{
const F32 mean = getMean();
return (mCount < 2) ? 0.f : sqrt(llmax(0.f,mSumOfSquares/mCount - (mean * mean)));
}
inline U32 getCount() const
{
return mCount;
}
inline void reset()
{
mCount = 0;
mSum = mSumOfSquares = 0.f;
mMinValue = 0.0f;
mMaxValue = 0.0f;
mCountOfNextUpdatesToIgnore = 0;
}
inline LLSD asLLSD() const
{
LLSD data;
data["mean"] = getMean();
data["std_dev"] = getStdDev();
data["count"] = (S32)mCount;
data["min"] = getMinValue();
data["max"] = getMaxValue();
return data;
}
};
// Phase tracking (originally put in for avatar rezzing), tracking
// Phase tracking (originally put in for avatar rezzing), tracking
// progress of active/completed phases for activities like outfit changing.
typedef std::map<std::string,LLTimer> phase_map_t;
typedef std::map<std::string,StatsAccumulator> phase_stats_t;