From 1f44919b3fc994adc27d16fc9dbd476c08d24e3a Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Mon, 6 Mar 2017 17:07:44 -0500 Subject: [PATCH 1/5] SL-409 - use AP_TEXTURE policy for other viewer asset fetches. Enables pipelining. --- indra/newview/llviewerassetstorage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index 41f407b8d9..153108d8a9 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -33,6 +33,7 @@ #include "message.h" #include "llagent.h" +#include "llappcorehttp.h" #include "llviewerregion.h" #include "lltransfersourceasset.h" @@ -474,7 +475,7 @@ void LLViewerAssetStorage::assetRequestCoro( std::string url = getAssetURL(uuid,atype); LL_DEBUGS("ViewerAsset") << "request url: " << url << LL_ENDL; - LLCore::HttpRequest::policy_t httpPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + LLCore::HttpRequest::policy_t httpPolicy(LLAppCoreHttp::AP_TEXTURE); LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter(new LLCoreHttpUtil::HttpCoroutineAdapter("assetRequestCoro", httpPolicy)); LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); From 2cd4418eab69bdd664148c21460327a36ed396ba Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Mon, 6 Mar 2017 19:28:17 -0500 Subject: [PATCH 2/5] SL-409 - build fix, removed unreferenced vars --- indra/newview/llmeshrepository.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a0dced3cfd..a28ab499fd 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -343,10 +343,6 @@ LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space -const S32 REQUEST_HIGH_WATER_MIN = 32; // Limits for GetMesh regions -const S32 REQUEST_HIGH_WATER_MAX = 150; // Should remain under 2X throttle -const S32 REQUEST_LOW_WATER_MIN = 16; -const S32 REQUEST_LOW_WATER_MAX = 75; const S32 REQUEST2_HIGH_WATER_MIN = 32; // Limits for GetMesh2 regions const S32 REQUEST2_HIGH_WATER_MAX = 100; From f028290c9e1afd241944ad63e8ce34af755a2233 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 7 Mar 2017 08:06:03 -0500 Subject: [PATCH 3/5] SL-409 - added size stats to metrics for textures --- indra/newview/lltexturefetch.cpp | 23 ++++++++++++++++------- scripts/metrics/viewer_asset_logs.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 7c0d02c9ca..9469a3c373 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -485,7 +485,7 @@ private: void recordTextureStart(bool is_http); // Threads: Ttf - void recordTextureDone(bool is_http); + void recordTextureDone(bool is_http, F64 byte_count); void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } @@ -1426,6 +1426,13 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (processSimulatorPackets()) { + // Capture some measure of total size for metrics + F64 byte_count = 0; + for (S32 i=mFirstPacket; i<=mLastPacket; i++) + { + byte_count += mPackets[i]->mSize; + } + LL_DEBUGS(LOG_TXT) << mID << ": Loaded from Sim. Bytes: " << mFormattedImage->getDataSize() << LL_ENDL; mFetcher->removeFromNetworkQueue(this, false); if (mFormattedImage.isNull() || !mFormattedImage->getDataSize()) @@ -1443,7 +1450,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } setState(DECODE_IMAGE); mWriteToCacheState = SHOULD_WRITE; - recordTextureDone(false); + + recordTextureDone(false, byte_count); } else { @@ -2093,7 +2101,7 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe mFetcher->removeFromHTTPQueue(mID, data_size); - recordTextureDone(true); + recordTextureDone(true, data_size); } // -Mw @@ -2493,14 +2501,15 @@ void LLTextureFetchWorker::recordTextureStart(bool is_http) // Threads: Ttf -void LLTextureFetchWorker::recordTextureDone(bool is_http) +void LLTextureFetchWorker::recordTextureDone(bool is_http, F64 byte_count) { if (mMetricsStartTime.value()) { LLViewerAssetStatsFF::record_response(LLViewerAssetType::AT_TEXTURE, - is_http, - LLImageBase::TYPE_AVATAR_BAKE == mType, - LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType, + LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime, + byte_count); mMetricsStartTime = (U32Seconds)0; } LLViewerAssetStatsFF::record_dequeue(LLViewerAssetType::AT_TEXTURE, diff --git a/scripts/metrics/viewer_asset_logs.py b/scripts/metrics/viewer_asset_logs.py index 8e46ca4707..e48286f696 100644 --- a/scripts/metrics/viewer_asset_logs.py +++ b/scripts/metrics/viewer_asset_logs.py @@ -96,7 +96,7 @@ if __name__ == "__main__": for key in sorted(stats.keys()): val = stats[key] if val["count"] > 0: - print key,"count",val["count"],"mean_time",val["sum"]/val["count"],"mean_bytes",val["sum_bytes"]/val["count"],"enqueued",val["enqueued"],"dequeued",val["dequeued"] + print key,"count",val["count"],"mean_time",val["sum"]/val["count"],"mean_bytes",val["sum_bytes"]/val["count"],"net bytes/sec",val["sum_bytes"]/val["sum"],"enqueued",val["enqueued"],"dequeued",val["dequeued"] else: print key,"count",val["count"],"enqueued",val["enqueued"],"dequeued",val["dequeued"] From e9c786ec6060da836bf6aa49f0e443f41f41a2c0 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 10 Mar 2017 11:29:48 -0500 Subject: [PATCH 4/5] SL-409 - fixes for viewer asset metrics. Copy and/or assignment of Recording objects is failing, changed to create LLSD blob earlier and just copy that. --- indra/newview/llappviewer.cpp | 11 ++---- indra/newview/lltexturefetch.cpp | 50 +++++++++++----------------- indra/newview/lltexturefetch.h | 2 +- indra/newview/llviewerassetstats.cpp | 6 ++-- 4 files changed, 27 insertions(+), 42 deletions(-) diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 5eb82a2d0b..b138631f92 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5912,23 +5912,18 @@ void LLAppViewer::metricsSend(bool enable_reporting) { std::string caps_url = regionp->getCapability("ViewerMetrics"); + LLSD sd = gViewerAssetStats->asLLSD(true); if (gSavedSettings.getBOOL("QAModeMetrics")) { - dump_sequential_xml("metric_asset_stats",gViewerAssetStats->asLLSD(true)); + dump_sequential_xml("metric_asset_stats",sd); } - // Make a copy of the main stats to send into another thread. - // Receiving thread takes ownership. - LLViewerAssetStats * main_stats(new LLViewerAssetStats(*gViewerAssetStats)); - main_stats->stop(); - // Send a report request into 'thread1' to get the rest of the data // and provide some additional parameters while here. LLAppViewer::sTextureFetch->commandSendMetrics(caps_url, gAgentSessionID, gAgentID, - main_stats); - main_stats = 0; // Ownership transferred + sd); } else { diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 9469a3c373..6468991c79 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -824,7 +824,7 @@ public: TFReqSendMetrics(const std::string & caps_url, const LLUUID & session_id, const LLUUID & agent_id, - LLViewerAssetStats * main_stats); + LLSD& stats_sd); TFReqSendMetrics & operator=(const TFReqSendMetrics &); // Not defined virtual ~TFReqSendMetrics(); @@ -835,7 +835,7 @@ public: const std::string mCapsURL; const LLUUID mSessionID; const LLUUID mAgentID; - LLViewerAssetStats * mMainStats; + LLSD mStatsSD; private: LLCore::HttpHandler::ptr_t mHandler; @@ -3872,9 +3872,9 @@ void LLTextureFetch::commandSetRegion(U64 region_handle) void LLTextureFetch::commandSendMetrics(const std::string & caps_url, const LLUUID & session_id, const LLUUID & agent_id, - LLViewerAssetStats * main_stats) + LLSD& stats_sd) { - TFReqSendMetrics * req = new TFReqSendMetrics(caps_url, session_id, agent_id, main_stats); + TFReqSendMetrics * req = new TFReqSendMetrics(caps_url, session_id, agent_id, stats_sd); cmdEnqueue(req); } @@ -3983,22 +3983,20 @@ TFReqSetRegion::doWork(LLTextureFetch *) } TFReqSendMetrics::TFReqSendMetrics(const std::string & caps_url, - const LLUUID & session_id, - const LLUUID & agent_id, - LLViewerAssetStats * main_stats): + const LLUUID & session_id, + const LLUUID & agent_id, + LLSD& stats_sd): LLTextureFetch::TFRequest(), mCapsURL(caps_url), mSessionID(session_id), mAgentID(agent_id), - mMainStats(main_stats), + mStatsSD(stats_sd), mHandler(new AssetReportHandler) {} TFReqSendMetrics::~TFReqSendMetrics() { - delete mMainStats; - mMainStats = 0; } @@ -4019,26 +4017,18 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) static volatile bool reporting_started(false); static volatile S32 report_sequence(0); - // We've taken over ownership of the stats copy at this - // point. Get a working reference to it for merging here - // but leave it in 'this'. Destructor will rid us of it. - LLViewerAssetStats & main_stats = *mMainStats; - - LLViewerAssetStats::AssetStats stats; - main_stats.getStats(stats, true); - //LLSD merged_llsd = main_stats.asLLSD(); + // In mStatsSD, we have a copy we own of the LLSD representation + // of the asset stats. Add some additional fields and ship it off. bool initial_report = !reporting_started; - stats.session_id = mSessionID; - stats.agent_id = mAgentID; - stats.message = "ViewerAssetMetrics"; - stats.sequence = static_cast(report_sequence); - stats.initial = initial_report; - stats.break_ = static_cast(LLTextureFetch::svMetricsDataBreak); + mStatsSD["session_id"] = mSessionID; + mStatsSD["agent_id"] = mAgentID; + mStatsSD["message"] = "ViewerAssetMetrics"; + mStatsSD["sequence"] = report_sequence; + mStatsSD["initial"] = initial_report; + mStatsSD["break"] = static_cast(LLTextureFetch::svMetricsDataBreak); - LLSD sd; - LLParamSDParser parser; - parser.writeSD(sd, stats); + LL_INFOS(LOG_TXT) << "ViewerAssetMetrics after fields added\n" << ll_pretty_print_sd(mStatsSD) << LL_ENDL; // Update sequence number if (S32_MAX == ++report_sequence) @@ -4049,7 +4039,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Limit the size of the stats report if necessary. - sd["truncated"] = truncate_viewer_metrics(10, sd); + mStatsSD["truncated"] = truncate_viewer_metrics(10, mStatsSD); if (! mCapsURL.empty()) { @@ -4058,7 +4048,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) fetcher->getMetricsPolicyClass(), report_priority, mCapsURL, - sd, + mStatsSD, LLCore::HttpOptions::ptr_t(), fetcher->getMetricsHeaders(), mHandler); @@ -4072,7 +4062,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // In QA mode, Metrics submode, log the result for ease of testing if (fetcher->isQAMode()) { - LL_INFOS(LOG_TXT) << ll_pretty_print_sd(sd) << LL_ENDL; + LL_INFOS(LOG_TXT) << "ViewerAssetMetrics as submitted\n" << ll_pretty_print_sd(mStatsSD) << LL_ENDL; } return true; diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 072e6a3307..cfa312ccd9 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -160,7 +160,7 @@ public: void commandSendMetrics(const std::string & caps_url, const LLUUID & session_id, const LLUUID & agent_id, - LLViewerAssetStats * main_stats); + LLSD& stats_sd); // Threads: T* void commandDataBreak(); diff --git a/indra/newview/llviewerassetstats.cpp b/indra/newview/llviewerassetstats.cpp index a9e0ba7b5d..14e05fd440 100644 --- a/indra/newview/llviewerassetstats.cpp +++ b/indra/newview/llviewerassetstats.cpp @@ -323,10 +323,10 @@ void LLViewerAssetStats::getStats(AssetStats& stats, bool compact_output) grid_from_region_handle(it->first, &grid_x, &grid_y); r .grid_x(grid_x) .grid_y(grid_y) - .duration(F64Microseconds(rec.getDuration()).value()); + .duration(F64Seconds(rec.getDuration()).value()); } - stats.duration(mCurRecording ? F64Microseconds(mCurRecording->getDuration()).value() : 0.0); + stats.duration(mCurRecording ? F64Seconds(mCurRecording->getDuration()).value() : 0.0); } LLSD LLViewerAssetStats::asLLSD(bool compact_output) @@ -376,7 +376,7 @@ void record_response(LLViewerAssetType::EType at, bool with_http, bool is_temp, { const EViewerAssetCategories eac(asset_type_to_category(at, with_http, is_temp)); - record(sResponse[int(eac)], F64Microseconds(duration)); + record(sResponse[int(eac)], F64Seconds(duration)); record(sBytesFetched[int(eac)], bytes); } From 295c8a437afb29be1ec4b59acad60c35afb6ffc3 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Mon, 20 Mar 2017 08:12:37 -0400 Subject: [PATCH 5/5] MAINT-7195 possible fix --- indra/newview/llviewerassetstorage.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index 153108d8a9..496fbbecff 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -428,6 +428,11 @@ void LLViewerAssetStorage::queueRequestHttp( BOOL is_priority) { LL_DEBUGS("ViewerAsset") << "Request asset via HTTP " << uuid << " type " << LLAssetType::lookup(atype) << LL_ENDL; + if (!gAgent.getRegion()) + { + LL_WARNS() << "No region, fetch fails" << LL_ENDL; + return; + } std::string cap_url = gAgent.getRegion()->getCapability("ViewerAsset"); if (cap_url.empty()) {