diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index f2ecd9f5b0..01ed49d0d9 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -44,7 +44,12 @@ static const char* subdirs = "0123456789abcdef"; LLDiskCache::LLDiskCache(const std::string cache_dir, const uintmax_t max_size_bytes, - const bool enable_cache_debug_info) : + const bool enable_cache_debug_info +// Add High/Low water mark support + ,const F32 highwater_mark_percent + ,const F32 lowwater_mark_percent +// + ) : mCacheDir(cache_dir), mMaxSizeBytes(max_size_bytes), mEnableCacheDebugInfo(enable_cache_debug_info) @@ -113,6 +118,8 @@ void LLDiskCache::purge() #else std::string cache_path(mCacheDir); #endif + uintmax_t file_size_total = 0; // try to make simple cache less naive. + if (boost::filesystem::is_directory(cache_path, ec) && !ec.failed()) { // Optimize asset simple disk cache @@ -137,6 +144,7 @@ void LLDiskCache::purge() { continue; } + file_size_total += file_size; // try to make simple cache less naive. file_info.push_back(file_info_t(file_time, { file_size, file_path })); } @@ -145,52 +153,106 @@ void LLDiskCache::purge() } } + // add high water/low water thresholds to reduce the churn in the cache. + LL_DEBUGS("LLDiskCache") << "Cache is " << (int)(((F32)file_size_total)/mMaxSizeBytes*100.0) << "% full" << LL_ENDL; + if( file_size_total < mMaxSizeBytes * (mHighPercent/100) ) + { + // Nothing to do here + LL_DEBUGS("LLDiskCache") << "Not exceded high water - do nothing" << LL_ENDL; + return; + } + // If we reach here we are above the trigger level so we must purge until we've removed enough to take us down to the low water mark. + // std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { - return x.first > y.first; + return x.first < y.first; // sort oldest to newest, to we can remove the oldest files first. }); - LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; + + // add high water/low water thresholds to reduce the churn in the cache. + auto target_size = (uintmax_t)(mMaxSizeBytes * (mLowPercent/100)); + LL_INFOS() << "Purging cache to a maximum of " << target_size << " bytes" << LL_ENDL; + // // Extra accounting to track the retention of static assets //std::vector file_removed; - std::vector file_removed; - int keep{0}; - int del{0}; - int skip{0}; + enum class purge_action { delete_file=0, keep_file, skip_file }; + std::map file_removed; + auto keep{file_info.size()}; + auto del{0}; + auto skip{0}; // - if (mEnableCacheDebugInfo) - { - file_removed.reserve(file_info.size()); - } - uintmax_t file_size_total = 0; + // revised purge logic to track amount removed not retained to shortern loop + // uintmax_t file_size_total = 0; + // if (mEnableCacheDebugInfo) + // { + // file_removed.reserve(file_info.size()); + // } + // uintmax_t file_size_total = 0; + // for (file_info_t& entry : file_info) + // { + // file_size_total += entry.second.first; + + // bool should_remove = file_size_total > mMaxSizeBytes; + // // Make sure static assets are not eliminated + // S32 action{ should_remove ? 0 : 1 }; + // if (should_remove) + // { + // auto uuid_as_string = gDirUtilp->getBaseFileName(entry.second.second, true); + // uuid_as_string = uuid_as_string.substr(mCacheFilenamePrefix.size() + 1, 36);// skip "sl_cache_" and trailing "_N" + // // LL_INFOS() << "checking UUID=" < + // if (mEnableCacheDebugInfo) + // { + // // Static asset stuff + // //file_removed.push_back(should_remove); + // file_removed.push_back(action); + // } + uintmax_t deleted_size_total = 0; for (file_info_t& entry : file_info) { - file_size_total += entry.second.first; + // first check if we still need to delete more files + bool should_remove = (file_size_total - deleted_size_total) > target_size; - bool should_remove = file_size_total > mMaxSizeBytes; // Make sure static assets are not eliminated - S32 action{ should_remove ? 0 : 1 }; - if (should_remove) + auto action{ should_remove ? purge_action::delete_file : purge_action::keep_file }; + if (!should_remove) { - auto uuid_as_string = gDirUtilp->getBaseFileName(entry.second.second, true); - uuid_as_string = uuid_as_string.substr(mCacheFilenamePrefix.size() + 1, 36);// skip "sl_cache_" and trailing "_N" - // LL_INFOS() << "checking UUID=" < + + auto this_file_size = entry.second.first; + deleted_size_total += this_file_size; + auto uuid_as_string = gDirUtilp->getBaseFileName(entry.second.second, true); + uuid_as_string = uuid_as_string.substr(mCacheFilenamePrefix.size() + 1, 36);// skip "sl_cache_" and trailing "_N" + // LL_INFOS() << "checking UUID=" < Static asset stuff - //file_removed.push_back(should_remove); - file_removed.push_back(action); + // Static asset stuff + //file_removed.push_back(should_remove); + file_removed.emplace(entry.second.second, action); } + // if (should_remove) { boost::filesystem::remove(entry.second.second, ec); @@ -200,37 +262,52 @@ void LLDiskCache::purge() } } } - +// update the debug logging to be more useful + auto end_time = std::chrono::high_resolution_clock::now(); + auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); +// if (mEnableCacheDebugInfo) { - auto end_time = std::chrono::high_resolution_clock::now(); - auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + // update the debug logging to be more useful + // auto end_time = std::chrono::high_resolution_clock::now(); + // auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + // // Log afterward so it doesn't affect the time measurement // Logging thousands of file results can take hundreds of milliseconds + auto deleted_so_far = 0; // update the debug logging to be more useful for (size_t i = 0; i < file_info.size(); ++i) { const file_info_t& entry = file_info[i]; // Static asset stuff + deleted_so_far += entry.second.first; // update the debug logging to be more useful //const bool removed = file_removed[i]; //const std::string action = removed ? "DELETE:" : "KEEP:"; std::string action{}; - switch (file_removed[i]) - { - default: - case 0: - action = "KEEP"; - keep++; - break; - case 1: + + // Check if the file exists in the map + auto& filename{ entry.second.second }; + if (file_removed.find(filename) != file_removed.end()) { + // File found in the map, retrieve the corresponding enum value + switch (file_removed[filename]) { + case purge_action::delete_file: action = "DELETE"; del++; - break; - case 2: + break; + case purge_action::skip_file: action = "STATIC"; skip++; - break; + break; + default: + // Handle any unexpected enum value + action = "UNKNOWN"; + break; } + } + else + { + action = "KEEP"; + } // // have to do this because of LL_INFO/LL_END weirdness @@ -240,14 +317,22 @@ void LLDiskCache::purge() line << entry.first << " "; line << entry.second.first << " "; line << entry.second.second; - line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; + line << " (" << file_size_total - deleted_so_far << "/" << mMaxSizeBytes << ")"; // update the debug logging to be more useful LL_INFOS() << line.str() << LL_ENDL; } - - LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; - LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; - LL_INFOS() << "Deleted: " << del << " Skipped: " << skip << " Kept: " << keep << LL_ENDL; // Extra accounting to track the retention of static assets +// make the summary stats more easily enabled. } + // update the debug logging to be more useful + // LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; + // LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; + + auto newCacheSize = updateCacheSize(file_size_total - deleted_size_total); + LL_INFOS("LLDiskCache") << "Total dir size after purge is " << newCacheSize << LL_ENDL; + LL_INFOS("LLDiskCache") << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; +// + LL_INFOS("LLDiskCache") << "Deleted: " << del << " Skipped: " << skip << " Kept: " << keep << LL_ENDL; // Extra accounting to track the retention of static assets + LL_INFOS("LLDiskCache") << "Total of " << deleted_size_total << " bytes removed." << LL_ENDL; // Extra accounting to track the retention of static assets + // } this bracket was moved up a few lines. } const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) @@ -534,8 +619,32 @@ void LLDiskCache::removeOldVFSFiles() } } -uintmax_t LLDiskCache::dirFileSize(const std::string& dir) +// Lets not scan every single time if we can avoid it eh? +// uintmax_t LLDiskCache::dirFileSize(const std::string& dir) +// { +uintmax_t LLDiskCache::updateCacheSize(const uintmax_t newsize) { + mStoredCacheSize = newsize; + mLastScanTime = system_clock::now(); + return mStoredCacheSize; +} + +uintmax_t LLDiskCache::dirFileSize(const std::string& dir, bool force ) +{ + using namespace std::chrono; + const seconds cache_duration{ 120 };// A rather arbitrary number. it takes 5 seconds+ on a fast drive to scan 80K+ items. purge runs every minute and will update. so 120 should mean we never need a superfluous cache scan. + + const auto current_time = system_clock::now(); + + const auto time_difference = duration_cast(current_time - mLastScanTime); + + // Check if the cached result can be used + if( !force && time_difference < cache_duration ) + { + LL_DEBUGS("LLDiskCache") << "Using cached result: " << mStoredCacheSize << LL_ENDL; + return mStoredCacheSize; + } +// uintmax_t total_file_size = 0; /** @@ -577,7 +686,10 @@ uintmax_t LLDiskCache::dirFileSize(const std::string& dir) } } - return total_file_size; +// Lets not scan every single time if we can avoid it eh? + // return total_file_size; + return updateCacheSize(total_file_size); +// } LLPurgeDiskCacheThread::LLPurgeDiskCacheThread() : diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 4b51b66e1f..fc1be25b9f 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -63,6 +63,9 @@ #define _LLDISKCACHE #include "llsingleton.h" +#include +using namespace std::chrono; + class LLDiskCache : public LLParamSingleton @@ -92,7 +95,18 @@ class LLDiskCache : * if there are bugs, we can ask uses to enable this * setting and send us their logs */ - const bool enable_cache_debug_info); + const bool enable_cache_debug_info, + // Add high/low threshold controls for cache purging + /** + * A floating point percentage of the max_size_bytes above which the cache purge will trigger. + */ + const F32 highwater_mark_percent, + /** + * A floating point percentage of the max_size_bytes which the cache purge will aim to reach once triggered. + */ + const F32 lowwater_mark_percent + // + ); virtual ~LLDiskCache() = default; @@ -157,6 +171,10 @@ class LLDiskCache : // Better asset cache size control void setMaxSizeBytes(uintmax_t size) { mMaxSizeBytes = size; } + // High/Low water control + void setHighWaterPercentage(F32 HiPct) { mHighPercent = llclamp(HiPct, mLowPercent, 100.0); }; + void setLowWaterPercentage(F32 LowPct) { mLowPercent = llclamp(LowPct, 0.0, mHighPercent); }; + // private: /** @@ -164,7 +182,8 @@ class LLDiskCache : * directory. Primarily used here to determine the directory size * before and after the cache purge */ - uintmax_t dirFileSize(const std::string& dir); + uintmax_t updateCacheSize(const uintmax_t newsize); // enable time based caching of dirfilesize except when force is true. + uintmax_t dirFileSize(const std::string& dir, bool force=false); // enable time based caching of dirfilesize except when force is true. /** * Utility function to convert an LLAssetType enum into a @@ -172,6 +191,13 @@ class LLDiskCache : */ const std::string assetTypeToString(LLAssetType::EType at); + /** + * cache the directory size cos it takes forever to calculate it + * + */ + uintmax_t mStoredCacheSize{ 0 }; + time_point mLastScanTime{ }; + private: /** * The maximum size of the cache in bytes. After purge is called, the @@ -179,6 +205,10 @@ class LLDiskCache : * less than this value */ uintmax_t mMaxSizeBytes; + // High/Low water control + F32 mHighPercent { 95.0 }; + F32 mLowPercent { 70.0 }; + // /** * The folder that holds the cached files. The consumer of this diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 4dcf7dd3c3..9f9ac1049d 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -2952,6 +2952,28 @@ Value 2048 + FSDiskCacheHighWaterPercent + + Comment + Trigger point above which we should start to clear out older cache entries + Persist + 1 + Type + F32 + Value + 95.0 + + FSDiskCacheLowWaterPercent + + Comment + Level to drain cache to once it goes over the high water limit. + Persist + 1 + Type + F32 + Value + 70.0 + CacheLocation Comment diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 486ead0f37..5fb5a21137 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5195,7 +5195,10 @@ bool LLAppViewer::initCache() // const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name); - LLDiskCache::initParamSingleton(cache_dir, disk_cache_size, enable_cache_debug_info); + // Improve cache purge triggering + // LLDiskCache::initParamSingleton(cache_dir, disk_cache_size, enable_cache_debug_info); + LLDiskCache::initParamSingleton(cache_dir, disk_cache_size, enable_cache_debug_info, gSavedSettings.getF32("FSDiskCacheHighWaterPercent"), gSavedSettings.getF32("FSDiskCacheLowWaterPercent")); + // if (!read_only) { diff --git a/indra/newview/llappviewerwin32.cpp b/indra/newview/llappviewerwin32.cpp index f3b43244c4..26c0a04254 100644 --- a/indra/newview/llappviewerwin32.cpp +++ b/indra/newview/llappviewerwin32.cpp @@ -1196,7 +1196,7 @@ DWORD WINAPI purgeThread( LPVOID lpParameter ) for( auto dir : vctDirs ) { - LL_INFOS("CachePurge") << "Removing an old cache" << LL_ENDL; + LL_INFOS("LLDiskCache") << "Removing an old cache" << LL_ENDL; // consistent tagging to help searching log files deleteCacheDirectory( dir ); } @@ -1214,7 +1214,7 @@ void LLAppViewerWin32::startCachePurge() if( !hThread ) { - LL_WARNS("CachePurge") << "CreateThread failed: " << GetLastError() << LL_ENDL; + LL_WARNS("LLDiskCache") << "CreateThread failed: " << GetLastError() << LL_ENDL; // consistent tagging to help searching log files } else SetThreadPriority( hThread, THREAD_MODE_BACKGROUND_BEGIN ); diff --git a/indra/newview/llviewercontrol.cpp b/indra/newview/llviewercontrol.cpp index 85203f2503..a22a5734a4 100644 --- a/indra/newview/llviewercontrol.cpp +++ b/indra/newview/llviewercontrol.cpp @@ -1087,6 +1087,20 @@ void handleDiskCacheSizeChanged(const LLSD& newValue) } // +// Better asset cache purge control +void handleDiskCacheHighWaterPctChanged(const LLSD& newValue) +{ + const auto new_high = newValue.asReal(); + LLDiskCache::getInstance()->setHighWaterPercentage(new_high); +} + +void handleDiskCacheLowWaterPctChanged(const LLSD& newValue) +{ + const auto new_low = newValue.asReal(); + LLDiskCache::getInstance()->setHighWaterPercentage(new_low); +} +// + void handleTargetFPSChanged(const LLSD& newValue) { const auto targetFPS = gSavedSettings.getU32("TargetFPS"); @@ -1464,7 +1478,10 @@ void settings_setup_listeners() // Better asset cache size control setting_setup_signal_listener(gSavedSettings, "FSDiskCacheSize", handleDiskCacheSizeChanged); - + // Better asset cache purge control + setting_setup_signal_listener(gSavedSettings, "FSDiskCacheHighWaterPercent", handleDiskCacheHighWaterPctChanged); + setting_setup_signal_listener(gSavedSettings, "FSDiskCacheHighWaterPercent", handleDiskCacheLowWaterPctChanged); + // // Handle IME text input getting enabled or disabled #if LL_SDL2