livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#3969 Make inventory creation from cache faster 

Still has a lot of space for improvements, but should be 2.5 times faster
master
Andrey Kleshchev 2025-07-25 19:06:06 +03:00 committed by Andrey Kleshchev
parent 329e71a7d9
commit 3b8b408b90
8 changed files with 221 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
indra/llinventory/llinventory.cpp
View File

@ -995,6 +995,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
// TODO - figure out if this should be moved into the noclobber fields above
mThumbnailUUID.setNull();
mFavorite = false;
mPermissions.init(LLUUID::null, LLUUID::null, LLUUID::null, LLUUID::null);
// iterate as map to avoid making unnecessary temp copies of everything
LLSD::map_const_iterator i, end;
@ -1053,7 +1054,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
if (i->first == INV_PERMISSIONS_LABEL)
{
mPermissions = ll_permissions_from_sd(i->second);
mPermissions.importLLSD(i->second);
continue;
}
@ -1522,53 +1523,68 @@ void LLInventoryCategory::exportLLSD(LLSD& cat_data) const
}
}
bool LLInventoryCategory::importLLSD(const LLSD& cat_data)
bool LLInventoryCategory::importLLSDMap(const LLSD& cat_data)
{
if (cat_data.has(INV_FOLDER_ID_LABEL))
LLSD::map_const_iterator i, end;
end = cat_data.endMap();
for ( i = cat_data.beginMap(); i != end; ++i)
{
setUUID(cat_data[INV_FOLDER_ID_LABEL].asUUID());
importLLSD(i->first, i->second);
}
if (cat_data.has(INV_PARENT_ID_LABEL))
return true;
}
bool LLInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
{
if (label == INV_FOLDER_ID_LABEL)
{
setParent(cat_data[INV_PARENT_ID_LABEL].asUUID());
setUUID(value.asUUID());
return true;
}
if (cat_data.has(INV_ASSET_TYPE_LABEL))
else if (label == INV_PARENT_ID_LABEL)
{
setType(LLAssetType::lookup(cat_data[INV_ASSET_TYPE_LABEL].asString()));
setParent(value.asUUID());
return true;
}
if (cat_data.has(INV_PREFERRED_TYPE_LABEL))
else if (label == INV_ASSET_TYPE_LABEL)
{
setPreferredType(LLFolderType::lookup(cat_data[INV_PREFERRED_TYPE_LABEL].asString()));
setType(LLAssetType::lookup(value.asString()));
return true;
}
if (cat_data.has(INV_THUMBNAIL_LABEL))
else if (label == INV_PREFERRED_TYPE_LABEL)
{
setPreferredType(LLFolderType::lookup(value.asString()));
return true;
}
else if (label == INV_THUMBNAIL_LABEL)
{
LLUUID thumbnail_uuid;
const LLSD &thumbnail_data = cat_data[INV_THUMBNAIL_LABEL];
if (thumbnail_data.has(INV_ASSET_ID_LABEL))
if (value.has(INV_ASSET_ID_LABEL))
{
thumbnail_uuid = thumbnail_data[INV_ASSET_ID_LABEL].asUUID();
thumbnail_uuid = value[INV_ASSET_ID_LABEL].asUUID();
}
setThumbnailUUID(thumbnail_uuid);
return true;
}
if (cat_data.has(INV_FAVORITE_LABEL))
if (label == INV_FAVORITE_LABEL)
{
bool favorite = false;
const LLSD& favorite_data = cat_data[INV_FAVORITE_LABEL];
if (favorite_data.has(INV_TOGGLED_LABEL))
if (value.has(INV_TOGGLED_LABEL))
{
favorite = favorite_data[INV_TOGGLED_LABEL].asBoolean();
favorite = value[INV_TOGGLED_LABEL].asBoolean();
}
setFavorite(favorite);
}
if (cat_data.has(INV_NAME_LABEL))
else if (label == INV_NAME_LABEL)
{
mName = cat_data[INV_NAME_LABEL].asString();
mName = value.asString();
LLStringUtil::replaceNonstandardASCII(mName, ' ');
LLStringUtil::replaceChar(mName, '|', ' ');
}
return true;
}
return false;
}
///----------------------------------------------------------------------------
/// Local function definitions for testing purposes
///----------------------------------------------------------------------------

3
indra/llinventory/llinventory.h
View File

@ -274,7 +274,8 @@ public:
virtual bool exportLegacyStream(std::ostream& output_stream, bool include_asset_key = true) const;
virtual void exportLLSD(LLSD& sd) const;
bool importLLSD(const LLSD& cat_data);
bool importLLSDMap(const LLSD& cat_data);
virtual bool importLLSD(const std::string& label, const LLSD& value);
//--------------------------------------------------------------------
// Member Variables
//--------------------------------------------------------------------

104
indra/llinventory/llpermissions.cpp
View File

@ -704,6 +704,79 @@ bool LLPermissions::exportLegacyStream(std::ostream& output_stream) const
return true;
}
static const std::string PERM_CREATOR_ID_LABEL("creator_id");
static const std::string PERM_OWNER_ID_LABEL("owner_id");
static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
static const std::string PERM_GROUP_ID_LABEL("group_id");
static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
static const std::string PERM_BASE_MASK_LABEL("base_mask");
static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
static const std::string PERM_GROUP_MASK_LABEL("group_mask");
static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
void LLPermissions::importLLSD(const LLSD& sd_perm)
{
LLSD::map_const_iterator i, end;
end = sd_perm.endMap();
for (i = sd_perm.beginMap(); i != end; ++i)
{
const std::string& label = i->first;
if (label == PERM_CREATOR_ID_LABEL)
{
mCreator = i->second.asUUID();
continue;
}
if (label == PERM_OWNER_ID_LABEL)
{
mOwner = i->second.asUUID();
continue;
}
if (label == PERM_LAST_OWNER_ID_LABEL)
{
mLastOwner = i->second.asUUID();
continue;
}
if (label == PERM_GROUP_ID_LABEL)
{
mGroup = i->second.asUUID();
continue;
}
if (label == PERM_BASE_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskBase = mask;
continue;
}
if (label == PERM_OWNER_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskOwner = mask;
continue;
}
if (label == PERM_EVERYONE_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskEveryone = mask;
continue;
}
if (label == PERM_GROUP_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskGroup = mask;
continue;
}
if (label == PERM_NEXT_OWNER_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskNextOwner = mask;
continue;
}
}
fix();
}
bool LLPermissions::operator==(const LLPermissions &rhs) const
{
return
@ -998,16 +1071,6 @@ std::string mask_to_string(U32 mask)
///----------------------------------------------------------------------------
/// exported functions
///----------------------------------------------------------------------------
static const std::string PERM_CREATOR_ID_LABEL("creator_id");
static const std::string PERM_OWNER_ID_LABEL("owner_id");
static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
static const std::string PERM_GROUP_ID_LABEL("group_id");
static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
static const std::string PERM_BASE_MASK_LABEL("base_mask");
static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
static const std::string PERM_GROUP_MASK_LABEL("group_mask");
static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
LLSD ll_create_sd_from_permissions(const LLPermissions& perm)
{
@ -1032,25 +1095,6 @@ void ll_fill_sd_from_permissions(LLSD& rv, const LLPermissions& perm)
LLPermissions ll_permissions_from_sd(const LLSD& sd_perm)
{
LLPermissions rv;
rv.init(
sd_perm[PERM_CREATOR_ID_LABEL].asUUID(),
sd_perm[PERM_OWNER_ID_LABEL].asUUID(),
sd_perm[PERM_LAST_OWNER_ID_LABEL].asUUID(),
sd_perm[PERM_GROUP_ID_LABEL].asUUID());
// We do a cast to U32 here since LLSD does not attempt to
// represent unsigned ints.
PermissionMask mask;
mask = (U32)(sd_perm[PERM_BASE_MASK_LABEL].asInteger());
rv.setMaskBase(mask);
mask = (U32)(sd_perm[PERM_OWNER_MASK_LABEL].asInteger());
rv.setMaskOwner(mask);
mask = (U32)(sd_perm[PERM_EVERYONE_MASK_LABEL].asInteger());
rv.setMaskEveryone(mask);
mask = (U32)(sd_perm[PERM_GROUP_MASK_LABEL].asInteger());
rv.setMaskGroup(mask);
mask = (U32)(sd_perm[PERM_NEXT_OWNER_MASK_LABEL].asInteger());
rv.setMaskNext(mask);
rv.fix();
rv.importLLSD(sd_perm);
return rv;
}

2
indra/llinventory/llpermissions.h
View File

@ -299,6 +299,8 @@ public:
bool importLegacyStream(std::istream& input_stream);
bool exportLegacyStream(std::ostream& output_stream) const;
void importLLSD(const LLSD& sd_perm);
bool operator==(const LLPermissions &rhs) const;
bool operator!=(const LLPermissions &rhs) const;

2
indra/llinventory/tests/inventorymisc_test.cpp
View File

@ -518,7 +518,7 @@ namespace tut
file.close();
LLPointer<LLInventoryCategory> src2 = new LLInventoryCategory();
src2->importLLSD(s_item);
src2->importLLSDMap(s_item);
ensure_equals("1.item id::getUUID() failed", src1->getUUID(), src2->getUUID());
ensure_equals("2.parent::getParentUUID() failed", src1->getParentUUID(), src2->getParentUUID());

136
indra/newview/llinventorymodel.cpp
View File

@ -78,7 +78,7 @@
// Increment this if the inventory contents change in a non-backwards-compatible way.
// For viewer 2, the addition of link items makes a pre-viewer-2 cache incorrect.
const S32 LLInventoryModel::sCurrentInvCacheVersion = 4;
const S32 LLInventoryModel::sCurrentInvCacheVersion = 5;
bool LLInventoryModel::sFirstTimeInViewer2 = true;
S32 LLInventoryModel::sPendingSystemFolders = 0;
@ -3384,7 +3384,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
}
LL_INFOS(LOG_INV) << "loading inventory from: (" << filename << ")" << LL_ENDL;
llifstream file(filename.c_str());
llifstream file(filename.c_str(), std::ifstream::in | std::ifstream::binary);
if (!file.is_open())
{
@ -3393,55 +3393,66 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
}
is_cache_obsolete = true; // Obsolete until proven current
//U64 lines_count = 0U;
std::string line;
LLPointer<LLSDParser> parser = new LLSDNotationParser();
while (std::getline(file, line))
U32 value_nbo = 0;
file.read((char*)&value_nbo, sizeof(U32));
if (file.fail())
{
LLSD s_item;
std::istringstream iss(line);
if (parser->parse(iss, s_item, line.length()) == LLSDParser::PARSE_FAILURE)
{
LL_WARNS(LOG_INV)<< "Parsing inventory cache failed" << LL_ENDL;
break;
LL_WARNS(LOG_INV) << "Failed to read cache version. Unable to load inventory from: " << filename << LL_ENDL;
}
if (s_item.has("inv_cache_version"))
else
{
S32 version = s_item["inv_cache_version"].asInteger();
S32 version = (S32)ntohl(value_nbo);
if (version == sCurrentInvCacheVersion)
{
// Cache is up to date
is_cache_obsolete = false;
continue;
}
else
{
LL_WARNS(LOG_INV)<< "Inventory cache is out of date" << LL_ENDL;
break;
LL_WARNS(LOG_INV) << "Inventory cache is out of date" << LL_ENDL;
}
}
else if (s_item.has("cat_id"))
{
if (is_cache_obsolete)
break;
LLSD inventory;
if (!is_cache_obsolete)
{
LLPointer<LLSDParser> parser = new LLSDBinaryParser();
if (parser->parse(file, inventory, LLSDSerialize::SIZE_UNLIMITED) == LLSDParser::PARSE_FAILURE)
{
is_cache_obsolete = true;
LL_WARNS(LOG_INV) << "Parsing inventory cache failed" << LL_ENDL;
}
}
if (!is_cache_obsolete)
{
const LLSD& llsd_cats = inventory["categories"];
if (llsd_cats.isArray())
{
LLSD::array_const_iterator iter = llsd_cats.beginArray();
LLSD::array_const_iterator end = llsd_cats.endArray();
for (; iter != end; ++iter)
{
LLPointer<LLViewerInventoryCategory> inv_cat = new LLViewerInventoryCategory(LLUUID::null);
if(inv_cat->importLLSD(s_item))
if (inv_cat->importLLSDMap(*iter))
{
categories.push_back(inv_cat);
}
}
else if (s_item.has("item_id"))
{
if (is_cache_obsolete)
break;
}
LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem;
if( inv_item->fromLLSD(s_item) )
const LLSD& llsd_items = inventory["items"];
if (llsd_items.isArray())
{
if(inv_item->getUUID().isNull())
LLSD::array_const_iterator iter = llsd_items.beginArray();
LLSD::array_const_iterator end = llsd_items.endArray();
for (; iter != end; ++iter)
{
LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem;
if (inv_item->fromLLSD(*iter))
{
if (inv_item->getUUID().isNull())
{
LL_DEBUGS(LOG_INV) << "Ignoring inventory with null item id: "
<< inv_item->getName() << LL_ENDL;
@ -3458,15 +3469,16 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
}
}
}
}
// TODO(brad) - figure out how to reenable this without breaking everything else
// static constexpr U64 BATCH_SIZE = 512U;
// if ((++lines_count % BATCH_SIZE) == 0)
// {
// // SL-19968 - make sure message system code gets a chance to run every so often
// pump_idle_startup_network();
// }
// TODO(brad) - figure out how to reenable this without breaking everything else
// static constexpr U64 BATCH_SIZE = 512U;
// if ((++lines_count % BATCH_SIZE) == 0)
// {
// // SL-19968 - make sure message system code gets a chance to run every so often
// pump_idle_startup_network();
// }
}
}
}
file.close();
@ -3489,58 +3501,54 @@ bool LLInventoryModel::saveToFile(const std::string& filename,
try
{
llofstream fileXML(filename.c_str());
if (!fileXML.is_open())
llofstream fileSD(filename.c_str(), std::ios_base::out | std::ios_base::binary);
if (!fileSD.is_open())
{
LL_WARNS(LOG_INV) << "Failed to open file. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
LLSD cache_ver;
cache_ver["inv_cache_version"] = sCurrentInvCacheVersion;
if (fileXML.fail())
U32 value_nbo = htonl(sCurrentInvCacheVersion);
fileSD.write((const char*)(&value_nbo), sizeof(U32));
if (fileSD.fail())
{
LL_WARNS(LOG_INV) << "Failed to write cache version to file. Unable to save inventory to: " << filename << LL_ENDL;
LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
fileXML << LLSDOStreamer<LLSDNotationFormatter>(cache_ver) << std::endl;
LLSD inventory;
inventory["categories"] = LLSD::emptyArray();
LLSD& cat_array = inventory["categories"];
S32 cat_count = 0;
for (auto& cat : categories)
{
if (cat->getVersion() != LLViewerInventoryCategory::VERSION_UNKNOWN)
{
LLSD sd = LLSD::emptyMap();
LLSD sd;
cat->exportLLSD(sd);
fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl;
cat_array.append(sd);
cat_count++;
}
if (fileXML.fail())
{
LL_WARNS(LOG_INV) << "Failed to write a folder to file. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
}
inventory["items"] = LLSD::emptyArray();
LLSD& item_array = inventory["items"];
auto it_count = items.size();
for (auto& item : items)
{
LLSD sd = LLSD::emptyMap();
LLSD sd;
item->asLLSD(sd);
fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl;
if (fileXML.fail())
item_array.append(sd);
}
fileSD << LLSDOStreamer<LLSDBinaryFormatter>(inventory) << std::endl;
if (fileSD.fail())
{
LL_WARNS(LOG_INV) << "Failed to write an item to file. Unable to save inventory to: " << filename << LL_ENDL;
LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
}
fileXML.flush();
fileSD.flush();
fileXML.close();
fileSD.close();
LL_INFOS(LOG_INV) << "Inventory saved: " << (S32)cat_count << " categories, " << (S32)it_count << " items." << LL_ENDL;
}

23
indra/newview/llviewerinventory.cpp
View File

@ -761,18 +761,23 @@ void LLViewerInventoryCategory::exportLLSD(LLSD & cat_data) const
cat_data[INV_VERSION] = mVersion;
}
bool LLViewerInventoryCategory::importLLSD(const LLSD& cat_data)
bool LLViewerInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
{
LLInventoryCategory::importLLSD(cat_data);
if (cat_data.has(INV_OWNER_ID))
if (LLInventoryCategory::importLLSD(label, value))
{
mOwnerID = cat_data[INV_OWNER_ID].asUUID();
}
if (cat_data.has(INV_VERSION))
{
setVersion(cat_data[INV_VERSION].asInteger());
}
return true;
}
else if (label == INV_OWNER_ID)
{
mOwnerID = value.asUUID();
return true;
}
else if (label == INV_VERSION)
{
setVersion(value.asInteger());
return true;
}
return false;
}
bool LLViewerInventoryCategory::acceptItem(LLInventoryItem* inv_item)

2
indra/newview/llviewerinventory.h
View File

@ -233,7 +233,7 @@ public:
S32 getViewerDescendentCount() const;
virtual void exportLLSD(LLSD &sd) const;
virtual bool importLLSD(const LLSD& cat_data);
virtual bool importLLSD(const std::string& label, const LLSD& value);
void determineFolderType();
void changeType(LLFolderType::EType new_folder_type);