livie
/
phoenix-firestorm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#3969 Make inventory creation from cache faster 

Still has a lot of space for improvements, but should be 2.5 times faster
master
Andrey Kleshchev 2025-07-25 19:06:06 +03:00 committed by Andrey Kleshchev
parent 329e71a7d9
commit 3b8b408b90
8 changed files with 221 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
indra/llinventory/llinventory.cpp
View File

@ -995,6 +995,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
// TODO - figure out if this should be moved into the noclobber fields above // TODO - figure out if this should be moved into the noclobber fields above
mThumbnailUUID.setNull(); mThumbnailUUID.setNull();
mFavorite = false; mFavorite = false;
mPermissions.init(LLUUID::null, LLUUID::null, LLUUID::null, LLUUID::null);
// iterate as map to avoid making unnecessary temp copies of everything // iterate as map to avoid making unnecessary temp copies of everything
LLSD::map_const_iterator i, end; LLSD::map_const_iterator i, end;
@ -1053,7 +1054,7 @@ bool LLInventoryItem::fromLLSD(const LLSD& sd, bool is_new)
if (i->first == INV_PERMISSIONS_LABEL) if (i->first == INV_PERMISSIONS_LABEL)
{ {
mPermissions = ll_permissions_from_sd(i->second); mPermissions.importLLSD(i->second);
continue; continue;
} }
@ -1522,53 +1523,68 @@ void LLInventoryCategory::exportLLSD(LLSD& cat_data) const
} }
} }
bool LLInventoryCategory::importLLSD(const LLSD& cat_data) bool LLInventoryCategory::importLLSDMap(const LLSD& cat_data)
{ {
if (cat_data.has(INV_FOLDER_ID_LABEL)) LLSD::map_const_iterator i, end;
end = cat_data.endMap();
for ( i = cat_data.beginMap(); i != end; ++i)
{ {
setUUID(cat_data[INV_FOLDER_ID_LABEL].asUUID()); importLLSD(i->first, i->second);
} }
if (cat_data.has(INV_PARENT_ID_LABEL)) return true;
}
bool LLInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
{
if (label == INV_FOLDER_ID_LABEL)
{ {
setParent(cat_data[INV_PARENT_ID_LABEL].asUUID()); setUUID(value.asUUID());
return true;
} }
if (cat_data.has(INV_ASSET_TYPE_LABEL)) else if (label == INV_PARENT_ID_LABEL)
{ {
setType(LLAssetType::lookup(cat_data[INV_ASSET_TYPE_LABEL].asString())); setParent(value.asUUID());
return true;
} }
if (cat_data.has(INV_PREFERRED_TYPE_LABEL)) else if (label == INV_ASSET_TYPE_LABEL)
{ {
setPreferredType(LLFolderType::lookup(cat_data[INV_PREFERRED_TYPE_LABEL].asString())); setType(LLAssetType::lookup(value.asString()));
return true;
} }
if (cat_data.has(INV_THUMBNAIL_LABEL)) else if (label == INV_PREFERRED_TYPE_LABEL)
{
setPreferredType(LLFolderType::lookup(value.asString()));
return true;
}
else if (label == INV_THUMBNAIL_LABEL)
{ {
LLUUID thumbnail_uuid; LLUUID thumbnail_uuid;
const LLSD &thumbnail_data = cat_data[INV_THUMBNAIL_LABEL]; if (value.has(INV_ASSET_ID_LABEL))
if (thumbnail_data.has(INV_ASSET_ID_LABEL))
{ {
thumbnail_uuid = thumbnail_data[INV_ASSET_ID_LABEL].asUUID(); thumbnail_uuid = value[INV_ASSET_ID_LABEL].asUUID();
} }
setThumbnailUUID(thumbnail_uuid); setThumbnailUUID(thumbnail_uuid);
return true;
} }
if (cat_data.has(INV_FAVORITE_LABEL)) if (label == INV_FAVORITE_LABEL)
{ {
bool favorite = false; bool favorite = false;
const LLSD& favorite_data = cat_data[INV_FAVORITE_LABEL]; if (value.has(INV_TOGGLED_LABEL))
if (favorite_data.has(INV_TOGGLED_LABEL))
{ {
favorite = favorite_data[INV_TOGGLED_LABEL].asBoolean(); favorite = value[INV_TOGGLED_LABEL].asBoolean();
} }
setFavorite(favorite); setFavorite(favorite);
} }
if (cat_data.has(INV_NAME_LABEL)) else if (label == INV_NAME_LABEL)
{ {
mName = cat_data[INV_NAME_LABEL].asString(); mName = value.asString();
LLStringUtil::replaceNonstandardASCII(mName, ' '); LLStringUtil::replaceNonstandardASCII(mName, ' ');
LLStringUtil::replaceChar(mName, '|', ' '); LLStringUtil::replaceChar(mName, '|', ' ');
return true;
} }
return false;
return true;
} }
///---------------------------------------------------------------------------- ///----------------------------------------------------------------------------
/// Local function definitions for testing purposes /// Local function definitions for testing purposes
///---------------------------------------------------------------------------- ///----------------------------------------------------------------------------

3
indra/llinventory/llinventory.h
View File

@ -274,7 +274,8 @@ public:
virtual bool exportLegacyStream(std::ostream& output_stream, bool include_asset_key = true) const; virtual bool exportLegacyStream(std::ostream& output_stream, bool include_asset_key = true) const;
virtual void exportLLSD(LLSD& sd) const; virtual void exportLLSD(LLSD& sd) const;
bool importLLSD(const LLSD& cat_data); bool importLLSDMap(const LLSD& cat_data);
virtual bool importLLSD(const std::string& label, const LLSD& value);
//-------------------------------------------------------------------- //--------------------------------------------------------------------
// Member Variables // Member Variables
//-------------------------------------------------------------------- //--------------------------------------------------------------------

104
indra/llinventory/llpermissions.cpp
View File

@ -704,6 +704,79 @@ bool LLPermissions::exportLegacyStream(std::ostream& output_stream) const
return true; return true;
} }
static const std::string PERM_CREATOR_ID_LABEL("creator_id");
static const std::string PERM_OWNER_ID_LABEL("owner_id");
static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
static const std::string PERM_GROUP_ID_LABEL("group_id");
static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
static const std::string PERM_BASE_MASK_LABEL("base_mask");
static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
static const std::string PERM_GROUP_MASK_LABEL("group_mask");
static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
void LLPermissions::importLLSD(const LLSD& sd_perm)
{
LLSD::map_const_iterator i, end;
end = sd_perm.endMap();
for (i = sd_perm.beginMap(); i != end; ++i)
{
const std::string& label = i->first;
if (label == PERM_CREATOR_ID_LABEL)
{
mCreator = i->second.asUUID();
continue;
}
if (label == PERM_OWNER_ID_LABEL)
{
mOwner = i->second.asUUID();
continue;
}
if (label == PERM_LAST_OWNER_ID_LABEL)
{
mLastOwner = i->second.asUUID();
continue;
}
if (label == PERM_GROUP_ID_LABEL)
{
mGroup = i->second.asUUID();
continue;
}
if (label == PERM_BASE_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskBase = mask;
continue;
}
if (label == PERM_OWNER_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskOwner = mask;
continue;
}
if (label == PERM_EVERYONE_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskEveryone = mask;
continue;
}
if (label == PERM_GROUP_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskGroup = mask;
continue;
}
if (label == PERM_NEXT_OWNER_MASK_LABEL)
{
PermissionMask mask = i->second.asInteger();
mMaskNextOwner = mask;
continue;
}
}
fix();
}
bool LLPermissions::operator==(const LLPermissions &rhs) const bool LLPermissions::operator==(const LLPermissions &rhs) const
{ {
return return
@ -998,16 +1071,6 @@ std::string mask_to_string(U32 mask)
///---------------------------------------------------------------------------- ///----------------------------------------------------------------------------
/// exported functions /// exported functions
///---------------------------------------------------------------------------- ///----------------------------------------------------------------------------
static const std::string PERM_CREATOR_ID_LABEL("creator_id");
static const std::string PERM_OWNER_ID_LABEL("owner_id");
static const std::string PERM_LAST_OWNER_ID_LABEL("last_owner_id");
static const std::string PERM_GROUP_ID_LABEL("group_id");
static const std::string PERM_IS_OWNER_GROUP_LABEL("is_owner_group");
static const std::string PERM_BASE_MASK_LABEL("base_mask");
static const std::string PERM_OWNER_MASK_LABEL("owner_mask");
static const std::string PERM_GROUP_MASK_LABEL("group_mask");
static const std::string PERM_EVERYONE_MASK_LABEL("everyone_mask");
static const std::string PERM_NEXT_OWNER_MASK_LABEL("next_owner_mask");
LLSD ll_create_sd_from_permissions(const LLPermissions& perm) LLSD ll_create_sd_from_permissions(const LLPermissions& perm)
{ {
@ -1032,25 +1095,6 @@ void ll_fill_sd_from_permissions(LLSD& rv, const LLPermissions& perm)
LLPermissions ll_permissions_from_sd(const LLSD& sd_perm) LLPermissions ll_permissions_from_sd(const LLSD& sd_perm)
{ {
LLPermissions rv; LLPermissions rv;
rv.init( rv.importLLSD(sd_perm);
sd_perm[PERM_CREATOR_ID_LABEL].asUUID(),
sd_perm[PERM_OWNER_ID_LABEL].asUUID(),
sd_perm[PERM_LAST_OWNER_ID_LABEL].asUUID(),
sd_perm[PERM_GROUP_ID_LABEL].asUUID());
// We do a cast to U32 here since LLSD does not attempt to
// represent unsigned ints.
PermissionMask mask;
mask = (U32)(sd_perm[PERM_BASE_MASK_LABEL].asInteger());
rv.setMaskBase(mask);
mask = (U32)(sd_perm[PERM_OWNER_MASK_LABEL].asInteger());
rv.setMaskOwner(mask);
mask = (U32)(sd_perm[PERM_EVERYONE_MASK_LABEL].asInteger());
rv.setMaskEveryone(mask);
mask = (U32)(sd_perm[PERM_GROUP_MASK_LABEL].asInteger());
rv.setMaskGroup(mask);
mask = (U32)(sd_perm[PERM_NEXT_OWNER_MASK_LABEL].asInteger());
rv.setMaskNext(mask);
rv.fix();
return rv; return rv;
} }

2
indra/llinventory/llpermissions.h
View File

@ -299,6 +299,8 @@ public:
bool importLegacyStream(std::istream& input_stream); bool importLegacyStream(std::istream& input_stream);
bool exportLegacyStream(std::ostream& output_stream) const; bool exportLegacyStream(std::ostream& output_stream) const;
void importLLSD(const LLSD& sd_perm);
bool operator==(const LLPermissions &rhs) const; bool operator==(const LLPermissions &rhs) const;
bool operator!=(const LLPermissions &rhs) const; bool operator!=(const LLPermissions &rhs) const;

2
indra/llinventory/tests/inventorymisc_test.cpp
View File

@ -518,7 +518,7 @@ namespace tut
file.close(); file.close();
LLPointer<LLInventoryCategory> src2 = new LLInventoryCategory(); LLPointer<LLInventoryCategory> src2 = new LLInventoryCategory();
src2->importLLSD(s_item); src2->importLLSDMap(s_item);
ensure_equals("1.item id::getUUID() failed", src1->getUUID(), src2->getUUID()); ensure_equals("1.item id::getUUID() failed", src1->getUUID(), src2->getUUID());
ensure_equals("2.parent::getParentUUID() failed", src1->getParentUUID(), src2->getParentUUID()); ensure_equals("2.parent::getParentUUID() failed", src1->getParentUUID(), src2->getParentUUID());

174
indra/newview/llinventorymodel.cpp
View File

@ -78,7 +78,7 @@
// Increment this if the inventory contents change in a non-backwards-compatible way. // Increment this if the inventory contents change in a non-backwards-compatible way.
// For viewer 2, the addition of link items makes a pre-viewer-2 cache incorrect. // For viewer 2, the addition of link items makes a pre-viewer-2 cache incorrect.
const S32 LLInventoryModel::sCurrentInvCacheVersion = 4; const S32 LLInventoryModel::sCurrentInvCacheVersion = 5;
bool LLInventoryModel::sFirstTimeInViewer2 = true; bool LLInventoryModel::sFirstTimeInViewer2 = true;
S32 LLInventoryModel::sPendingSystemFolders = 0; S32 LLInventoryModel::sPendingSystemFolders = 0;
@ -3384,7 +3384,7 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
} }
LL_INFOS(LOG_INV) << "loading inventory from: (" << filename << ")" << LL_ENDL; LL_INFOS(LOG_INV) << "loading inventory from: (" << filename << ")" << LL_ENDL;
llifstream file(filename.c_str()); llifstream file(filename.c_str(), std::ifstream::in | std::ifstream::binary);
if (!file.is_open()) if (!file.is_open())
{ {
@ -3393,80 +3393,92 @@ bool LLInventoryModel::loadFromFile(const std::string& filename,
} }
is_cache_obsolete = true; // Obsolete until proven current is_cache_obsolete = true; // Obsolete until proven current
U32 value_nbo = 0;
//U64 lines_count = 0U; file.read((char*)&value_nbo, sizeof(U32));
std::string line; if (file.fail())
LLPointer<LLSDParser> parser = new LLSDNotationParser();
while (std::getline(file, line))
{ {
LLSD s_item; LL_WARNS(LOG_INV) << "Failed to read cache version. Unable to load inventory from: " << filename << LL_ENDL;
std::istringstream iss(line); }
if (parser->parse(iss, s_item, line.length()) == LLSDParser::PARSE_FAILURE) else
{
S32 version = (S32)ntohl(value_nbo);
if (version == sCurrentInvCacheVersion)
{ {
LL_WARNS(LOG_INV)<< "Parsing inventory cache failed" << LL_ENDL; // Cache is up to date
break; is_cache_obsolete = false;
} }
else
if (s_item.has("inv_cache_version"))
{ {
S32 version = s_item["inv_cache_version"].asInteger(); LL_WARNS(LOG_INV) << "Inventory cache is out of date" << LL_ENDL;
if (version == sCurrentInvCacheVersion)
{
// Cache is up to date
is_cache_obsolete = false;
continue;
}
else
{
LL_WARNS(LOG_INV)<< "Inventory cache is out of date" << LL_ENDL;
break;
}
} }
else if (s_item.has("cat_id")) }
{
if (is_cache_obsolete)
break;
LLPointer<LLViewerInventoryCategory> inv_cat = new LLViewerInventoryCategory(LLUUID::null); LLSD inventory;
if(inv_cat->importLLSD(s_item)) if (!is_cache_obsolete)
{ {
categories.push_back(inv_cat); LLPointer<LLSDParser> parser = new LLSDBinaryParser();
}
if (parser->parse(file, inventory, LLSDSerialize::SIZE_UNLIMITED) == LLSDParser::PARSE_FAILURE)
{
is_cache_obsolete = true;
LL_WARNS(LOG_INV) << "Parsing inventory cache failed" << LL_ENDL;
} }
else if (s_item.has("item_id")) }
{
if (is_cache_obsolete)
break;
LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem; if (!is_cache_obsolete)
if( inv_item->fromLLSD(s_item) ) {
const LLSD& llsd_cats = inventory["categories"];
if (llsd_cats.isArray())
{
LLSD::array_const_iterator iter = llsd_cats.beginArray();
LLSD::array_const_iterator end = llsd_cats.endArray();
for (; iter != end; ++iter)
{ {
if(inv_item->getUUID().isNull()) LLPointer<LLViewerInventoryCategory> inv_cat = new LLViewerInventoryCategory(LLUUID::null);
if (inv_cat->importLLSDMap(*iter))
{ {
LL_DEBUGS(LOG_INV) << "Ignoring inventory with null item id: " categories.push_back(inv_cat);
<< inv_item->getName() << LL_ENDL;
} }
else }
}
const LLSD& llsd_items = inventory["items"];
if (llsd_items.isArray())
{
LLSD::array_const_iterator iter = llsd_items.beginArray();
LLSD::array_const_iterator end = llsd_items.endArray();
for (; iter != end; ++iter)
{
LLPointer<LLViewerInventoryItem> inv_item = new LLViewerInventoryItem;
if (inv_item->fromLLSD(*iter))
{ {
if (inv_item->getType() == LLAssetType::AT_UNKNOWN) if (inv_item->getUUID().isNull())
{ {
cats_to_update.insert(inv_item->getParentUUID()); LL_DEBUGS(LOG_INV) << "Ignoring inventory with null item id: "
<< inv_item->getName() << LL_ENDL;
} }
else else
{ {
items.push_back(inv_item); if (inv_item->getType() == LLAssetType::AT_UNKNOWN)
{
cats_to_update.insert(inv_item->getParentUUID());
}
else
{
items.push_back(inv_item);
}
} }
} }
// TODO(brad) - figure out how to reenable this without breaking everything else
// static constexpr U64 BATCH_SIZE = 512U;
// if ((++lines_count % BATCH_SIZE) == 0)
// {
// // SL-19968 - make sure message system code gets a chance to run every so often
// pump_idle_startup_network();
// }
} }
} }
// TODO(brad) - figure out how to reenable this without breaking everything else
// static constexpr U64 BATCH_SIZE = 512U;
// if ((++lines_count % BATCH_SIZE) == 0)
// {
// // SL-19968 - make sure message system code gets a chance to run every so often
// pump_idle_startup_network();
// }
} }
file.close(); file.close();
@ -3489,58 +3501,54 @@ bool LLInventoryModel::saveToFile(const std::string& filename,
try try
{ {
llofstream fileXML(filename.c_str()); llofstream fileSD(filename.c_str(), std::ios_base::out | std::ios_base::binary);
if (!fileXML.is_open()) if (!fileSD.is_open())
{ {
LL_WARNS(LOG_INV) << "Failed to open file. Unable to save inventory to: " << filename << LL_ENDL; LL_WARNS(LOG_INV) << "Failed to open file. Unable to save inventory to: " << filename << LL_ENDL;
return false; return false;
} }
U32 value_nbo = htonl(sCurrentInvCacheVersion);
LLSD cache_ver; fileSD.write((const char*)(&value_nbo), sizeof(U32));
cache_ver["inv_cache_version"] = sCurrentInvCacheVersion; if (fileSD.fail())
if (fileXML.fail())
{ {
LL_WARNS(LOG_INV) << "Failed to write cache version to file. Unable to save inventory to: " << filename << LL_ENDL; LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
return false; return false;
} }
fileXML << LLSDOStreamer<LLSDNotationFormatter>(cache_ver) << std::endl; LLSD inventory;
inventory["categories"] = LLSD::emptyArray();
LLSD& cat_array = inventory["categories"];
S32 cat_count = 0; S32 cat_count = 0;
for (auto& cat : categories) for (auto& cat : categories)
{ {
if (cat->getVersion() != LLViewerInventoryCategory::VERSION_UNKNOWN) if (cat->getVersion() != LLViewerInventoryCategory::VERSION_UNKNOWN)
{ {
LLSD sd = LLSD::emptyMap(); LLSD sd;
cat->exportLLSD(sd); cat->exportLLSD(sd);
fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl; cat_array.append(sd);
cat_count++; cat_count++;
} }
if (fileXML.fail())
{
LL_WARNS(LOG_INV) << "Failed to write a folder to file. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
} }
inventory["items"] = LLSD::emptyArray();
LLSD& item_array = inventory["items"];
auto it_count = items.size(); auto it_count = items.size();
for (auto& item : items) for (auto& item : items)
{ {
LLSD sd = LLSD::emptyMap(); LLSD sd;
item->asLLSD(sd); item->asLLSD(sd);
fileXML << LLSDOStreamer<LLSDNotationFormatter>(sd) << std::endl; item_array.append(sd);
if (fileXML.fail())
{
LL_WARNS(LOG_INV) << "Failed to write an item to file. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
} }
fileXML.flush(); fileSD << LLSDOStreamer<LLSDBinaryFormatter>(inventory) << std::endl;
if (fileSD.fail())
{
LL_WARNS(LOG_INV) << "Failed to write cache. Unable to save inventory to: " << filename << LL_ENDL;
return false;
}
fileSD.flush();
fileXML.close(); fileSD.close();
LL_INFOS(LOG_INV) << "Inventory saved: " << (S32)cat_count << " categories, " << (S32)it_count << " items." << LL_ENDL; LL_INFOS(LOG_INV) << "Inventory saved: " << (S32)cat_count << " categories, " << (S32)it_count << " items." << LL_ENDL;
} }

19
indra/newview/llviewerinventory.cpp
View File

@ -761,18 +761,23 @@ void LLViewerInventoryCategory::exportLLSD(LLSD & cat_data) const
cat_data[INV_VERSION] = mVersion; cat_data[INV_VERSION] = mVersion;
} }
bool LLViewerInventoryCategory::importLLSD(const LLSD& cat_data) bool LLViewerInventoryCategory::importLLSD(const std::string& label, const LLSD& value)
{ {
LLInventoryCategory::importLLSD(cat_data); if (LLInventoryCategory::importLLSD(label, value))
if (cat_data.has(INV_OWNER_ID))
{ {
mOwnerID = cat_data[INV_OWNER_ID].asUUID(); return true;
} }
if (cat_data.has(INV_VERSION)) else if (label == INV_OWNER_ID)
{ {
setVersion(cat_data[INV_VERSION].asInteger()); mOwnerID = value.asUUID();
return true;
} }
return true; else if (label == INV_VERSION)
{
setVersion(value.asInteger());
return true;
}
return false;
} }
bool LLViewerInventoryCategory::acceptItem(LLInventoryItem* inv_item) bool LLViewerInventoryCategory::acceptItem(LLInventoryItem* inv_item)

2
indra/newview/llviewerinventory.h
View File

@ -233,7 +233,7 @@ public:
S32 getViewerDescendentCount() const; S32 getViewerDescendentCount() const;
virtual void exportLLSD(LLSD &sd) const; virtual void exportLLSD(LLSD &sd) const;
virtual bool importLLSD(const LLSD& cat_data); virtual bool importLLSD(const std::string& label, const LLSD& value);
void determineFolderType(); void determineFolderType();
void changeType(LLFolderType::EType new_folder_type); void changeType(LLFolderType::EType new_folder_type);