diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 82a9a968b9..35ac41420c 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -15,8 +15,8 @@ jobs: - uses: actions/stale@v6 id: stale with: - stale-pr-message: This pull request is stale because it has been open 60 days with no activity. Remove stale label or comment or it will be closed in 7 days - days-before-stale: 60 + stale-pr-message: This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or it will be closed in 7 days + days-before-stale: 30 days-before-close: 7 exempt-pr-labels: blocked,must,should,keep stale-pr-label: stale diff --git a/autobuild.xml b/autobuild.xml index c696a68f4c..5def9acc48 100644 --- a/autobuild.xml +++ b/autobuild.xml @@ -3353,6 +3353,36 @@ Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors version 0.54.1.555529 + xxhash + + copyright + Copyright 2012-2020 Yann Collet + description + xxHash Extremely fast hash algorithm + license + bsd + license_file + LICENSES/xxhash.txt + name + xxhash + platforms + + common + + archive + + hash + e4f77ba0a9b8ec3cc3fabc51c4da81d2 + url + https://automated-builds-secondlife-com.s3.amazonaws.com/ct2/110070/956941/xxhash-0.8.1.578006-windows-578006.tar.bz2 + + name + common + + + version + 0.8.1 + zlib-ng canonical_repo diff --git a/doc/contributions.txt b/doc/contributions.txt index e1e2043552..67f1a5796f 100755 --- a/doc/contributions.txt +++ b/doc/contributions.txt @@ -593,6 +593,8 @@ Henri Beauchamp VWR-1406 VWR-4157 SL-15175 + SL-19110 + SL-19159 herina Bode Hikkoshi Sakai VWR-429 @@ -1187,6 +1189,7 @@ PanteraPolnocy SL-18891 SL-18904 SL-18937 + SL-19207 Parvati Silverweb Patric Mills VWR-2645 diff --git a/indra/cmake/CMakeLists.txt b/indra/cmake/CMakeLists.txt index 7bd4dce8fc..94f1c29493 100644 --- a/indra/cmake/CMakeLists.txt +++ b/indra/cmake/CMakeLists.txt @@ -96,6 +96,7 @@ set(cmake_SOURCE_FILES VisualLeakDetector.cmake LibVLCPlugin.cmake XmlRpcEpi.cmake + xxHash.cmake ZLIBNG.cmake ) diff --git a/indra/cmake/LLCommon.cmake b/indra/cmake/LLCommon.cmake index 53871791fd..528b43c3fc 100644 --- a/indra/cmake/LLCommon.cmake +++ b/indra/cmake/LLCommon.cmake @@ -4,6 +4,7 @@ include(APR) include(Boost) include(EXPAT) include(Tracy) +include(xxHash) include(ZLIBNG) set(LLCOMMON_INCLUDE_DIRS diff --git a/indra/cmake/xxHash.cmake b/indra/cmake/xxHash.cmake new file mode 100644 index 0000000000..a7c1cba62c --- /dev/null +++ b/indra/cmake/xxHash.cmake @@ -0,0 +1,8 @@ +# -*- cmake -*- +if (XXHASH_CMAKE_INCLUDED) + return() +endif (XXHASH_CMAKE_INCLUDED) +set (XXHASH_CMAKE_INCLUDED TRUE) + +include(Prebuilt) +use_prebuilt_binary(xxhash) diff --git a/indra/llappearance/llavatarappearance.cpp b/indra/llappearance/llavatarappearance.cpp index fcda8fa8d6..062a59ed92 100644 --- a/indra/llappearance/llavatarappearance.cpp +++ b/indra/llappearance/llavatarappearance.cpp @@ -1086,7 +1086,6 @@ BOOL LLAvatarAppearance::loadSkeletonNode () // SKELETAL DISTORTIONS { - LLAvatarXmlInfo::skeletal_distortion_info_list_t::iterator iter; for (LLViewerVisualParamInfo* visual_param_info : sAvatarXmlInfo->mSkeletalDistortionInfoList) { LLPolySkeletalDistortionInfo *info = (LLPolySkeletalDistortionInfo*)visual_param_info; diff --git a/indra/llappearance/llpolymorph.cpp b/indra/llappearance/llpolymorph.cpp index 98b01ee708..4cf422089d 100644 --- a/indra/llappearance/llpolymorph.cpp +++ b/indra/llappearance/llpolymorph.cpp @@ -365,7 +365,6 @@ BOOL LLPolyMorphTarget::setInfo(LLPolyMorphTargetInfo* info) setWeight(getDefaultWeight(), FALSE); LLAvatarAppearance* avatarp = mMesh->getAvatar(); - LLPolyMorphTargetInfo::volume_info_list_t::iterator iter; for (LLPolyVolumeMorphInfo& volume_info : getInfo()->mVolumeInfoList) { for (S32 i = 0; i < avatarp->mNumCollisionVolumes; i++) diff --git a/indra/llappearance/llpolyskeletaldistortion.cpp b/indra/llappearance/llpolyskeletaldistortion.cpp index 4d45f78509..0d51ea0577 100644 --- a/indra/llappearance/llpolyskeletaldistortion.cpp +++ b/indra/llappearance/llpolyskeletaldistortion.cpp @@ -145,7 +145,6 @@ BOOL LLPolySkeletalDistortion::setInfo(LLPolySkeletalDistortionInfo *info) //setWeight(getDefaultWeight()); setWeight(getDefaultWeight(), FALSE); - LLPolySkeletalDistortionInfo::bone_info_list_t::iterator iter; for (LLPolySkeletalBoneInfo& bone_info : getInfo()->mBoneInfoList) { LLJoint* joint = mAvatar->getJoint(bone_info.mBoneName); diff --git a/indra/llappearance/llwearable.h b/indra/llappearance/llwearable.h index a63b38361b..e2b8a7068c 100644 --- a/indra/llappearance/llwearable.h +++ b/indra/llappearance/llwearable.h @@ -32,7 +32,7 @@ #include "llsaleinfo.h" #include "llwearabletype.h" -class LLMD5; +class LLMD5; // [Legacy Bake] class LLVisualParam; class LLTexGlobalColorInfo; class LLTexGlobalColor; @@ -131,6 +131,7 @@ public: // Something happened that requires the wearable to be updated (e.g. worn/unworn). virtual void setUpdated() const = 0; + // [Legacy Bake] // Update the baked texture hash. virtual void addToBakedTextureHash(LLMD5& hash) const = 0; diff --git a/indra/llappearance/llwearabledata.cpp b/indra/llappearance/llwearabledata.cpp index 5d23d3a876..bc3d83ee1e 100644 --- a/indra/llappearance/llwearabledata.cpp +++ b/indra/llappearance/llwearabledata.cpp @@ -31,7 +31,7 @@ #include "llavatarappearance.h" #include "llavatarappearancedefines.h" #include "lldriverparam.h" -#include "llmd5.h" +#include "llmd5.h" // [Legacy Bake] LLWearableData::LLWearableData() : mAvatarAppearance(NULL) @@ -355,6 +355,7 @@ U32 LLWearableData::getWearableCount(const U32 tex_index) const return getWearableCount(wearable_type); } +// [Legacy Bake] LLUUID LLWearableData::computeBakedTextureHash(LLAvatarAppearanceDefines::EBakedTextureIndex baked_index, BOOL generate_valid_hash) // Set to false if you want to upload the baked texture w/o putting it in the cache { @@ -391,5 +392,6 @@ LLUUID LLWearableData::computeBakedTextureHash(LLAvatarAppearanceDefines::EBaked return hash_id; } +// [Legacy Bake] diff --git a/indra/llappearance/llwearabledata.h b/indra/llappearance/llwearabledata.h index cb274ca524..8ecf29a7a0 100644 --- a/indra/llappearance/llwearabledata.h +++ b/indra/llappearance/llwearabledata.h @@ -85,6 +85,7 @@ protected: private: void pullCrossWearableValues(const LLWearableType::EType type); +// [Legacy Bake] //-------------------------------------------------------------------- // Server Communication //-------------------------------------------------------------------- @@ -93,6 +94,7 @@ public: BOOL generate_valid_hash = TRUE); protected: virtual void invalidateBakedTextureHash(LLMD5& hash) const {} +// [Legacy Bake] //-------------------------------------------------------------------- // Member variables diff --git a/indra/llcharacter/llbvhloader.cpp b/indra/llcharacter/llbvhloader.cpp index 0e6b85732c..c87852123c 100644 --- a/indra/llcharacter/llbvhloader.cpp +++ b/indra/llcharacter/llbvhloader.cpp @@ -954,7 +954,6 @@ ELoadStatus LLBVHLoader::loadBVHFile(const char *buffer, char* error_text, S32 & //------------------------------------------------------------------------ void LLBVHLoader::applyTranslations() { - JointVector::iterator ji; for (Joint* joint : mJoints) { //---------------------------------------------------------------- @@ -1069,7 +1068,6 @@ void LLBVHLoader::optimize() mEaseOut *= factor; } - JointVector::iterator ji; for (Joint* joint : mJoints) { BOOL pos_changed = FALSE; diff --git a/indra/llcharacter/lljoint.cpp b/indra/llcharacter/lljoint.cpp index d8be3f8d9d..c6f9d96a40 100644 --- a/indra/llcharacter/lljoint.cpp +++ b/indra/llcharacter/lljoint.cpp @@ -742,7 +742,6 @@ void LLJoint::showAttachmentScaleOverrides(const std::string& av_info) const { LL_DEBUGS("Avatar") << "no conflicts" << LL_ENDL; } - std::set::iterator dit = distinct_offsets.begin(); for (const LLVector3& offset : distinct_offsets) { std::string highlight = (has_active_override && offset == active_override) ? "*" : ""; diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 1cc7838127..d3d7488956 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -118,6 +118,7 @@ set(llcommon_SOURCE_FILES lluriparser.cpp lluuid.cpp llworkerthread.cpp + hbxxh.cpp u64.cpp threadpool.cpp workqueue.cpp @@ -254,6 +255,7 @@ set(llcommon_HEADER_FILES llwin32headers.h llwin32headerslean.h llworkerthread.h + hbxxh.h lockstatic.h stdtypes.h stringize.h diff --git a/indra/llcommon/hbxxh.cpp b/indra/llcommon/hbxxh.cpp new file mode 100644 index 0000000000..388269d6c8 --- /dev/null +++ b/indra/llcommon/hbxxh.cpp @@ -0,0 +1,377 @@ +/** + * @file hbxxh.cpp + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +// This define ensures that xxHash will be compiled within this module, with +// vectorized (*) and inlined functions (with no exported API symbol); our +// xxhash "pre-built library" package actually only contains the xxhash.h +// header (no library needed at link time). +// (*) SSE2 is normally used for x86(_64) builds, unless you enabled AVX2 +// in your build, in which case the latter would be used instead. For ARM64 +// builds, this would also automatically enable NEON vectorization. +#define XXH_INLINE_ALL +#include "xxhash/xxhash.h" + +#include "hbxxh.h" + +// How many bytes to grab at a time when hashing files or streams +constexpr size_t BLOCK_LEN = 4096; + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH64 class +/////////////////////////////////////////////////////////////////////////////// + +//static +U64 HBXXH64::digest(const void* buffer, size_t len) +{ + return XXH3_64bits(buffer, len); +} + +//static +U64 HBXXH64::digest(const char* str) +{ + return XXH3_64bits((const void*)str, strlen(str)); +} + +//static +U64 HBXXH64::digest(const std::string& str) +{ + return XXH3_64bits((const void*)str.c_str(), str.size()); +} + +// Must be called by all constructors. +void HBXXH64::init() +{ + mDigest = 0; + mState = (void*)XXH3_createState(); + if (!mState || XXH3_64bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH64::~HBXXH64() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH64::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(const std::string& str) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH64::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH64::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + mDigest = XXH3_64bits_digest((XXH3_state_t*)mState); + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +U64 HBXXH64::digest() const +{ + return mState ? XXH3_64bits_digest((XXH3_state_t*)mState) : mDigest; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH64 context) +{ + stream << context.digest(); + return stream; +} + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH128 class +/////////////////////////////////////////////////////////////////////////////// + +//static +LLUUID HBXXH128::digest(const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +void HBXXH128::digest(LLUUID& result, const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +// Must be called by all constructors. +void HBXXH128::init() +{ + mState = (void*)XXH3_createState(); + if (!mState || XXH3_128bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH128::~HBXXH128() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH128::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(const std::string& str) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH128::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH128::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +const LLUUID& HBXXH128::digest() const +{ + if (mState) + { + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + // We cheat the const-ness of the method here, but this is OK, since + // mDigest is private and cannot be accessed indirectly by other + // methods than digest() ones, that do check for mState to decide + // wether mDigest's current value may be provided as is or not. This + // cheat saves us a temporary LLLUID copy. + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + } + return mDigest; +} + +void HBXXH128::digest(LLUUID& result) const +{ + if (!mState) + { + result = mDigest; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH128 context) +{ + stream << context.digest(); + return stream; +} diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h new file mode 100644 index 0000000000..9c0e9cf172 --- /dev/null +++ b/indra/llcommon/hbxxh.h @@ -0,0 +1,277 @@ +/** + * @file hbxxh.h + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_HBXXH_H +#define LL_HBXXH_H + +#include "lluuid.h" + +// HBXXH* classes are to be used where speed matters and cryptographic quality +// is not required (no "one-way" guarantee, though they are likely not worst in +// this respect than MD5 which got busted and is now considered too weak). The +// xxHash code they are built upon is vectorized and about 50 times faster than +// MD5. A 64 bits hash class is also provided for when 128 bits of entropy are +// not needed. The hashes collision rate is similar to MD5's. +// See https://github.com/Cyan4973/xxHash#readme for details. + +// 64 bits hashing class + +class HBXXH64 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH64); + +protected: + LOG_CLASS(HBXXH64); + +public: + inline HBXXH64() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH64(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH64(const HBXXH64&) noexcept = delete; + HBXXH64& operator=(const HBXXH64&) noexcept = delete; + + ~HBXXH64(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH64 before using digest(), and you may keep updating() it even after + // you got a first digest() (the next digest would of course change after + // any update). It is still useful to use finalize() when you do not want + // to store a final digest() result in a separate U64; after this method + // has been called, digest() simply returns mDigest value. + void finalize(); + + U64 digest() const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static U64 digest(const void* buffer, size_t len); + static U64 digest(const char* str); // str must be NUL-terminated + static U64 digest(const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + U64 mDigest; +}; + +inline bool operator==(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() != b.digest(); +} + +// 128 bits hashing class + +class HBXXH128 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH128); + +protected: + LOG_CLASS(HBXXH128); + +public: + inline HBXXH128() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH128(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH128(const HBXXH128&) noexcept = delete; + HBXXH128& operator=(const HBXXH128&) noexcept = delete; + + ~HBXXH128(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH128 before using digest(), and you may keep updating() it even + // after you got a first digest() (the next digest would of course change + // after any update). It is still useful to use finalize() when you do not + // want to store a final digest() result in a separate LLUUID; after this + // method has been called, digest() simply returns a reference on mDigest. + void finalize(); + + // We use an LLUUID for the digest, since this is a 128 bits wide native + // type available in the viewer code, making it easy to manipulate. It also + // allows to use HBXXH128 efficiently in LLUUID generate() and combine() + // methods. + const LLUUID& digest() const; + + // Here, we avoid an LLUUID copy whenever we already got one to store the + // result *and* we did not yet call finalize(). + void digest(LLUUID& result) const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static LLUUID digest(const void* buffer, size_t len); + static LLUUID digest(const char* str); // str must be NUL-terminated + static LLUUID digest(const std::string& str); + // Same as above, but saves you from an LLUUID copy when you already got + // one for storage use. + static void digest(LLUUID& result, const void* buffer, size_t len); + static void digest(LLUUID& result, const char* str); // str NUL-terminated + static void digest(LLUUID& result, const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + LLUUID mDigest; +}; + +inline bool operator==(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() != b.digest(); +} + +#endif // LL_HBXXH_H diff --git a/indra/llcommon/llallocator_heap_profile.cpp b/indra/llcommon/llallocator_heap_profile.cpp index f4008c40aa..d3c6f6dede 100644 --- a/indra/llcommon/llallocator_heap_profile.cpp +++ b/indra/llcommon/llallocator_heap_profile.cpp @@ -141,7 +141,6 @@ void LLAllocatorHeapProfile::parse(std::string const & prof_text) void LLAllocatorHeapProfile::dump(std::ostream & out) const { - lines_t::const_iterator i; for (const LLAllocatorHeapProfile::line& line : mLines) { out << line.mLiveCount << ": " << line.mLiveSize << '[' << line.mTotalCount << ": " << line.mTotalSize << "] @"; diff --git a/indra/llcommon/lldependencies.h b/indra/llcommon/lldependencies.h index fa54a944c8..950af4a4ad 100644 --- a/indra/llcommon/lldependencies.h +++ b/indra/llcommon/lldependencies.h @@ -514,16 +514,21 @@ public: // former broken behavior has finally been fixed -- and our builds // treat warnings as errors. { - for (typename const DepNodeMap::value_type& nm_pair : mNodes) + for (typename DepNodeMap::const_iterator nmi = mNodes.begin(), nmend = mNodes.end(); + nmi != nmend; ++nmi) { - vmap.insert(typename VertexMap::value_type(nm_pair.first, vmap.size())); - for (typename const KEY& after_k : nm_pair.second.after) + vmap.insert(typename VertexMap::value_type(nmi->first, vmap.size())); + for (typename DepNode::dep_set::const_iterator ai = nmi->second.after.begin(), + aend = nmi->second.after.end(); + ai != aend; ++ai) { - vmap.insert(typename VertexMap::value_type(after_k, vmap.size())); + vmap.insert(typename VertexMap::value_type(*ai, vmap.size())); } - for (typename const KEY& before_k : nm_pair.second.before) + for (typename DepNode::dep_set::const_iterator bi = nmi->second.before.begin(), + bend = nmi->second.before.end(); + bi != bend; ++bi) { - vmap.insert(typename VertexMap::value_type(before_k, vmap.size())); + vmap.insert(typename VertexMap::value_type(*bi, vmap.size())); } } } @@ -531,19 +536,24 @@ public: // all the known key dependencies to integer pairs. EdgeList edges; { - for (typename const DepNodeMap::value_type& nm_pair : mNodes) + for (typename DepNodeMap::const_iterator nmi = mNodes.begin(), nmend = mNodes.end(); + nmi != nmend; ++nmi) { - auto thisnode = vmap[nm_pair.first]; + auto thisnode = vmap[nmi->first]; // after dependencies: build edges from the named node to this one - for (typename const KEY& after_k : nm_pair.second.after) + for (typename DepNode::dep_set::const_iterator ai = nmi->second.after.begin(), + aend = nmi->second.after.end(); + ai != aend; ++ai) { - edges.push_back(EdgeList::value_type(vmap[after_k], thisnode)); + edges.push_back(EdgeList::value_type(vmap[*ai], thisnode)); } // before dependencies: build edges from this node to the // named one - for (typename const KEY& before_k : nm_pair.second.before) + for (typename DepNode::dep_set::const_iterator bi = nmi->second.before.begin(), + bend = nmi->second.before.end(); + bi != bend; ++bi) { - edges.push_back(EdgeList::value_type(thisnode, vmap[before_k])); + edges.push_back(EdgeList::value_type(thisnode, vmap[*bi])); } } } @@ -555,19 +565,21 @@ public: // and we're certain that the associated int values are distinct // indexes. The fact that they're not in order is irrelevant. KeyList vkeys(vmap.size()); - for (typename const VertexMap::value_type& vm_pair : vmap) + for (typename VertexMap::const_iterator vmi = vmap.begin(), vmend = vmap.end(); + vmi != vmend; ++vmi) { - vkeys[vm_pair.second] = vm_pair.first; + vkeys[vmi->second] = vmi->first; } // Walk the sorted output list, building the result into mCache so // we'll have it next time someone asks. mCache.clear(); - for (const size_t sv : sorted) + for (VertexList::const_iterator svi = sorted.begin(), svend = sorted.end(); + svi != svend; ++svi) { - // We're certain that vkeys[sv] exists. However, there might not + // We're certain that vkeys[*svi] exists. However, there might not // yet be a corresponding entry in mNodes. self_type* non_const_this(const_cast(this)); - typename DepNodeMap::iterator found = non_const_this->mNodes.find(vkeys[sv]); + typename DepNodeMap::iterator found = non_const_this->mNodes.find(vkeys[*svi]); if (found != non_const_this->mNodes.end()) { // Make an iterator of appropriate type. diff --git a/indra/llcommon/llinitparam.cpp b/indra/llcommon/llinitparam.cpp index 9d3394b4f7..d15bd2f619 100644 --- a/indra/llcommon/llinitparam.cpp +++ b/indra/llcommon/llinitparam.cpp @@ -316,7 +316,7 @@ namespace LLInitParam { // Ensure this param has not already been inspected bool duplicate = false; - for (const ParamDescriptorPtr ptr : block_data.mUnnamedParams) + for (const ParamDescriptorPtr &ptr : block_data.mUnnamedParams) { if (param_handle == ptr->mParamHandle) { diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index c87c0758fe..259f5bc505 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -204,30 +204,35 @@ public: LLSD packet(LLSDMap("pump", pump)("data", data)); std::ostringstream buffer; - buffer << LLSDNotationStreamer(packet); + // SL-18330: for large data blocks, it's much faster to parse binary + // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than + // directly calling LLSDBinaryFormatter because, unlike the latter, + // serialize() prepends the relevant header, needed by a general- + // purpose LLSD parser to distinguish binary from notation. + LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY, + LLSDFormatter::OPTIONS_NONE); /*==========================================================================*| // DEBUGGING ONLY: don't copy str() if we can avoid it. std::string strdata(buffer.str()); if (std::size_t(buffer.tellp()) != strdata.length()) { - LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != " + LL_ERRS("LLLeap") << "tellp() -> " << static_cast(buffer.tellp()) << " != " << "str().length() -> " << strdata.length() << LL_ENDL; } // DEBUGGING ONLY: reading back is terribly inefficient. std::istringstream readback(strdata); LLSD echo; - LLPointer parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(readback, echo, strdata.length())); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length())); + if (! parse_status) { - LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of " - << "LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of " + << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL; } if (! llsd_equals(echo, packet)) { - LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD " - << "than passed to LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD " + << "than passed to LLSDSerialize::serialize()" << LL_ENDL; } |*==========================================================================*/ @@ -314,9 +319,17 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; +#if 1 + // specifically require notation LLSD from child LLPointer parser(new LLSDNotationParser()); S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); if (parse_status == LLSDParser::PARSE_FAILURE) +#else + // SL-18330: accept any valid LLSD serialization format from child + // Unfortunately this runs into trouble we have not yet debugged. + bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); + if (! parse_status) +#endif { bad_protocol("unparseable LLSD data"); } diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 94fe04d5b8..68202833eb 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -48,6 +48,7 @@ #endif #include "lldate.h" +#include "llmemorystream.h" #include "llsd.h" #include "llstring.h" #include "lluri.h" @@ -64,6 +65,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation"); #define windowBits 15 #define ENABLE_ZLIB_GZIP 32 +// If we published this in llsdserialize.h, we could use it in the +// implementation of LLSDOStreamer's operator<<(). +template +void format_using(const LLSD& data, std::ostream& ostr, + LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY) +{ + LLPointer f{ new Formatter }; + f->format(data, ostr, options); +} + +template +S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1) +{ + LLPointer p{ new Parser }; + return p->parse(istr, data, max_bytes, max_depth); +} + /** * LLSDSerialize */ @@ -86,10 +104,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize f = new LLSDXMLFormatter; break; - case LLSD_NOTATION: - str << "\n"; - f = new LLSDNotationFormatter; - break; + case LLSD_NOTATION: + str << "\n"; + f = new LLSDNotationFormatter; + break; default: LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL; @@ -104,18 +122,31 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize // static bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) { - LLPointer p = NULL; char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ - int i; - int inbuf = 0; - bool legacy_no_header = false; bool fail_if_not_legacy = false; - std::string header; /* - * Get the first line before anything. + * Get the first line before anything. Don't read more than max_bytes: + * this get() overload reads no more than (count-1) bytes into the + * specified buffer. In the usual case when max_bytes exceeds + * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. */ - str.get(hdr_buf, MAX_HDR_LEN, '\n'); + str.get(hdr_buf, llmin(max_bytes+1, sizeof(hdr_buf)-1), '\n'); + auto inbuf = str.gcount(); + // https://en.cppreference.com/w/cpp/io/basic_istream/get + // When the get() above sees the specified delimiter '\n', it stops there + // without pulling it from the stream. If it turns out that the stream + // does NOT contain a header, and the content includes meaningful '\n', + // it's important to pull that into hdr_buf too. + if (inbuf < max_bytes && str.get(hdr_buf[inbuf])) + { + // got the delimiting '\n' + ++inbuf; + // None of the following requires that hdr_buf contain a final '\0' + // byte. We could store one if needed, since even the incremented + // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. + } + std::string header{ hdr_buf, narrow(inbuf) }; if (str.fail()) { str.clear(); @@ -123,79 +154,97 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) } if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ - { - legacy_no_header = true; - inbuf = (int)str.gcount(); + { // Create a LLSD XML parser, and parse the first chunk read above. + LLSDXMLParser x; + x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read + auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it + // Formally we should probably check (parsed != PARSE_FAILURE && + // parsed > 0), but since PARSE_FAILURE is -1, this suffices. + return (parsed > 0); } - else + + if (fail_if_not_legacy) { - if (fail_if_not_legacy) - goto fail; - /* - * Remove the newline chars - */ - for (i = 0; i < MAX_HDR_LEN; i++) - { - if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' || - hdr_buf[i] == '\n') - { - hdr_buf[i] = 0; - break; - } - } - header = hdr_buf; + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; + return false; + } - std::string::size_type start = std::string::npos; - std::string::size_type end = std::string::npos; - start = header.find_first_not_of(" header syntax + auto start = header.find_first_not_of("parsePart(hdr_buf, inbuf); // Parse the first part that was already read - x->parseLines(str, sd); // Parse the rest of it - delete x; - return true; - } - - if (header == LLSD_BINARY_HEADER) + if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER)) { - p = new LLSDBinaryParser; + return (parse_using(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_XML_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER)) { - p = new LLSDXMLParser; + return (parse_using(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_NOTATION_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER)) { - p = new LLSDNotationParser; + return (parse_using(str, sd, max_bytes-inbuf) > 0); } - else + else // no header we recognize { - LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL; + LLPointer p; + if (inbuf && hdr_buf[0] == '<') + { + // looks like XML + LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL; + p = new LLSDXMLParser; + } + else + { + // assume notation + LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + p = new LLSDNotationParser; + } + // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that + // data to whatever remains in 'str'. + LLMemoryStreamBuf already(reinterpret_cast(hdr_buf), inbuf); + cat_streambuf prebuff(&already, str.rdbuf()); + std::istream prepend(&prebuff); +#if 1 + return (p->parse(prepend, sd, max_bytes) > 0); +#else + // debugging the reconstituted 'prepend' stream + // allocate a buffer that we hope is big enough for the whole thing + std::vector wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes); + prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size())); + LLMemoryStream replay(reinterpret_cast(wholemsg.data()), prepend.gcount()); + auto success{ p->parse(replay, sd, prepend.gcount()) > 0 }; + { + LL_DEBUGS() << (success? "parsed: $$" : "failed: '") + << std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$" + << LL_ENDL; + } + return success; +#endif } - - if (p.notNull()) - { - p->parse(str, sd, max_bytes); - return true; - } - -fail: - LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; - return false; } /** @@ -2294,169 +2343,41 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, const U8* in, S32 free(result); return ZR_OK; } -// //This unzip function will only work with a gzip header and trailer - while the contents //of the actual compressed data is the same for either format (gzip vs zlib ), the headers //and trailers are different for the formats. U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size ) { - // Add non-allocating variants of unzip_llsd - // if (size == 0) - // { - // LL_WARNS() << "No data to unzip." << LL_ENDL; - // return NULL; - // } - - // U8* result = NULL; - // U32 cur_size = 0; - // z_stream strm; - - // const U32 CHUNK = 0x4000; - - // U8 *in = new(std::nothrow) U8[size]; - // if (in == NULL) - // { - // LL_WARNS() << "Memory allocation failure." << LL_ENDL; - // return NULL; - // } - // is.read((char*) in, size); - - // U8 out[CHUNK]; - - // strm.zalloc = Z_NULL; - // strm.zfree = Z_NULL; - // strm.opaque = Z_NULL; - // strm.avail_in = size; - // strm.next_in = in; - - // valid = true; // Default is all okay. - // S32 ret = inflateInit2(&strm, windowBits | ENABLE_ZLIB_GZIP ); - // do - // { - // strm.avail_out = CHUNK; - // strm.next_out = out; - // ret = inflate(&strm, Z_NO_FLUSH); - // if (ret == Z_STREAM_ERROR) - // { - // inflateEnd(&strm); - // // free(result); - // if( result ) - // free(result); - // delete [] in; - // in = NULL; result = NULL;// Or we get a double free aftr the while loop ... - // valid = false; - // } - - // switch (ret) - // { - // case Z_NEED_DICT: - // ret = Z_DATA_ERROR; - // case Z_DATA_ERROR: - // case Z_MEM_ERROR: - // inflateEnd(&strm); - // // free(result); - // if( result ) - // free(result); - // delete [] in; - // valid = false; - // in = NULL; result = NULL;// Or we get a double free aftr the while loop ... - // break; - // } - - // if( valid ) {// in case this stream is invalid, do not pass the already freed buffer to realloc. - - // U32 have = CHUNK-strm.avail_out; - - // U8* new_result = (U8*) realloc(result, cur_size + have); - // if (new_result == NULL) - // { - // LL_WARNS() << "Failed to unzip LLSD NavMesh block: can't reallocate memory, current size: " << cur_size - // << " bytes; requested " << cur_size + have - // << " bytes; total syze: ." << size << " bytes." - // << LL_ENDL; - // inflateEnd(&strm); - // if (result) - // { - // free(result); - // } - // delete[] in; - // valid = false; - // return NULL; - // } - // result = new_result; - // memcpy(result+cur_size, out, have); - // cur_size += have; - - // } // - - // } while (ret == Z_OK); - - // inflateEnd(&strm); - // delete [] in; - - // if (ret != Z_STREAM_END) - // { - // // result might have been freed above. And calling free with a null pointer is not defined. - // // free(result); - // if( result ) - // free(result); - // // - - // valid = false; - // return NULL; - // } - - // //result now points to the decompressed LLSD block - // { - // outsize= cur_size; - // valid = true; - // } - - // return result; - // } if (size == 0) { LL_WARNS() << "No data to unzip." << LL_ENDL; - return nullptr; - } - std::unique_ptr in; - try - { - in = std::make_unique(size); - } - catch (const std::bad_alloc&) - { - LL_WARNS() << "Memory allocation failure." << LL_ENDL; - return nullptr; + return NULL; } - is.read((char*)in.get(), size); - return unzip_llsdNavMesh(valid, outsize, in.get(), size); -} - -U8* unzip_llsdNavMesh(bool& valid, unsigned int& outsize, const U8* in, S32 size) -{ - if (size == 0) - { - LL_WARNS() << "No data to unzip." << LL_ENDL; - return nullptr; - } - U8* result = nullptr; + U8* result = NULL; U32 cur_size = 0; z_stream strm; - + const U32 CHUNK = 0x4000; - U8 out[CHUNK]; + U8 *in = new(std::nothrow) U8[size]; + if (in == NULL) + { + LL_WARNS() << "Memory allocation failure." << LL_ENDL; + return NULL; + } + is.read((char*) in, size); + U8 out[CHUNK]; + strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = size; - strm.next_in = const_cast(in); + strm.next_in = in; - - S32 ret = inflateInit2(&strm, windowBits | ENABLE_ZLIB_GZIP); + + S32 ret = inflateInit2(&strm, windowBits | ENABLE_ZLIB_GZIP ); do { strm.avail_out = CHUNK; @@ -2466,9 +2387,10 @@ U8* unzip_llsdNavMesh(bool& valid, unsigned int& outsize, const U8* in, S32 size { inflateEnd(&strm); free(result); - return nullptr; + delete [] in; + valid = false; } - + switch (ret) { case Z_NEED_DICT: @@ -2478,14 +2400,15 @@ U8* unzip_llsdNavMesh(bool& valid, unsigned int& outsize, const U8* in, S32 size case Z_MEM_ERROR: inflateEnd(&strm); free(result); + delete [] in; valid = false; - return nullptr; + break; } - U32 have = CHUNK - strm.avail_out; + U32 have = CHUNK-strm.avail_out; - U8* new_result = (U8*)realloc(result, cur_size + have); - if (!new_result) + U8* new_result = (U8*) realloc(result, cur_size + have); + if (new_result == NULL) { LL_WARNS() << "Failed to unzip LLSD NavMesh block: can't reallocate memory, current size: " << cur_size << " bytes; requested " << cur_size + have @@ -2496,33 +2419,34 @@ U8* unzip_llsdNavMesh(bool& valid, unsigned int& outsize, const U8* in, S32 size { free(result); } + delete[] in; valid = false; - return nullptr; + return NULL; } result = new_result; - memcpy(result + cur_size, out, have); + memcpy(result+cur_size, out, have); cur_size += have; } while (ret == Z_OK); inflateEnd(&strm); + delete [] in; if (ret != Z_STREAM_END) { free(result); valid = false; - return nullptr; + return NULL; } //result now points to the decompressed LLSD block { - outsize = cur_size; - valid = true; + outsize= cur_size; + valid = true; } return result; } -// char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size) { diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 49880ec1b5..676b7bfd6a 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -871,8 +871,6 @@ LL_COMMON_API std::string zip_llsd(LLSD& data); LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); -// Add non-allocating variants of unzip_llsd -LL_COMMON_API U8* unzip_llsdNavMesh(bool& valid, unsigned int& outsize, const U8* in, S32 size); // returns a pointer to the array or past the array if the deprecated header exists LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size = nullptr); diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp index 1ff15fcf89..bc32b6fd9e 100644 --- a/indra/llcommon/llstreamtools.cpp +++ b/indra/llcommon/llstreamtools.cpp @@ -513,3 +513,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck) } return str; } + +int cat_streambuf::underflow() +{ + if (gptr() == egptr()) + { + // here because our buffer is empty + std::streamsize size = 0; + // Until we've run out of mInputs, try reading the first of them + // into mBuffer. If that fetches some characters, break the loop. + while (! mInputs.empty() + && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size()))) + { + // We tried to read mInputs.front() but got zero characters. + // Discard the first streambuf and try the next one. + mInputs.pop_front(); + } + // Either we ran out of mInputs or we succeeded in reading some + // characters, that is, size != 0. Tell base class what we have. + setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size); + } + // If we fell out of the above loop with mBuffer still empty, return + // eof(), otherwise return the next character. + return (gptr() == egptr()) + ? std::char_traits::eof() + : std::char_traits::to_int_type(*gptr()); +} diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h index 1b04bf91d7..bb7bc20327 100644 --- a/indra/llcommon/llstreamtools.h +++ b/indra/llcommon/llstreamtools.h @@ -27,8 +27,10 @@ #ifndef LL_STREAM_TOOLS_H #define LL_STREAM_TOOLS_H +#include #include #include +#include // unless specifed otherwise these all return input_stream.good() @@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread( LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck); +/** + * cat_streambuf is a std::streambuf subclass that accepts a variadic number + * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates + * their contents. + */ +// derived from https://stackoverflow.com/a/49441066/5533635 +class cat_streambuf: public std::streambuf +{ +private: + std::deque mInputs; + std::vector mBuffer; + +public: + // only valid for std::streambuf* arguments + template + cat_streambuf(Inputs... inputs): + mInputs{inputs...}, + mBuffer(1024) + {} + + int underflow() override; +}; + #endif - - diff --git a/indra/llcommon/llstring.h b/indra/llcommon/llstring.h index 0eca77128b..d1e44efc22 100644 --- a/indra/llcommon/llstring.h +++ b/indra/llcommon/llstring.h @@ -1682,13 +1682,13 @@ std::basic_string LLStringUtilBase::quote(const string_type& str, // For whatever reason, we must quote this string. string_type result; result.push_back('"'); - for (typename const S8 c : str) + for (typename string_type::const_iterator ci(str.begin()), cend(str.end()); ci != cend; ++ci) { - if (c == '"') + if (*ci == '"') { result.append(escape); } - result.push_back(c); + result.push_back(*ci); } result.push_back('"'); return result; diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index e870ffbecb..8511d1bc08 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -40,11 +40,13 @@ #include "lluuid.h" #include "llerror.h" #include "llrand.h" -#include "llmd5.h" #include "llstring.h" #include "lltimer.h" #include "llthread.h" #include "llmutex.h" +#include "llmd5.h" +#include "hbxxh.h" + #include "llprofiler.h" const LLUUID LLUUID::null; const LLTransactionID LLTransactionID::tnull; @@ -403,6 +405,9 @@ LLUUID LLUUID::operator^(const LLUUID& rhs) const return id; } +// WARNING: this algorithm SHALL NOT be changed. It is also used by the server +// and plays a role in some assets validation (e.g. clothing items). Changing +// it would cause invalid assets. void LLUUID::combine(const LLUUID& other, LLUUID& result) const { LLMD5 md5_uuid; @@ -860,17 +865,12 @@ void LLUUID::generate() tmp >>= 8; mData[8] = (unsigned char) tmp; - LLMD5 md5_uuid; - - md5_uuid.update(mData,16); - md5_uuid.finalize(); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, (const void*)mData, 16); } void LLUUID::generate(const std::string& hash_string) { - LLMD5 md5_uuid((U8*)hash_string.c_str()); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, hash_string); } U32 LLUUID::getRandomSeed() @@ -888,13 +888,8 @@ U32 LLUUID::getRandomSeed() seed[7]=(unsigned char)(pid); getSystemTime((uuid_time_t *)(&seed[8])); - LLMD5 md5_seed; - - md5_seed.update(seed,16); - md5_seed.finalize(); - md5_seed.raw_digest(seed); - - return(*(U32 *)seed); + U64 seed64 = HBXXH64::digest((const void*)seed, 16); + return U32(seed64) ^ U32(seed64 >> 32); } BOOL LLUUID::parseUUID(const std::string& buf, LLUUID* value) diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index 5f0d7579f2..3962c348ae 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -171,6 +171,14 @@ public: U16 getCRC16() const; U32 getCRC32() const; + // Returns a 64 bits digest of the UUID, by XORing its two 64 bits long + // words. HB + inline U64 getDigest64() const + { + U64* tmp = (U64*)mData; + return tmp[0] ^ tmp[1]; + } + static BOOL validate(const std::string& in_string); // Validate that the UUID string is legal. static const LLUUID null; @@ -222,29 +230,23 @@ public: LLAssetID makeAssetID(const LLUUID& session) const; }; -// Generate a hash of an LLUUID object using the boost hash templates. - -// GCC 4.9 does not like the specialization in form of boost::hash but rather wants a namespace -// template <> -// struct boost::hash -namespace boost { template <> struct hash -// +// std::hash implementation for LLUUID +namespace std { - typedef LLUUID argument_type; - typedef std::size_t result_type; - result_type operator()(argument_type const& s) const - { - result_type seed(0); + template<> struct hash + { + inline size_t operator()(const LLUUID& id) const noexcept + { + return (size_t)id.getDigest64(); + } + }; +} - for (S32 i = 0; i < UUID_BYTES; ++i) - { - boost::hash_combine(seed, s.mData[i]); - } - - return seed; - } -}; -} // close namespace +// For use with boost containers. +inline size_t hash_value(const LLUUID& id) noexcept +{ + return (size_t)id.getDigest64(); +} // UUID hash calculation struct FSUUIDHash @@ -256,17 +258,4 @@ struct FSUUIDHash }; // UUID hash calculation -// Adapt boost hash to std hash -namespace std -{ - template<> struct hash - { - std::size_t operator()(LLUUID const& s) const noexcept - { - return boost::hash()(s); - } - }; -} -#endif - - +#endif // LL_LLUUID_H diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index d12f2935e8..9e4333b67b 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -110,12 +110,12 @@ namespace tut "import os\n" "import sys\n" "\n" - // Don't forget that this Python script is written to some - // temp directory somewhere! Its __file__ is useless in - // finding indra/lib/python. Use our __FILE__, with - // raw-string syntax to deal with Windows pathnames. - "mydir = os.path.dirname(r'" << __FILE__ << "')\n" - "from llbase import llsd\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" @@ -126,26 +126,26 @@ namespace tut " pass\n" "\n" "def get():\n" - " hdr = ''\n" - " while ':' not in hdr and len(hdr) < 20:\n" - " hdr += sys.stdin.read(1)\n" + " hdr = []\n" + " while b':' not in hdr and len(hdr) < 20:\n" + " hdr.append(sys.stdin.buffer.read(1))\n" " if not hdr:\n" " sys.exit(0)\n" - " if not hdr.endswith(':'):\n" + " if not hdr[-1] == b':':\n" " raise ProtocolError('Expected len:data, got %r' % hdr, hdr)\n" " try:\n" - " length = int(hdr[:-1])\n" + " length = int(b''.join(hdr[:-1]))\n" " except ValueError:\n" " raise ProtocolError('Non-numeric len %r' % hdr[:-1], hdr[:-1])\n" " parts = []\n" " received = 0\n" " while received < length:\n" - " parts.append(sys.stdin.read(length - received))\n" + " parts.append(sys.stdin.buffer.read(length - received))\n" " received += len(parts[-1])\n" - " data = ''.join(parts)\n" + " data = b''.join(parts)\n" " assert len(data) == length\n" " try:\n" - " return llsd.parse(data.encode())\n" + " return llsd.parse(data)\n" // Seems the old indra.base.llsd module didn't properly // convert IndexError (from running off end of string) to // LLSDParseError. @@ -185,11 +185,11 @@ namespace tut " return _reply\n" "\n" "def put(req):\n" - " sys.stdout.write(':'.join((str(len(req)), req)))\n" + " sys.stdout.buffer.write(b'%d:%b' % (len(req), req))\n" " sys.stdout.flush()\n" "\n" "def send(pump, data):\n" - " put(llsd.format_notation(dict(pump=pump, data=data)).decode())\n" + " put(llsd.format_notation(dict(pump=pump, data=data)))\n" "\n" "def request(pump, data):\n" " # we expect 'data' is a dict\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..29e3007aff 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -46,20 +46,24 @@ typedef U32 uint32_t; #include "boost/range.hpp" #include "boost/foreach.hpp" -#include "boost/function.hpp" #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" using namespace boost::phoenix; -#include "../llsd.h" -#include "../llsdserialize.h" +#include "llsd.h" +#include "llsdserialize.h" #include "llsdutil.h" -#include "../llformat.h" +#include "llformat.h" +#include "llmemorystream.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include + +typedef std::function FormatterFunction; +typedef std::function ParserFunction; std::vector string_to_vector(const std::string& str) { @@ -112,7 +116,7 @@ namespace tut mSD = LLUUID::null; expected = "\n"; xml_test("null uuid", expected); - + mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed"); expected = "c96f9b1e-f589-4100-9774-d98643ce0bed\n"; xml_test("uuid", expected); @@ -136,7 +140,7 @@ namespace tut expected = "aGVsbG8=\n"; xml_test("binary", expected); } - + template<> template<> void sd_xml_object::test<2>() { @@ -225,7 +229,7 @@ namespace tut expected = "bazfoobar\n"; xml_test("2 element map", expected); } - + template<> template<> void sd_xml_object::test<6>() { @@ -241,7 +245,7 @@ namespace tut expected = "Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==\n"; xml_test("binary", expected); } - + class TestLLSDSerializeData { public: @@ -250,9 +254,34 @@ namespace tut void doRoundTripTests(const std::string&); void checkRoundTrip(const std::string&, const LLSD& v); - - LLPointer mFormatter; - LLPointer mParser; + + void setFormatterParser(LLPointer formatter, LLPointer parser) + { + mFormatter = [formatter](const LLSD& data, std::ostream& str) + { + formatter->format(data, str); + }; + // this lambda must be mutable since otherwise the bound 'parser' + // is assumed to point to a const LLSDParser + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) mutable + { + // reset() call is needed since test code re-uses parser object + parser->reset(); + return (parser->parse(istr, data, max_bytes) > 0); + }; + } + + void setParser(bool (*parser)(LLSD&, std::istream&, llssize)) + { + // why does LLSDSerialize::deserialize() reverse the parse() params?? + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) + { + return (parser(data, istr, max_bytes) > 0); + }; + } + + FormatterFunction mFormatter; + ParserFunction mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -265,12 +294,11 @@ namespace tut void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v) { - std::stringstream stream; - mFormatter->format(v, stream); + std::stringstream stream; + mFormatter(v, stream); //LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL; LLSD w; - mParser->reset(); // reset() call is needed since test code re-uses mParser - mParser->parse(stream, w, stream.str().size()); + mParser(stream, w, stream.str().size()); try { @@ -299,52 +327,52 @@ namespace tut fillmap(root[key], width, depth - 1); } } - + void TestLLSDSerializeData::doRoundTripTests(const std::string& msg) { LLSD v; checkRoundTrip(msg + " undefined", v); - + v = true; checkRoundTrip(msg + " true bool", v); - + v = false; checkRoundTrip(msg + " false bool", v); - + v = 1; checkRoundTrip(msg + " positive int", v); - + v = 0; checkRoundTrip(msg + " zero int", v); - + v = -1; checkRoundTrip(msg + " negative int", v); - + v = 1234.5f; checkRoundTrip(msg + " positive float", v); - + v = 0.0f; checkRoundTrip(msg + " zero float", v); - + v = -1234.5f; checkRoundTrip(msg + " negative float", v); - + // FIXME: need a NaN test - + v = LLUUID::null; checkRoundTrip(msg + " null uuid", v); - + LLUUID newUUID; newUUID.generate(); v = newUUID; checkRoundTrip(msg + " new uuid", v); - + v = ""; checkRoundTrip(msg + " empty string", v); - + v = "some string"; checkRoundTrip(msg + " non-empty string", v); - + v = "Second Life is a 3-D virtual world entirely built and owned by its residents. " "Since opening to the public in 2003, it has grown explosively and today is " @@ -372,7 +400,7 @@ namespace tut for (U32 block = 0x000000; block <= 0x10ffff; block += block_size) { std::ostringstream out; - + for (U32 c = block; c < block + block_size; ++c) { if (c <= 0x000001f @@ -386,7 +414,7 @@ namespace tut if (0x00fdd0 <= c && c <= 0x00fdef) { continue; } if ((c & 0x00fffe) == 0x00fffe) { continue; } // see Unicode standard, section 15.8 - + if (c <= 0x00007f) { out << (char)(c & 0x7f); @@ -410,55 +438,55 @@ namespace tut out << (char)(0x80 | ((c >> 0) & 0x3f)); } } - + v = out.str(); std::ostringstream blockmsg; blockmsg << msg << " unicode string block 0x" << std::hex << block; checkRoundTrip(blockmsg.str(), v); } - + LLDate epoch; v = epoch; checkRoundTrip(msg + " epoch date", v); - + LLDate aDay("2002-12-07T05:07:15.00Z"); v = aDay; checkRoundTrip(msg + " date", v); - + LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/"); v = path; checkRoundTrip(msg + " url", v); - + const char source[] = "it must be a blue moon again"; std::vector data; // note, includes terminating '\0' copy(&source[0], &source[sizeof(source)], back_inserter(data)); - + v = data; checkRoundTrip(msg + " binary", v); - + v = LLSD::emptyMap(); checkRoundTrip(msg + " empty map", v); - + v = LLSD::emptyMap(); v["name"] = "luke"; //v.insert("name", "luke"); v["age"] = 3; //v.insert("age", 3); checkRoundTrip(msg + " map", v); - + v.clear(); v["a"]["1"] = true; v["b"]["0"] = false; checkRoundTrip(msg + " nested maps", v); - + v = LLSD::emptyArray(); checkRoundTrip(msg + " empty array", v); - + v = LLSD::emptyArray(); v.append("ali"); v.append(28); checkRoundTrip(msg + " array", v); - + v.clear(); v[0][0] = true; v[1][0] = false; @@ -468,7 +496,7 @@ namespace tut fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } - + typedef tut::test_group TestLLSDSerializeGroup; typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject; TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization"); @@ -476,35 +504,106 @@ namespace tut template<> template<> void TestLLSDSerializeObject::test<1>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY), + new LLSDNotationParser()); doRoundTripTests("pretty binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<2>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); doRoundTripTests("raw binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<3>() { - mFormatter = new LLSDXMLFormatter(); - mParser = new LLSDXMLParser(); + setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser()); doRoundTripTests("xml serialization"); } template<> template<> void TestLLSDSerializeObject::test<4>() { - mFormatter = new LLSDBinaryFormatter(); - mParser = new LLSDBinaryParser(); + setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser()); doRoundTripTests("binary serialization"); } + template<> template<> + void TestLLSDSerializeObject::test<5>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_BINARY)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<6>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_XML)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<7>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION); + }; + setParser(LLSDSerialize::deserialize); + // In this test, serialize(LLSD_NOTATION) emits a header recognized by + // deserialize(). + doRoundTripTests("serialize(LLSD_NOTATION)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<8>() + { + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDNotationFormatter does not + // emit an llsd/notation header. + doRoundTripTests("LLSDNotationFormatter -> deserialize"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<9>() + { + setFormatterParser(new LLSDXMLFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDXMLParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDXMLFormatter does not + // emit an LLSD/XML header. + doRoundTripTests("LLSDXMLFormatter -> deserialize"); + }; + +/*==========================================================================*| + // We do not expect this test to succeed. Without a header, neither + // notation LLSD nor binary LLSD reliably start with a distinct character, + // the way XML LLSD starts with '<'. By convention, we default to notation + // rather than binary. + template<> template<> + void TestLLSDSerializeObject::test<10>() + { + setFormatterParser(new LLSDBinaryFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDBinaryParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDBinaryFormatter does not + // emit an LLSD/Binary header. + doRoundTripTests("LLSDBinaryFormatter -> deserialize"); + }; +|*==========================================================================*/ /** * @class TestLLSDParsing @@ -555,7 +654,7 @@ namespace tut public: TestLLSDXMLParsing() {} }; - + typedef tut::test_group TestLLSDXMLParsingGroup; typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject; TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing"); @@ -586,8 +685,8 @@ namespace tut LLSD(), LLSDParser::PARSE_FAILURE); } - - + + template<> template<> void TestLLSDXMLParsingObject::test<2>() { @@ -596,7 +695,7 @@ namespace tut v["amy"] = 23; v["bob"] = LLSD(); v["cam"] = 1.23; - + ensureParse( "unknown data type", "" @@ -607,16 +706,16 @@ namespace tut v, v.size() + 1); } - + template<> template<> void TestLLSDXMLParsingObject::test<3>() { // test handling of nested bad data - + LLSD v; v["amy"] = 23; v["cam"] = 1.23; - + ensureParse( "map with html", "" @@ -626,7 +725,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["cam"] = 1.23; @@ -639,7 +738,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["bob"] = LLSD::emptyMap(); @@ -661,7 +760,7 @@ namespace tut v[0] = 23; v[1] = LLSD(); v[2] = 1.23; - + ensureParse( "array value of html", "" @@ -671,7 +770,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v[0] = 23; v[1] = LLSD::emptyMap(); @@ -1225,7 +1324,7 @@ namespace tut vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c'; vec[3] = '3'; vec[4] = '2'; vec[5] = '1'; LLSD value = vec; - + vec.resize(11); vec[0] = 'b'; // for binary vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c'; @@ -1694,85 +1793,83 @@ namespace tut ensureBinaryAndXML("map", test); } - struct TestPythonCompatible + // helper for TestPythonCompatible + static std::string import_llsd("import os.path\n" + "import sys\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n"); + + // helper for TestPythonCompatible + template + void python(const std::string& desc, const CONTENT& script, int expect=0) { - TestPythonCompatible(): - // Note the peculiar insertion of __FILE__ into this string. Since - // this script is being written into a platform-dependent temp - // directory, we can't locate indra/lib/python relative to - // Python's __file__. Use __FILE__ instead, navigating relative - // to this C++ source file. Use Python raw-string syntax so - // Windows pathname backslashes won't mislead Python's string - // scanner. - import_llsd("import os.path\n" - "import sys\n" - "from llbase import llsd\n") - {} - ~TestPythonCompatible() {} + auto PYTHON(LLStringUtil::getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - std::string import_llsd; - - template - void python(const std::string& desc, const CONTENT& script, int expect=0) - { - auto PYTHON(LLStringUtil::getenv("PYTHON")); - ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - - NamedTempFile scriptfile("py", script); + NamedTempFile scriptfile("py", script); #if LL_WINDOWS - std::string q("\""); - std::string qPYTHON(q + PYTHON + q); - std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); - if (rc == -1) - { - char buffer[256]; - strerror_s(buffer, errno); // C++ can infer the buffer size! :-O - ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); - } - else - { - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); - } + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) + { + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } #else // LL_DARWIN, LL_LINUX - LLProcess::Params params; - params.executable = PYTHON; - params.args.add(scriptfile.getName()); - LLProcessPtr py(LLProcess::create(params)); - ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); - // Implementing timeout would mean messing with alarm() and - // catching SIGALRM... later maybe... - int status(0); - if (waitpid(py->getProcessID(), &status, 0) == -1) + LLProcess::Params params; + params.executable = PYTHON; + params.args.add(scriptfile.getName()); + LLProcessPtr py(LLProcess::create(params)); + ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py->getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) { - int waitpid_errno(errno); - ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " - "waitpid() errno " << waitpid_errno), - waitpid_errno, ECHILD); + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); + } + else if (WIFSIGNALED(status)) + { + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); } else { - if (WIFEXITED(status)) - { - int rc(WEXITSTATUS(status)); - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), - rc, expect); - } - else if (WIFSIGNALED(status)) - { - ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), - false); - } - else - { - ensure(STRINGIZE(desc << " script produced impossible status " << status), - false); - } + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); } -#endif } +#endif + } + + struct TestPythonCompatible + { + TestPythonCompatible() {} + ~TestPythonCompatible() {} }; typedef tut::test_group TestPythonCompatibleGroup; @@ -1798,25 +1895,33 @@ namespace tut "print('Running on', sys.platform)\n"); } - // helper for test<3> - static void writeLLSDArray(std::ostream& out, const LLSD& array) + // helper for test<3> - test<7> + static void writeLLSDArray(const FormatterFunction& serialize, + std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item : llsd::inArray(array)) { - LLSDSerialize::toNotation(item, out); - // It's important to separate with newlines because Python's llsd - // module doesn't support parsing from a file stream, only from a - // string, so we have to know how much of the file to read into a - // string. - out << '\n'; + // It's important to delimit the entries in this file somehow + // because, although Python's llsd.parse() can accept a file + // stream, the XML parser expects EOF after a single outer element + // -- it doesn't just stop. So we must extract a sequence of bytes + // strings from the file. But since one of the serialization + // formats we want to test is binary, we can't pick any single + // byte value as a delimiter! Use a binary integer length prefix + // instead. + std::ostringstream buffer; + serialize(item, buffer); + auto buffstr{ buffer.str() }; + int bufflen{ static_cast(buffstr.length()) }; + out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); + out.write(buffstr.c_str(), buffstr.length()); } } - template<> template<> - void TestPythonCompatibleObject::test<3>() + // helper for test<3> - test<7> + static void toPythonUsing(const std::string& desc, + const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(LLSDArray(17)(3.14) ("This string\n" "has several\n" @@ -1836,7 +1941,7 @@ namespace tut " except StopIteration:\n" " pass\n" " else:\n" - " assert False, 'Too many data items'\n"; + " raise AssertionError('Too many data items')\n"; // Create an llsdXXXXXX file containing 'data' serialized to // notation. @@ -1845,32 +1950,128 @@ namespace tut // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [serialize, cdata] + (std::ostream& out) + { writeLLSDArray(serialize, out, cdata); }); - python("read C++ notation", + python("read C++ " + desc, placeholders::arg1 << import_llsd << - "def parse_each(iterable):\n" - " for item in iterable:\n" - " yield llsd.parse(item)\n" << - pydata << + "from functools import partial\n" + "import io\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " try:\n" + " frombytes = llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" + << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } + template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_XML)"); + toPythonUsing("LLSD_XML", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_XML); }); + } + template<> template<> void TestPythonCompatibleObject::test<4>() { - set_test_name("verify sequence from Python"); + set_test_name("to Python using LLSDSerialize::serialize(LLSD_NOTATION)"); + toPythonUsing("LLSD_NOTATION", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_NOTATION); }); + } + template<> template<> + void TestPythonCompatibleObject::test<5>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_BINARY)"); + toPythonUsing("LLSD_BINARY", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_BINARY); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<6>() + { + set_test_name("to Python using LLSDSerialize::toXML()"); + toPythonUsing("toXML()", LLSDSerialize::toXML); + } + + template<> template<> + void TestPythonCompatibleObject::test<7>() + { + set_test_name("to Python using LLSDSerialize::toNotation()"); + toPythonUsing("toNotation()", LLSDSerialize::toNotation); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("to Python using LLSDSerialize::toBinary()"); + // We don't expect this to work because, without a header, + // llsd.parse() will assume notation rather than binary. + toPythonUsing("toBinary()", LLSDSerialize::toBinary); + } +|*==========================================================================*/ + + // helper for test<8> - test<12> + bool itemFromStream(std::istream& istr, LLSD& item, const ParserFunction& parse) + { + // reset the output value for debugging clarity + item.clear(); + // We use an int length prefix as a foolproof delimiter even for + // binary serialized streams. + int length{ 0 }; + istr.read(reinterpret_cast(&length), sizeof(length)); +// return parse(istr, item, length); + // Sadly, as of 2022-12-01 it seems we can't really trust our LLSD + // parsers to honor max_bytes: this test works better when we read + // each item into its own distinct LLMemoryStream, instead of passing + // the original istr with a max_bytes constraint. + std::vector buffer(length); + istr.read(reinterpret_cast(buffer.data()), length); + LLMemoryStream stream(buffer.data(), length); + return parse(stream, item, length); + } + + // helper for test<8> - test<12> + void fromPythonUsing(const std::string& pyformatter, + const ParserFunction& parse= + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::deserialize(data, istr, max_bytes); }) + { // Create an empty data file. This is just a placeholder for our // script to write into. Create it to establish a unique name that // we know. NamedTempFile file("llsd", ""); - python("write Python notation", + python("Python " + pyformatter, placeholders::arg1 << import_llsd << + "import struct\n" + "lenformat = struct.Struct('i')\n" "DATA = [\n" " 17,\n" " 3.14,\n" @@ -1881,34 +2082,87 @@ namespace tut "]\n" // Don't forget raw-string syntax for Windows pathnames. // N.B. Using 'print' implicitly adds newlines. - "with open(r'" << file.getName() << "', 'w') as f:\n" + "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " serialized = llsd." << pyformatter << "(item)\n" + " f.write(lenformat.pack(len(serialized)))\n" + " f.write(serialized)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; - // Notice that we're not doing anything special to parse out the - // newlines: LLSDSerialize::fromNotation ignores them. While it would - // seem they're not strictly necessary, going in this direction, we - // want to ensure that notation-separated-by-newlines works in both - // directions -- since in practice, a given file might be read by - // either language. - ensure_equals("Failed to read LLSD::Integer from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asInteger(), 17); - ensure_equals("Failed to read LLSD::Real from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_approximately_equals("Bad LLSD::Real value from Python", - item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure_equals("Failed to read LLSD::String from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asString(), - "This string\n" - "has several\n" - "lines."); - + try + { + ensure("Failed to read LLSD::Integer from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asInteger(), 17); + ensure("Failed to read LLSD::Real from Python", + itemFromStream(inf, item, parse)); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure("Failed to read LLSD::String from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } + catch (const tut::failure& err) + { + std::cout << "for " << err.what() << ", item = " << item << std::endl; + throw; + } } + + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("from Python XML using LLSDSerialize::deserialize()"); + fromPythonUsing("format_xml"); + } + + template<> template<> + void TestPythonCompatibleObject::test<9>() + { + set_test_name("from Python notation using LLSDSerialize::deserialize()"); + fromPythonUsing("format_notation"); + } + + template<> template<> + void TestPythonCompatibleObject::test<10>() + { + set_test_name("from Python binary using LLSDSerialize::deserialize()"); + fromPythonUsing("format_binary"); + } + + template<> template<> + void TestPythonCompatibleObject::test<11>() + { + set_test_name("from Python XML using fromXML()"); + // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors + fromPythonUsing("format_xml", + [](std::istream& istr, LLSD& data, llssize) + { return LLSDSerialize::fromXML(data, istr) > 0; }); + } + + template<> template<> + void TestPythonCompatibleObject::test<12>() + { + set_test_name("from Python notation using fromNotation()"); + fromPythonUsing("format_notation", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<13>() + { + set_test_name("from Python binary using fromBinary()"); + // We don't expect this to work because format_binary() emits a + // header, but fromBinary() won't recognize a header. + fromPythonUsing("format_binary", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); + } +|*==========================================================================*/ } diff --git a/indra/llprimitive/llmaterialid.h b/indra/llprimitive/llmaterialid.h index ee663f8f99..c66e3e30a3 100644 --- a/indra/llprimitive/llmaterialid.h +++ b/indra/llprimitive/llmaterialid.h @@ -66,6 +66,14 @@ public: static const LLMaterialID null; + // Returns a 64 bits digest of the material Id, by XORing its two 64 bits + // long words. HB + inline U64 getDigest64() const + { + U64* tmp = (U64*)mID; + return tmp[0] ^ tmp[1]; + } + private: void parseFromBinary(const LLSD::Binary& pMaterialID); void copyFromOtherMaterialID(const LLMaterialID& pOtherMaterialID); @@ -74,5 +82,23 @@ private: U8 mID[MATERIAL_ID_SIZE]; } ; +// std::hash implementation for LLMaterialID +namespace std +{ + template<> struct hash + { + inline size_t operator()(const LLMaterialID& id) const noexcept + { + return (size_t)id.getDigest64(); + } + }; +} + +// For use with boost containers. +inline size_t hash_value(const LLMaterialID& id) noexcept +{ + return (size_t)id.getDigest64(); +} + #endif // LL_LLMATERIALID_H diff --git a/indra/llprimitive/llmodel.cpp b/indra/llprimitive/llmodel.cpp index 567dae644e..67c9ea64c4 100644 --- a/indra/llprimitive/llmodel.cpp +++ b/indra/llprimitive/llmodel.cpp @@ -31,7 +31,7 @@ #include "llconvexdecomposition.h" #include "llsdserialize.h" #include "llvector4a.h" -#include "llmd5.h" +#include "hbxxh.h" #include "llcontrol.h" #ifdef LL_USESYSTEMLIBS @@ -1580,7 +1580,7 @@ LLSD LLMeshSkinInfo::asLLSD(bool include_joints, bool lock_scale_if_joint_positi void LLMeshSkinInfo::updateHash() { // get hash of data relevant to render batches - LLMD5 hash; + HBXXH64 hash; //mJointNames for (auto& name : mJointNames) @@ -1591,24 +1591,19 @@ void LLMeshSkinInfo::updateHash() } //mJointNums - hash.update((U8*)&(mJointNums[0]), sizeof(S32) * mJointNums.size()); + hash.update((const void*)mJointNums.data(), sizeof(S32) * mJointNums.size()); //mInvBindMatrix F32* src = mInvBindMatrix[0].getF32ptr(); - for (int i = 0; i < mInvBindMatrix.size() * 16; ++i) + for (size_t i = 0, count = mInvBindMatrix.size() * 16; i < count; ++i) { S32 t = llround(src[i] * 10000.f); - hash.update((U8*)&t, sizeof(S32)); + hash.update((const void*)&t, sizeof(S32)); } - //hash.update((U8*)&(mInvBindMatrix[0]), sizeof(LLMatrix4a) * mInvBindMatrix.size()); + //hash.update((const void*)mInvBindMatrix.data(), sizeof(LLMatrix4a) * mInvBindMatrix.size()); - hash.finalize(); - - U64 digest[2]; - hash.raw_digest((U8*) digest); - - mHash = digest[0]; + mHash = hash.digest(); } U32 LLMeshSkinInfo::sizeBytes() const diff --git a/indra/llprimitive/llmodelloader.cpp b/indra/llprimitive/llmodelloader.cpp index 3bacbfc522..5b7164dc8d 100644 --- a/indra/llprimitive/llmodelloader.cpp +++ b/indra/llprimitive/llmodelloader.cpp @@ -25,6 +25,8 @@ */ #include "llmodelloader.h" + +#include "llapp.h" #include "llsdserialize.h" #include "lljoint.h" #include "llcallbacklist.h" @@ -381,7 +383,10 @@ bool LLModelLoader::isAlive(LLModelLoader* loader) void LLModelLoader::loadModelCallback() { - mLoadCallback(mScene,mModelList,mLod, mOpaqueData); + if (!LLApp::isExiting()) + { + mLoadCallback(mScene, mModelList, mLod, mOpaqueData); + } while (!isStopped()) { //wait until this thread is stopped before deleting self diff --git a/indra/newview/llagentwearables.cpp b/indra/newview/llagentwearables.cpp index 0a9e67505e..a274dc6346 100644 --- a/indra/newview/llagentwearables.cpp +++ b/indra/newview/llagentwearables.cpp @@ -41,7 +41,7 @@ #include "llinventoryobserver.h" #include "llinventorypanel.h" #include "lllocaltextureobject.h" -#include "llmd5.h" +#include "llmd5.h" // [Legacy Bake] #include "llnotificationsutil.h" #include "lloutfitobserver.h" #include "llsidepanelappearance.h" diff --git a/indra/newview/llappearancemgr.cpp b/indra/newview/llappearancemgr.cpp index 771d7398da..08fd4975d4 100644 --- a/indra/newview/llappearancemgr.cpp +++ b/indra/newview/llappearancemgr.cpp @@ -41,7 +41,9 @@ #include "llinventorybridge.h" #include "llinventoryfunctions.h" #include "llinventoryobserver.h" +#include "llmd5.h" #include "llnotificationsutil.h" +#include "llmd5.h" #include "lloutfitobserver.h" #include "lloutfitslist.h" #include "llselectmgr.h" diff --git a/indra/newview/llfilepicker.cpp b/indra/newview/llfilepicker.cpp index 942e4e53a1..3589365dc3 100644 --- a/indra/newview/llfilepicker.cpp +++ b/indra/newview/llfilepicker.cpp @@ -311,6 +311,15 @@ BOOL LLFilePicker::getOpenFile(ELoadFilter filter, bool blocking) return success; } +BOOL LLFilePicker::getOpenFileModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata) +{ + // not supposed to be used yet, use LLFilePickerThread + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + BOOL LLFilePicker::getMultipleOpenFiles(ELoadFilter filter, bool blocking) { if( mLocked ) @@ -388,6 +397,15 @@ BOOL LLFilePicker::getMultipleOpenFiles(ELoadFilter filter, bool blocking) return success; } +BOOL LLFilePicker::getMultipleOpenFilesModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata ) +{ + // not supposed to be used yet, use LLFilePickerThread + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + BOOL LLFilePicker::getSaveFile(ESaveFilter filter, const std::string& filename, bool blocking) { if( mLocked ) @@ -641,6 +659,16 @@ BOOL LLFilePicker::getSaveFile(ESaveFilter filter, const std::string& filename, return success; } +BOOL LLFilePicker::getSaveFileModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata) +{ + // not supposed to be used yet, use LLFilePickerThread + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + #elif LL_DARWIN std::unique_ptr> LLFilePicker::navOpenFilterProc(ELoadFilter filter) //(AEDesc *theItem, void *info, void *callBackUD, NavFilterModes filterMode) @@ -747,129 +775,123 @@ bool LLFilePicker::doNavChooseDialog(ELoadFilter filter) return false; } +bool LLFilePicker::doNavChooseDialogModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &,void*), + void *userdata) +{ + // if local file browsing is turned off, return without opening dialog + if ( check_local_file_access_enabled() == false ) + { + return false; + } + + std::unique_ptr> allowed_types=navOpenFilterProc(filter); + + doLoadDialogModeless(allowed_types.get(), + mPickOptions, + callback, + userdata); + + return true; +} + +void set_nav_save_data(LLFilePicker::ESaveFilter filter, std::string &extension, std::string &type, std::string &creator) +{ + switch (filter) + { + case LLFilePicker::FFSAVE_WAV: + type = "WAVE"; + creator = "TVOD"; + extension = "wav"; + break; + case LLFilePicker::FFSAVE_TGA: + type = "TPIC"; + creator = "prvw"; + extension = "tga"; + break; + case LLFilePicker::FFSAVE_TGAPNG: + type = "PNG"; + creator = "prvw"; + extension = "png,tga"; + break; + case LLFilePicker::FFSAVE_BMP: + type = "BMPf"; + creator = "prvw"; + extension = "bmp"; + break; + case LLFilePicker::FFSAVE_JPEG: + type = "JPEG"; + creator = "prvw"; + extension = "jpeg"; + break; + case LLFilePicker::FFSAVE_PNG: + type = "PNG "; + creator = "prvw"; + extension = "png"; + break; + case LLFilePicker::FFSAVE_AVI: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = "mov"; + break; + + case LLFilePicker::FFSAVE_ANIM: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = "xaf"; + break; + +#ifdef _CORY_TESTING + case LLFilePicker::FFSAVE_GEOMETRY: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = "slg"; + break; +#endif + + case LLFilePicker::FFSAVE_XML: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = "xml"; + break; + + case LLFilePicker::FFSAVE_RAW: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = "raw"; + break; + + case LLFilePicker::FFSAVE_J2C: + type = "\?\?\?\?"; + creator = "prvw"; + extension = "j2c"; + break; + + case LLFilePicker::FFSAVE_SCRIPT: + type = "LSL "; + creator = "\?\?\?\?"; + extension = "lsl"; + break; + + case LLFilePicker::FFSAVE_ALL: + default: + type = "\?\?\?\?"; + creator = "\?\?\?\?"; + extension = ""; + break; + } +} + bool LLFilePicker::doNavSaveDialog(ESaveFilter filter, const std::string& filename) { - // Setup the type, creator, and extension std::string extension, type, creator; - switch (filter) - { - case FFSAVE_WAV: - type = "WAVE"; - creator = "TVOD"; - extension = "wav"; - break; - case FFSAVE_TGA: - type = "TPIC"; - creator = "prvw"; - extension = "tga"; - break; - case FFSAVE_TGAPNG: - type = "PNG"; - creator = "prvw"; - extension = "png,tga"; - break; - case FFSAVE_BMP: - type = "BMPf"; - creator = "prvw"; - extension = "bmp"; - break; - case FFSAVE_JPEG: - type = "JPEG"; - creator = "prvw"; - extension = "jpeg"; - break; - case FFSAVE_PNG: - type = "PNG "; - creator = "prvw"; - extension = "png"; - break; - case FFSAVE_AVI: - type = "\?\?\?\?"; - creator = "\?\?\?\?"; - extension = "mov"; - break; - - case FFSAVE_ANIM: - type = "\?\?\?\?"; - creator = "\?\?\?\?"; - extension = "xaf"; - break; - -#ifdef _CORY_TESTING - case FFSAVE_GEOMETRY: - type = "\?\?\?\?"; - creator = "\?\?\?\?"; - extension = "slg"; - break; -#endif - // Compile fix - // case FFSAVE_XML: - // type = "\?\?\?\?"; - // creator = "\?\?\?\?"; - // extension = "xml"; - // break; - // Compile fix - - case FFSAVE_RAW: - type = "\?\?\?\?"; - creator = "\?\?\?\?"; - extension = "raw"; - break; - - case FFSAVE_J2C: - type = "\?\?\?\?"; - creator = "prvw"; - extension = "j2c"; - break; - - case FFSAVE_SCRIPT: - type = "LSL "; - creator = "\?\?\?\?"; - extension = "lsl"; - break; - // Export filter - case FFSAVE_EXPORT: - type = "OXP "; - creator = "\?\?\?\?"; - extension = "oxp"; - break; - case FFSAVE_COLLADA: - type = "DAE "; - creator = "\?\?\?\?"; - extension = "dae"; - break; - // CSV Filter - case FFSAVE_CSV: - type = "CSV "; - creator = "\?\?\?\?"; - extension = "csv"; - break; - // - case FFSAVE_BEAM: - case FFSAVE_XML: - type = "XML "; - creator = "\?\?\?\?"; - extension = "xml"; - break; - case FFSAVE_ALL: - default: - type = "\?\?\?\?"; - creator = "\?\?\?\?"; - extension = ""; - break; - } + set_nav_save_data(filter, extension, type, creator); std::string namestring = filename; if (namestring.empty()) namestring="Untitled"; -// if (! boost::algorithm::ends_with(namestring, extension) ) -// { -// namestring = namestring + "." + extension; -// -// } - gViewerWindow->getWindow()->beforeDialog(); // Run the dialog @@ -890,6 +912,30 @@ bool LLFilePicker::doNavSaveDialog(ESaveFilter filter, const std::string& filena return false; } +bool LLFilePicker::doNavSaveDialogModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata) +{ + // Setup the type, creator, and extension + std::string extension, type, creator; + + set_nav_save_data(filter, extension, type, creator); + + std::string namestring = filename; + if (namestring.empty()) namestring="Untitled"; + + // Run the dialog + doSaveDialogModeless(&namestring, + &type, + &creator, + &extension, + mPickOptions, + callback, + userdata); + return true; +} + BOOL LLFilePicker::getOpenFile(ELoadFilter filter, bool blocking) { if( mLocked ) @@ -945,18 +991,52 @@ BOOL LLFilePicker::getOpenFile(ELoadFilter filter, bool blocking) return success; } + +BOOL LLFilePicker::getOpenFileModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata) +{ + if( mLocked ) + return FALSE; + + // if local file browsing is turned off, return without opening dialog + if ( check_local_file_access_enabled() == false ) + { + return FALSE; + } + + reset(); + + mPickOptions &= ~F_MULTIPLE; + mPickOptions |= F_FILE; + + if (filter == FFLOAD_DIRECTORY) //This should only be called from lldirpicker. + { + + mPickOptions |= ( F_NAV_SUPPORT | F_DIRECTORY ); + mPickOptions &= ~F_FILE; + } + + if (filter == FFLOAD_ALL) // allow application bundles etc. to be traversed; important for DEV-16869, but generally useful + { + mPickOptions |= F_NAV_SUPPORT; + } + + return doNavChooseDialogModeless(filter, callback, userdata); +} + BOOL LLFilePicker::getMultipleOpenFiles(ELoadFilter filter, bool blocking) { if( mLocked ) return FALSE; - BOOL success = FALSE; - // if local file browsing is turned off, return without opening dialog if ( check_local_file_access_enabled() == false ) { return FALSE; } + + BOOL success = FALSE; reset(); @@ -990,6 +1070,29 @@ BOOL LLFilePicker::getMultipleOpenFiles(ELoadFilter filter, bool blocking) return success; } + +BOOL LLFilePicker::getMultipleOpenFilesModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata ) +{ + if( mLocked ) + return FALSE; + + // if local file browsing is turned off, return without opening dialog + if ( check_local_file_access_enabled() == false ) + { + return FALSE; + } + + reset(); + + mPickOptions |= F_FILE; + + mPickOptions |= F_MULTIPLE; + + return doNavChooseDialogModeless(filter, callback, userdata); +} + BOOL LLFilePicker::getSaveFile(ESaveFilter filter, const std::string& filename, bool blocking) { @@ -1030,6 +1133,27 @@ BOOL LLFilePicker::getSaveFile(ESaveFilter filter, const std::string& filename, LLFrameTimer::updateFrameTime(); return success; } + +BOOL LLFilePicker::getSaveFileModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata) +{ + if( mLocked ) + return false; + + // if local file browsing is turned off, return without opening dialog + if ( check_local_file_access_enabled() == false ) + { + return false; + } + + reset(); + + mPickOptions &= ~F_MULTIPLE; + + return doNavSaveDialogModeless(filter, filename, callback, userdata); +} //END LL_DARWIN #elif LL_LINUX @@ -1829,6 +1953,15 @@ BOOL LLFilePicker::getSaveFile( ESaveFilter filter, const std::string& filename, return FALSE; } +BOOL LLFilePicker::getSaveFileModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata) +{ + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + BOOL LLFilePicker::getOpenFile( ELoadFilter filter, bool blocking ) { // if local file browsing is turned off, return without opening dialog @@ -1854,6 +1987,14 @@ BOOL LLFilePicker::getOpenFile( ELoadFilter filter, bool blocking ) return TRUE; } +BOOL LLFilePicker::getOpenFileModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata) +{ + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + BOOL LLFilePicker::getMultipleOpenFiles( ELoadFilter filter, bool blocking) { // if local file browsing is turned off, return without opening dialog @@ -1867,6 +2008,14 @@ BOOL LLFilePicker::getMultipleOpenFiles( ELoadFilter filter, bool blocking) return FALSE; } +BOOL LLFilePicker::getMultipleOpenFilesModeless(ELoadFilter filter, + void (*callback)(bool, std::vector &, void*), + void *userdata ) +{ + LL_ERRS() << "NOT IMPLEMENTED" << LL_ENDL; + return FALSE; +} + #endif // LL_GTK #else // not implemented diff --git a/indra/newview/llfilepicker.h b/indra/newview/llfilepicker.h index 8f97bbec2e..2ae409f8ee 100644 --- a/indra/newview/llfilepicker.h +++ b/indra/newview/llfilepicker.h @@ -129,8 +129,16 @@ public: // open the dialog. This is a modal operation BOOL getSaveFile( ESaveFilter filter = FFSAVE_ALL, const std::string& filename = LLStringUtil::null, bool blocking = true); + BOOL getSaveFileModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata); BOOL getOpenFile( ELoadFilter filter = FFLOAD_ALL, bool blocking = true ); + // Todo: implement getOpenFileModeless and getMultipleOpenFilesModeless + // for windows and use directly instead of ugly LLFilePickerThread + BOOL getOpenFileModeless( ELoadFilter filter, void (*callback)(bool, std::vector &, void*), void *userdata); // MAC only. BOOL getMultipleOpenFiles( ELoadFilter filter = FFLOAD_ALL, bool blocking = true ); + BOOL getMultipleOpenFilesModeless( ELoadFilter filter, void (*callback)(bool, std::vector &, void*), void *userdata ); // MAC only // Get the filename(s) found. getFirstFile() sets the pointer to // the start of the structure and allows the start of iteration. @@ -181,8 +189,15 @@ private: std::vector mFileVector; bool doNavChooseDialog(ELoadFilter filter); + bool doNavChooseDialogModeless(ELoadFilter filter, + void (*callback)(bool, std::vector&, void*), + void *userdata); bool doNavSaveDialog(ESaveFilter filter, const std::string& filename); std::unique_ptr> navOpenFilterProc(ELoadFilter filter); + bool doNavSaveDialogModeless(ESaveFilter filter, + const std::string& filename, + void (*callback)(bool, std::string&, void*), + void *userdata); #endif #if LL_GTK diff --git a/indra/newview/llfilepicker_mac.h b/indra/newview/llfilepicker_mac.h index b2fb371afe..367c792969 100644 --- a/indra/newview/llfilepicker_mac.h +++ b/indra/newview/llfilepicker_mac.h @@ -41,11 +41,25 @@ //void modelessPicker(); std::unique_ptr> doLoadDialog(const std::vector* allowed_types, unsigned int flags); + +void doLoadDialogModeless(const std::vector* allowed_types, + unsigned int flags, + void (*callback)(bool, std::vector&, void*), + void *userdata); + std::unique_ptr doSaveDialog(const std::string* file, const std::string* type, const std::string* creator, const std::string* extension, unsigned int flags); + +void doSaveDialogModeless(const std::string* file, + const std::string* type, + const std::string* creator, + const std::string* extension, + unsigned int flags, + void (*callback)(bool, std::string&, void*), + void *userdata); enum { F_FILE = 0x00000001, F_DIRECTORY = 0x00000002, diff --git a/indra/newview/llfilepicker_mac.mm b/indra/newview/llfilepicker_mac.mm index 0ae5fc3f77..e5965abbd6 100644 --- a/indra/newview/llfilepicker_mac.mm +++ b/indra/newview/llfilepicker_mac.mm @@ -29,53 +29,62 @@ #include #include "llfilepicker_mac.h" +NSOpenPanel *init_panel(const std::vector* allowed_types, unsigned int flags) +{ + int i; + + NSOpenPanel *panel = [NSOpenPanel openPanel]; + NSMutableArray *fileTypes = nil; + + + if ( allowed_types && !allowed_types->empty()) + { + fileTypes = [[NSMutableArray alloc] init]; + + for (i=0;isize();++i) + { + [fileTypes addObject: + [NSString stringWithCString:(*allowed_types)[i].c_str() + encoding:[NSString defaultCStringEncoding]]]; + } + } + + //[panel setMessage:@"Import one or more files or directories."]; + [panel setAllowsMultipleSelection: ( (flags & F_MULTIPLE)?true:false ) ]; + [panel setCanChooseDirectories: ( (flags & F_DIRECTORY)?true:false ) ]; + [panel setCanCreateDirectories: true]; + [panel setResolvesAliases: true]; + [panel setCanChooseFiles: ( (flags & F_FILE)?true:false )]; + [panel setTreatsFilePackagesAsDirectories: ( flags & F_NAV_SUPPORT ) ]; + + if (fileTypes) + { + [panel setAllowedFileTypes:fileTypes]; + } + else + { + // I suggest it's better to open the last path and let this default to home dir as necessary + // for consistency with other OS X apps + // + //[panel setDirectoryURL: fileURLWithPath(NSHomeDirectory()) ]; + } + return panel; +} + std::unique_ptr> doLoadDialog(const std::vector* allowed_types, unsigned int flags) { std::unique_ptr> outfiles; - @autoreleasepool { - int i, result; + @autoreleasepool + { + int result; //Aura TODO: We could init a small window and release it at the end of this routine //for a modeless interface. - - NSOpenPanel *panel = [NSOpenPanel openPanel]; - //NSString *fileName = nil; - NSMutableArray *fileTypes = nil; - - if ( allowed_types && !allowed_types->empty()) - { - fileTypes = [[[NSMutableArray alloc] init] autorelease]; - - for (i=0;isize();++i) - { - [fileTypes addObject: - [NSString stringWithCString:(*allowed_types)[i].c_str() - encoding:[NSString defaultCStringEncoding]]]; - } - } - - //[panel setMessage:@"Import one or more files or directories."]; - [panel setAllowsMultipleSelection: ( (flags & F_MULTIPLE)?true:false ) ]; - [panel setCanChooseDirectories: ( (flags & F_DIRECTORY)?true:false ) ]; - [panel setCanCreateDirectories: true]; - [panel setResolvesAliases: true]; - [panel setCanChooseFiles: ( (flags & F_FILE)?true:false )]; - [panel setTreatsFilePackagesAsDirectories: ( flags & F_NAV_SUPPORT ) ]; - - if (fileTypes) - { - [panel setAllowedFileTypes:fileTypes]; - result = [panel runModal]; - } - else - { - // I suggest it's better to open the last path and let this default to home dir as necessary - // for consistency with other OS X apps - // - //[panel setDirectoryURL: fileURLWithPath(NSHomeDirectory()) ]; - result = [panel runModal]; - } + + NSOpenPanel *panel = init_panel(allowed_types,flags); + + result = [panel runModal]; if (result == NSOKButton) { @@ -98,6 +107,48 @@ std::unique_ptr> doLoadDialog(const std::vector* allowed_types, + unsigned int flags, + void (*callback)(bool, std::vector &, void*), + void *userdata) +{ + + @autoreleasepool + { + // Note: might need to return and save this panel + // so that it does not close immediately + NSOpenPanel *panel = init_panel(allowed_types,flags); + + [panel beginWithCompletionHandler:^(NSModalResponse result) + { + std::vector outfiles; + if (result == NSOKButton) + { + NSArray *filesToOpen = [panel URLs]; + int i, count = [filesToOpen count]; + + if (count > 0) + { + + for (i=0; i doSaveDialog(const std::string* file, const std::string* type, @@ -106,7 +157,8 @@ std::unique_ptr doSaveDialog(const std::string* file, unsigned int flags) { std::unique_ptr outfile; - @autoreleasepool { + @autoreleasepool + { NSSavePanel *panel = [NSSavePanel savePanel]; NSString *extensionns = [NSString stringWithCString:extension->c_str() encoding:[NSString defaultCStringEncoding]]; @@ -133,4 +185,48 @@ std::unique_ptr doSaveDialog(const std::string* file, return outfile; } +void doSaveDialogModeless(const std::string* file, + const std::string* type, + const std::string* creator, + const std::string* extension, + unsigned int flags, + void (*callback)(bool, std::string&, void*), + void *userdata) +{ + @autoreleasepool { + NSSavePanel *panel = [NSSavePanel savePanel]; + + NSString *extensionns = [NSString stringWithCString:extension->c_str() encoding:[NSString defaultCStringEncoding]]; + NSArray *fileType = [extensionns componentsSeparatedByString:@","]; + + //[panel setMessage:@"Save Image File"]; + [panel setTreatsFilePackagesAsDirectories: ( flags & F_NAV_SUPPORT ) ]; + [panel setCanSelectHiddenExtension:true]; + [panel setAllowedFileTypes:fileType]; + NSString *fileName = [NSString stringWithCString:file->c_str() encoding:[NSString defaultCStringEncoding]]; + + NSURL* url = [NSURL fileURLWithPath:fileName]; + [panel setNameFieldStringValue: fileName]; + [panel setDirectoryURL: url]; + + + [panel beginWithCompletionHandler:^(NSModalResponse result) + { + if (result == NSOKButton) + { + NSURL* url = [panel URL]; + NSString* p = [url path]; + std::string outfile([p UTF8String]); + + callback(true, outfile, userdata); + } + else // cancel + { + std::string outfile; + callback(false, outfile, userdata); + } + }]; + } +} + #endif diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 8ad8f2dd6d..1d2d6c2493 100644 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -62,6 +62,7 @@ #include "bufferarray.h" #include "bufferstream.h" #include "llcorehttputil.h" +#include "hbxxh.h" // [RLVa:KB] - Checked: 2011-05-22 (RLVa-1.3.1a) #include "rlvhandler.h" #include "rlvlocks.h" @@ -518,17 +519,16 @@ void LLInventoryModel::getDirectDescendentsOf(const LLUUID& cat_id, items = get_ptr_in_map(mParentChildItemTree, cat_id); } -LLMD5 LLInventoryModel::hashDirectDescendentNames(const LLUUID& cat_id) const +LLInventoryModel::digest_t LLInventoryModel::hashDirectDescendentNames(const LLUUID& cat_id) const { LLInventoryModel::cat_array_t* cat_array; LLInventoryModel::item_array_t* item_array; getDirectDescendentsOf(cat_id,cat_array,item_array); - LLMD5 item_name_hash; if (!item_array) { - item_name_hash.finalize(); - return item_name_hash; + return LLUUID::null; } + HBXXH128 item_name_hash; for (LLInventoryModel::item_array_t::const_iterator iter = item_array->begin(); iter != item_array->end(); iter++) @@ -538,8 +538,7 @@ LLMD5 LLInventoryModel::hashDirectDescendentNames(const LLUUID& cat_id) const continue; item_name_hash.update(item->getName()); } - item_name_hash.finalize(); - return item_name_hash; + return item_name_hash.digest(); } // SJB: Added version to lock the arrays to catch potential logic bugs diff --git a/indra/newview/llinventorymodel.h b/indra/newview/llinventorymodel.h index f4f0d5a74c..ee1241b1c2 100644 --- a/indra/newview/llinventorymodel.h +++ b/indra/newview/llinventorymodel.h @@ -39,7 +39,6 @@ #include "llpermissionsflags.h" #include "llviewerinventory.h" #include "llstring.h" -#include "llmd5.h" #include "httpcommon.h" #include "httprequest.h" #include "httpoptions.h" @@ -268,8 +267,9 @@ public: cat_array_t*& categories, item_array_t*& items) const; + typedef LLUUID digest_t; // To clarify the actual usage of this "UUID" // Compute a hash of direct descendant names (for detecting child name changes) - LLMD5 hashDirectDescendentNames(const LLUUID& cat_id) const; + digest_t hashDirectDescendentNames(const LLUUID& cat_id) const; // Starting with the object specified, add its descendants to the // array provided, but do not add the inventory object specified diff --git a/indra/newview/llinventoryobserver.cpp b/indra/newview/llinventoryobserver.cpp index e4a985cb8e..897495a10e 100644 --- a/indra/newview/llinventoryobserver.cpp +++ b/indra/newview/llinventoryobserver.cpp @@ -677,7 +677,7 @@ void LLInventoryCategoriesObserver::changed(U32 mask) // computed, or (b) a name has changed. if (!cat_data.mIsNameHashInitialized || (mask & LLInventoryObserver::LABEL)) { - LLMD5 item_name_hash = gInventory.hashDirectDescendentNames(cat_id); + digest_t item_name_hash = gInventory.hashDirectDescendentNames(cat_id); if (cat_data.mItemNameHash != item_name_hash) { cat_data.mIsNameHashInitialized = true; @@ -738,7 +738,7 @@ bool LLInventoryCategoriesObserver::addCategory(const LLUUID& cat_id, callback_t { if(init_name_hash) { - LLMD5 item_name_hash = gInventory.hashDirectDescendentNames(cat_id); + digest_t item_name_hash = gInventory.hashDirectDescendentNames(cat_id); mCategoryMap.insert(category_map_value_t(cat_id,LLCategoryData(cat_id, cb, version, current_num_known_descendents,item_name_hash))); } else @@ -764,11 +764,10 @@ LLInventoryCategoriesObserver::LLCategoryData::LLCategoryData( , mDescendentsCount(num_descendents) , mIsNameHashInitialized(false) { - mItemNameHash.finalize(); } LLInventoryCategoriesObserver::LLCategoryData::LLCategoryData( - const LLUUID& cat_id, callback_t cb, S32 version, S32 num_descendents, LLMD5 name_hash) + const LLUUID& cat_id, callback_t cb, S32 version, S32 num_descendents, const digest_t& name_hash) : mCatID(cat_id) , mCallback(cb) diff --git a/indra/newview/llinventoryobserver.h b/indra/newview/llinventoryobserver.h index 36d8ee3f59..4af8102055 100644 --- a/indra/newview/llinventoryobserver.h +++ b/indra/newview/llinventoryobserver.h @@ -28,7 +28,6 @@ #define LL_LLINVENTORYOBSERVERS_H #include "lluuid.h" -#include "llmd5.h" #include #include @@ -271,14 +270,15 @@ public: void removeCategory(const LLUUID& cat_id); protected: + typedef LLUUID digest_t; // To clarify the actual usage of this "UUID" struct LLCategoryData { LLCategoryData(const LLUUID& cat_id, callback_t cb, S32 version, S32 num_descendents); - LLCategoryData(const LLUUID& cat_id, callback_t cb, S32 version, S32 num_descendents, LLMD5 name_hash); + LLCategoryData(const LLUUID& cat_id, callback_t cb, S32 version, S32 num_descendents, const digest_t& name_hash); callback_t mCallback; S32 mVersion; S32 mDescendentsCount; - LLMD5 mItemNameHash; + digest_t mItemNameHash; bool mIsNameHashInitialized; LLUUID mCatID; }; diff --git a/indra/newview/llmaterialmgr.h b/indra/newview/llmaterialmgr.h index 843dc66fbc..8f1ec8584b 100644 --- a/indra/newview/llmaterialmgr.h +++ b/indra/newview/llmaterialmgr.h @@ -93,7 +93,11 @@ private: bool operator==(const TEMaterialPair& b) const { return (materialID == b.materialID) && (te == b.te); } }; - + + // definitions follow class + friend std::hash; + friend size_t hash_value(const TEMaterialPair&) noexcept; + friend inline bool operator<( const LLMaterialMgr::TEMaterialPair& lhs, const LLMaterialMgr::TEMaterialPair& rhs) @@ -102,13 +106,6 @@ private: (lhs.materialID < rhs.materialID); } - struct TEMaterialPairHasher - { - enum { bucket_size = 8 }; - size_t operator()(const TEMaterialPair& key_value) const { return *((size_t*)key_value.materialID.get()); } // cheesy, but effective - bool operator()(const TEMaterialPair& left, const TEMaterialPair& right) const { return left < right; } - }; - typedef std::set material_queue_t; typedef std::map get_queue_t; typedef std::pair pending_material_t; @@ -116,7 +113,7 @@ private: typedef std::map get_callback_map_t; - typedef boost::unordered_map get_callback_te_map_t; + typedef boost::unordered_map get_callback_te_map_t; typedef std::set getall_queue_t; typedef std::map getall_pending_map_t; typedef std::map getall_callback_map_t; @@ -146,5 +143,23 @@ private: U32 getMaxEntries(const LLViewerRegion* regionp); }; +// std::hash implementation for TEMaterialPair +namespace std +{ + template<> struct hash + { + inline size_t operator()(const LLMaterialMgr::TEMaterialPair& p) const noexcept + { + return size_t((p.te + 1) * p.materialID.getDigest64()); + } + }; +} + +// For use with boost containers. +inline size_t hash_value(const LLMaterialMgr::TEMaterialPair& p) noexcept +{ + return size_t((p.te + 1) * p.materialID.getDigest64()); +} + #endif // LL_LLMATERIALMGR_H diff --git a/indra/newview/lloutfitobserver.cpp b/indra/newview/lloutfitobserver.cpp index 5bb69367aa..3ec5bcd9fc 100644 --- a/indra/newview/lloutfitobserver.cpp +++ b/indra/newview/lloutfitobserver.cpp @@ -34,7 +34,6 @@ LLOutfitObserver::LLOutfitObserver() : mCOFLastVersion(LLViewerInventoryCategory::VERSION_UNKNOWN) { - mItemNameHash.finalize(); gInventory.addObserver(this); } @@ -83,7 +82,7 @@ bool LLOutfitObserver::checkCOF() return false; bool cof_changed = false; - LLMD5 item_name_hash = gInventory.hashDirectDescendentNames(cof); + LLUUID item_name_hash = gInventory.hashDirectDescendentNames(cof); if (item_name_hash != mItemNameHash) { cof_changed = true; diff --git a/indra/newview/lloutfitobserver.h b/indra/newview/lloutfitobserver.h index 77041db68d..2f136d48e8 100644 --- a/indra/newview/lloutfitobserver.h +++ b/indra/newview/lloutfitobserver.h @@ -28,7 +28,6 @@ #define LL_OUTFITOBSERVER_H #include "llsingleton.h" -#include "llmd5.h" /** * Outfit observer facade that provides simple possibility to subscribe on @@ -78,7 +77,7 @@ protected: bool mLastOutfitDirtiness; - LLMD5 mItemNameHash; + LLUUID mItemNameHash; private: signal_t mBOFReplaced; diff --git a/indra/newview/llpathfindingnavmesh.cpp b/indra/newview/llpathfindingnavmesh.cpp index bc69446814..c297cac771 100644 --- a/indra/newview/llpathfindingnavmesh.cpp +++ b/indra/newview/llpathfindingnavmesh.cpp @@ -140,17 +140,12 @@ void LLPathfindingNavMesh::handleNavMeshResult(const LLSD &pContent, U32 pNavMes if ( pContent.has(NAVMESH_DATA_FIELD) ) { const LLSD::Binary &value = pContent.get(NAVMESH_DATA_FIELD).asBinary(); - // Use new variants of unzip_llsd calls - // unsigned int binSize = value.size(); - // std::string newStr(reinterpret_cast(&value[0]), binSize); - // std::istringstream streamdecomp( newStr ); - // + unsigned int binSize = value.size(); + std::string newStr(reinterpret_cast(&value[0]), binSize); + std::istringstream streamdecomp( newStr ); size_t decompBinSize = 0; bool valid = false; - // Use new variants of unzip_llsd calls - // U8* pUncompressedNavMeshContainer = unzip_llsdNavMesh( valid, decompBinSize, streamdecomp, binSize ) ; - U8* pUncompressedNavMeshContainer = unzip_llsdNavMesh( valid, decompBinSize, value.data(), value.size() ) ; - // + U8* pUncompressedNavMeshContainer = unzip_llsdNavMesh( valid, decompBinSize, streamdecomp, binSize ) ; if ( !valid ) { LL_WARNS() << "Unable to decompress the navmesh llsd." << LL_ENDL; diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp index 1c4f1f646b..f0d00e9728 100644 --- a/indra/newview/llpreviewnotecard.cpp +++ b/indra/newview/llpreviewnotecard.cpp @@ -39,7 +39,9 @@ #include "llinventorydefines.h" #include "llinventorymodel.h" #include "lllineeditor.h" +#include "llmd5.h" #include "llnotificationsutil.h" +#include "llmd5.h" #include "llresmgr.h" #include "roles_constants.h" #include "llscrollbar.h" diff --git a/indra/newview/llpreviewscript.cpp b/indra/newview/llpreviewscript.cpp index 2091727291..4f48c548a1 100644 --- a/indra/newview/llpreviewscript.cpp +++ b/indra/newview/llpreviewscript.cpp @@ -41,6 +41,7 @@ #include "llinventorymodel.h" #include "llkeyboard.h" #include "lllineeditor.h" +#include "llmd5.h" #include "llhelp.h" #include "llnotificationsutil.h" #include "llresmgr.h" diff --git a/indra/newview/llviewermenufile.cpp b/indra/newview/llviewermenufile.cpp index ded7958cf5..8fff9a12ff 100644 --- a/indra/newview/llviewermenufile.cpp +++ b/indra/newview/llviewermenufile.cpp @@ -131,7 +131,10 @@ std::queue LLFilePickerThread::sDeadQ; void LLFilePickerThread::getFile() { #if LL_WINDOWS + // Todo: get rid of LLFilePickerThread and make this modeless start(); +#elif LL_DARWIN + runModeless(); #else run(); #endif @@ -174,7 +177,82 @@ void LLFilePickerThread::run() LLMutexLock lock(sMutex); sDeadQ.push(this); } +} +void LLFilePickerThread::runModeless() +{ + BOOL result = FALSE; + LLFilePicker picker; + + if (mIsSaveDialog) + { + result = picker.getSaveFileModeless(mSaveFilter, + mProposedName, + modelessStringCallback, + this); + } + else if (mIsGetMultiple) + { + result = picker.getMultipleOpenFilesModeless(mLoadFilter, modelessVectorCallback, this); + } + else + { + result = picker.getOpenFileModeless(mLoadFilter, modelessVectorCallback, this); + } + + if (!result) + { + LLMutexLock lock(sMutex); + sDeadQ.push(this); + } +} + +void LLFilePickerThread::modelessStringCallback(bool success, + std::string &response, + void *user_data) +{ + LLFilePickerThread *picker = (LLFilePickerThread*)user_data; + if (success) + { + picker->mResponses.push_back(response); + } + + { + LLMutexLock lock(sMutex); + sDeadQ.push(picker); + } +} + +void LLFilePickerThread::modelessVectorCallback(bool success, + std::vector &responses, + void *user_data) +{ + LLFilePickerThread *picker = (LLFilePickerThread*)user_data; + if (success) + { + if (picker->mIsGetMultiple) + { + picker->mResponses = responses; + } + else + { + std::vector::iterator iter = responses.begin(); + while (iter != responses.end()) + { + if (!iter->empty()) + { + picker->mResponses.push_back(*iter); + break; + } + iter++; + } + } + } + + { + LLMutexLock lock(sMutex); + sDeadQ.push(picker); + } } //static diff --git a/indra/newview/llviewermenufile.h b/indra/newview/llviewermenufile.h index b3b4ad71b9..0e877853a5 100644 --- a/indra/newview/llviewermenufile.h +++ b/indra/newview/llviewermenufile.h @@ -106,6 +106,9 @@ public: void getFile(); virtual void run(); + void runModeless(); + static void modelessStringCallback(bool success, std::string &response, void *user_data); + static void modelessVectorCallback(bool success, std::vector &responses, void *user_data); virtual void notify(const std::vector& filenames) = 0; }; diff --git a/indra/newview/llviewerwearable.cpp b/indra/newview/llviewerwearable.cpp index b64e251649..958eefdaf9 100644 --- a/indra/newview/llviewerwearable.cpp +++ b/indra/newview/llviewerwearable.cpp @@ -40,6 +40,7 @@ #include "llviewerwearable.h" #include "llviewercontrol.h" #include "llviewerregion.h" +#include "llmd5.h" // [Legacy Bake] using namespace LLAvatarAppearanceDefines; diff --git a/indra/newview/llviewerwearable.h b/indra/newview/llviewerwearable.h index 09a734bfb1..6bf3e9a765 100644 --- a/indra/newview/llviewerwearable.h +++ b/indra/newview/llviewerwearable.h @@ -98,7 +98,6 @@ public: // not the wearable asset itself. void refreshName(); // [Legacy Bake] - ///*virtual*/void addToBakedTextureHash(LLMD5& hash) const {} /*virtual*/void addToBakedTextureHash(LLMD5& hash) const; protected: diff --git a/indra/newview/skins/default/xui/da/floater_perm_prefs.xml b/indra/newview/skins/default/xui/da/floater_perm_prefs.xml deleted file mode 100644 index eecddbcdb0..0000000000 --- a/indra/newview/skins/default/xui/da/floater_perm_prefs.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - -