diff --git a/.hgtags b/.hgtags index 07ddf93c11..30e96d4a18 100755 --- a/.hgtags +++ b/.hgtags @@ -471,3 +471,7 @@ d40c66e410741de7e90b1ed6dac28dd8a2d7e1f6 3.6.8-release 5b54b36862ff8bc3b6935673c9d1c1f22ee8d521 3.6.10-release 2feb70a4cfde43f2898d95ff8fcae3e67805c7c2 3.6.11-release 88bbfd7a6971033f3aa103f3a3500ceb4c73521b 3.6.12-release +0d9b9e50f1a8880e05f15688a9ec7d09e0e81013 3.6.13-release +5d746de933a98ca17887cde2fece80e9c7ab0b98 3.7.0-release +dcb4981ce255841b6083d8f65444b65d5a733a17 3.7.1-release +b842534cb4d76c9ef87676a62b1d2d19e79c015f 3.7.2-release diff --git a/BuildParams b/BuildParams index acc1a0f4a8..c0676c2afd 100755 --- a/BuildParams +++ b/BuildParams @@ -49,11 +49,13 @@ viewer_channel = "Second Life Test" # Setup default packaging parameters. sourceid = "" -additional_packages = "Amazon Desura B C" +additional_packages = "Amazon Desura A B C" Amazon_sourceid = "1207v_Amazon" Amazon_viewer_channel_suffix = "Amazon" Desura_sourceid = "1208_desura" Desura_viewer_channel_suffix = "Desura" +A_sourceid = "1300_A" +A_viewer_channel_suffix = "A" B_sourceid = "1301_B" B_viewer_channel_suffix = "B" C_sourceid = "1302_C" diff --git a/autobuild.xml b/autobuild.xml index 8194b3d4eb..6226c2137b 100755 --- a/autobuild.xml +++ b/autobuild.xml @@ -282,9 +282,9 @@ archive hash - aaea644191807f51051cefa2fac11069 + f7d9b6a9c624364389b71209881f39de url - http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-darwin-20110316.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Darwin/installer/curl-7.24.0-darwin-20130826.tar.bz2 name darwin @@ -294,9 +294,9 @@ archive hash - 2d9377951d99a1aa4735cea8d4b5aa71 + 58b7bf45383c1b1bc24afb303b1519c8 url - http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-linux-20110316.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Linux/installer/curl-7.24.0-linux-20130826.tar.bz2 name linux @@ -306,9 +306,9 @@ archive hash - fea96aa2a7d513397317194f3d6c979b + 8d9ccb0277a26bfe3f346c3c49ce4b58 url - http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-windows-20110211.tar.bz2 + http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/CYGWIN/installer/curl-7.24.0-windows-20130826.tar.bz2 name windows diff --git a/build.sh b/build.sh index ccc8476cea..f908139979 100755 --- a/build.sh +++ b/build.sh @@ -36,24 +36,59 @@ build_dir_CYGWIN() echo build-vc100 } +viewer_channel_suffix() +{ + local package_name="$1" + local suffix_var="${package_name}_viewer_channel_suffix" + local suffix=$(eval "echo \$${suffix_var}") + if [ "$suffix"x = ""x ] + then + echo "" + else + echo "_$suffix" + fi +} + installer_Darwin() { - ls -1tr "$(build_dir_Darwin ${last_built_variant:-Release})/newview/"*"$additional_package_name"*.dmg 2>/dev/null | sed 1q + local package_name="$1" + local package_dir="$(build_dir_Darwin ${last_built_variant:-Release})/newview/" + local pattern=".*$(viewer_channel_suffix ${package_name})_[0-9]+_[0-9]+_[0-9]+_[0-9]+_i386\\.dmg\$" + # since the additional packages are built after the base package, + # sorting oldest first ensures that the unqualified package is returned + # even if someone makes a qualified name that duplicates the last word of the base name + local package=$(ls -1tr "$package_dir" 2>/dev/null | grep -E "$pattern" | head -n 1) + test "$package"x != ""x && echo "$package_dir/$package" } installer_Linux() { - ls -1tr "$(build_dir_Linux ${last_built_variant:-Release})/newview/"*"$additional_package_name"*.tar.bz2 2>/dev/null | grep -v symbols | sed 1q + local package_name="$1" + local package_dir="$(build_dir_Linux ${last_built_variant:-Release})/newview/" + local pattern=".*$(viewer_channel_suffix ${package_name})_[0-9]+_[0-9]+_[0-9]+_[0-9]+_i686\\.tar\\.bz2\$" + # since the additional packages are built after the base package, + # sorting oldest first ensures that the unqualified package is returned + # even if someone makes a qualified name that duplicates the last word of the base name + package=$(ls -1tr "$package_dir" 2>/dev/null | grep -E "$pattern" | head -n 1) + test "$package"x != ""x && echo "$package_dir/$package" } installer_CYGWIN() { - v=${last_built_variant:-Release} - d=$(build_dir_CYGWIN $v) - if [ -r "$d/newview/$additional_package_name$v/touched.bat" ] + local package_name="$1" + local variant=${last_built_variant:-Release} + local build_dir=$(build_dir_CYGWIN ${variant}) + local package_dir + if [ "$package_name"x = ""x ] then - p=$(sed 's:.*=::' "$d/newview/$additional_package_name$v/touched.bat") - echo "$d/newview/$additional_package_name$v/$p" + package_dir="${build_dir}/newview/${variant}" + else + package_dir="${build_dir}/newview/${package_name}/${variant}" + fi + if [ -r "${package_dir}/touched.bat" ] + then + local package_file=$(sed 's:.*=::' "${package_dir}/touched.bat") + echo "${package_dir}/${package_file}" fi } @@ -275,7 +310,7 @@ then if $build_viewer_deb && [ "$last_built_variant" == "Release" ] then begin_section "Build Viewer Debian Package" - local have_private_repo=false + have_private_repo=false # mangle the changelog dch --force-bad-version \ --distribution unstable \ @@ -362,19 +397,15 @@ then # Upload additional packages. for package_id in $additional_packages do - case $arch in - CYGWIN) export additional_package_name="$package_id/" ;; - *) export additional_package_name=$package_id ;; - esac - package=$(installer_$arch) + package=$(installer_$arch "$package_id") if [ x"$package" != x ] then upload_item installer "$package" binary/octet-stream + upload_item quicklink "$package" binary/octet-stream else - record_failure "Failed to upload $package_id package ($package::$additional_package_name)." + record_failure "Failed to find additional package for '$package_id'." fi done - export additional_package_name="" case "$last_built_variant" in Release) @@ -384,6 +415,12 @@ then upload_item symbolfile "$build_dir/$symbolfile" binary/octet-stream done + # Upload the actual dependencies used + if [ -r "$build_dir/packages/installed-packages.xml" ] + then + upload_item installer "$build_dir/packages/installed-packages.xml" text/xml + fi + # Upload the llphysicsextensions_tpv package, if one was produced # *TODO: Make this an upload-extension if [ -r "$build_dir/llphysicsextensions_package" ] diff --git a/doc/contributions.txt b/doc/contributions.txt index 2f9d0c2c86..89390d9977 100755 --- a/doc/contributions.txt +++ b/doc/contributions.txt @@ -180,6 +180,9 @@ Ansariel Hiller MAINT-2368 STORM-1931 MAINT-2773 + BUG-3764 + STORM-1984 + STORM-1979 Aralara Rajal Arare Chantilly CHUIBUG-191 @@ -307,6 +310,7 @@ Christopher Organiser Ciaran Laval Cinder Roxley BUG-2326 + BUG-3863 OPEN-185 STORM-1703 STORM-1948 @@ -674,6 +678,19 @@ Jonathan Yap OPEN-161 STORM-1953 STORM-1957 + STORM-1993 + STORM-1980 + OPEN-113 + STORM-1975 + STORM-1982 + STORM-1975 + STORM-1987 + STORM-1982 + STORM-1992 + STORM-1989 + STORM-1987 + STORM-1986 + STORM-1981 Kadah Coba STORM-1060 STORM-1843 @@ -1140,6 +1157,7 @@ snowy Sidran Sovereign Engineer MAINT-2334 OPEN-189 + OPEN-195 SpacedOut Frye VWR-34 VWR-45 @@ -1259,6 +1277,8 @@ Tofu Buzzard Tony Kembia Tonya Souther STORM-1905 + BUG-3875 + BUG-3968 Torben Trautman TouchaHoney Perhaps TraductoresAnonimos Alter diff --git a/indra/edit-me-to-trigger-new-build.txt b/indra/edit-me-to-trigger-new-build.txt index 774e8c0676..beeb570496 100755 --- a/indra/edit-me-to-trigger-new-build.txt +++ b/indra/edit-me-to-trigger-new-build.txt @@ -4,3 +4,4 @@ Wed Nov 7 00:25:19 UTC 2012 + diff --git a/indra/lib/python/indra/base/lluuid.py b/indra/lib/python/indra/base/lluuid.py index 369ae4e92f..7413ffe10d 100755 --- a/indra/lib/python/indra/base/lluuid.py +++ b/indra/lib/python/indra/base/lluuid.py @@ -72,7 +72,7 @@ class UUID(object): ip = '' try: ip = socket.gethostbyname(socket.gethostname()) - except(socket.gaierror): + except(socket.gaierror, socket.error): # no ip address, so just default to somewhere in 10.x.x.x ip = '10' for i in range(3): diff --git a/indra/llaudio/llaudiodecodemgr.cpp b/indra/llaudio/llaudiodecodemgr.cpp index 6c97a64ed7..8c31f8b4de 100755 --- a/indra/llaudio/llaudiodecodemgr.cpp +++ b/indra/llaudio/llaudiodecodemgr.cpp @@ -135,7 +135,7 @@ int vfs_seek(void *datasource, ogg_int64_t offset, int whence) origin = -1; break; default: - llerrs << "Invalid whence argument to vfs_seek" << llendl; + LL_ERRS("AudioEngine") << "Invalid whence argument to vfs_seek" << LL_ENDL; return -1; } @@ -197,12 +197,12 @@ BOOL LLVorbisDecodeState::initDecode() vfs_callbacks.close_func = vfs_close; vfs_callbacks.tell_func = vfs_tell; - //llinfos << "Initing decode from vfile: " << mUUID << llendl; + LL_DEBUGS("AudioEngine") << "Initing decode from vfile: " << mUUID << LL_ENDL; mInFilep = new LLVFile(gVFS, mUUID, LLAssetType::AT_SOUND); if (!mInFilep || !mInFilep->getSize()) { - llwarns << "unable to open vorbis source vfile for reading" << llendl; + LL_WARNS("AudioEngine") << "unable to open vorbis source vfile for reading" << LL_ENDL; delete mInFilep; mInFilep = NULL; return FALSE; @@ -211,7 +211,7 @@ BOOL LLVorbisDecodeState::initDecode() int r = ov_open_callbacks(mInFilep, &mVF, NULL, 0, vfs_callbacks); if(r < 0) { - llwarns << r << " Input to vorbis decode does not appear to be an Ogg bitstream: " << mUUID << llendl; + LL_WARNS("AudioEngine") << r << " Input to vorbis decode does not appear to be an Ogg bitstream: " << mUUID << LL_ENDL; return(FALSE); } @@ -229,36 +229,36 @@ BOOL LLVorbisDecodeState::initDecode() if( vi->channels < 1 || vi->channels > LLVORBIS_CLIP_MAX_CHANNELS ) { abort_decode = true; - llwarns << "Bad channel count: " << vi->channels << llendl; + LL_WARNS("AudioEngine") << "Bad channel count: " << vi->channels << LL_ENDL; } } else // !vi { abort_decode = true; - llwarns << "No default bitstream found" << llendl; + LL_WARNS("AudioEngine") << "No default bitstream found" << LL_ENDL; } if( (size_t)sample_count > LLVORBIS_CLIP_REJECT_SAMPLES || (size_t)sample_count <= 0) { abort_decode = true; - llwarns << "Illegal sample count: " << sample_count << llendl; + LL_WARNS("AudioEngine") << "Illegal sample count: " << sample_count << LL_ENDL; } if( size_guess > LLVORBIS_CLIP_REJECT_SIZE || size_guess < 0) { abort_decode = true; - llwarns << "Illegal sample size: " << size_guess << llendl; + LL_WARNS("AudioEngine") << "Illegal sample size: " << size_guess << LL_ENDL; } if( abort_decode ) { - llwarns << "Canceling initDecode. Bad asset: " << mUUID << llendl; + LL_WARNS("AudioEngine") << "Canceling initDecode. Bad asset: " << mUUID << LL_ENDL; vorbis_comment* comment = ov_comment(&mVF,-1); if (comment && comment->vendor) { - llwarns << "Bad asset encoded by: " << comment->vendor << llendl; + LL_WARNS("AudioEngine") << "Bad asset encoded by: " << comment->vendor << LL_ENDL; } delete mInFilep; mInFilep = NULL; @@ -359,12 +359,12 @@ BOOL LLVorbisDecodeState::decodeSection() { if (!mInFilep) { - llwarns << "No VFS file to decode in vorbis!" << llendl; + LL_WARNS("AudioEngine") << "No VFS file to decode in vorbis!" << LL_ENDL; return TRUE; } if (mDone) { -// llwarns << "Already done with decode, aborting!" << llendl; +// LL_WARNS("AudioEngine") << "Already done with decode, aborting!" << LL_ENDL; return TRUE; } char pcmout[4096]; /*Flawfinder: ignore*/ @@ -377,14 +377,14 @@ BOOL LLVorbisDecodeState::decodeSection() eof = TRUE; mDone = TRUE; mValid = TRUE; -// llinfos << "Vorbis EOF" << llendl; +// LL_INFOS("AudioEngine") << "Vorbis EOF" << LL_ENDL; } else if (ret < 0) { /* error in the stream. Not a problem, just reporting it in case we (the app) cares. In this case, we don't. */ - llwarns << "BAD vorbis decode in decodeSection." << llendl; + LL_WARNS("AudioEngine") << "BAD vorbis decode in decodeSection." << LL_ENDL; mValid = FALSE; mDone = TRUE; @@ -393,7 +393,7 @@ BOOL LLVorbisDecodeState::decodeSection() } else { -// llinfos << "Vorbis read " << ret << "bytes" << llendl; +// LL_INFOS("AudioEngine") << "Vorbis read " << ret << "bytes" << LL_ENDL; /* we don't bother dealing with sample rate changes, etc, but. you'll have to*/ std::copy(pcmout, pcmout+ret, std::back_inserter(mWAVBuffer)); @@ -405,7 +405,7 @@ BOOL LLVorbisDecodeState::finishDecode() { if (!isValid()) { - llwarns << "Bogus vorbis decode state for " << getUUID() << ", aborting!" << llendl; + LL_WARNS("AudioEngine") << "Bogus vorbis decode state for " << getUUID() << ", aborting!" << LL_ENDL; return TRUE; // We've finished } @@ -480,7 +480,7 @@ BOOL LLVorbisDecodeState::finishDecode() if (36 == data_length) { - llwarns << "BAD Vorbis decode in finishDecode!" << llendl; + LL_WARNS("AudioEngine") << "BAD Vorbis decode in finishDecode!" << LL_ENDL; mValid = FALSE; return TRUE; // we've finished } @@ -497,7 +497,7 @@ BOOL LLVorbisDecodeState::finishDecode() { if (mBytesRead == 0) { - llwarns << "Unable to write file in LLVorbisDecodeState::finishDecode" << llendl; + LL_WARNS("AudioEngine") << "Unable to write file in LLVorbisDecodeState::finishDecode" << LL_ENDL; mValid = FALSE; return TRUE; // we've finished } @@ -515,7 +515,7 @@ BOOL LLVorbisDecodeState::finishDecode() LLVFile output(gVFS, mUUID, LLAssetType::AT_SOUND_WAV); output.write(&mWAVBuffer[0], mWAVBuffer.size()); #endif - //llinfos << "Finished decode for " << getUUID() << llendl; + LL_DEBUGS("AudioEngine") << "Finished decode for " << getUUID() << LL_ENDL; return TRUE; } @@ -524,7 +524,7 @@ void LLVorbisDecodeState::flushBadFile() { if (mInFilep) { - llwarns << "Flushing bad vorbis file from VFS for " << mUUID << llendl; + LL_WARNS("AudioEngine") << "Flushing bad vorbis file from VFS for " << mUUID << LL_ENDL; mInFilep->remove(); } } @@ -568,7 +568,7 @@ void LLAudioDecodeMgr::Impl::processQueue(const F32 num_secs) if (mCurrentDecodep->isDone() && !mCurrentDecodep->isValid()) { // We had an error when decoding, abort. - llwarns << mCurrentDecodep->getUUID() << " has invalid vorbis data, aborting decode" << llendl; + LL_WARNS("AudioEngine") << mCurrentDecodep->getUUID() << " has invalid vorbis data, aborting decode" << LL_ENDL; mCurrentDecodep->flushBadFile(); LLAudioData *adp = gAudiop->getAudioData(mCurrentDecodep->getUUID()); adp->setHasValidData(false); @@ -590,7 +590,7 @@ void LLAudioDecodeMgr::Impl::processQueue(const F32 num_secs) LLAudioData *adp = gAudiop->getAudioData(mCurrentDecodep->getUUID()); if (!adp) { - llwarns << "Missing LLAudioData for decode of " << mCurrentDecodep->getUUID() << llendl; + LL_WARNS("AudioEngine") << "Missing LLAudioData for decode of " << mCurrentDecodep->getUUID() << LL_ENDL; } else if (mCurrentDecodep->isValid() && mCurrentDecodep->isDone()) { @@ -601,12 +601,12 @@ void LLAudioDecodeMgr::Impl::processQueue(const F32 num_secs) // At this point, we could see if anyone needs this sound immediately, but // I'm not sure that there's a reason to - we need to poll all of the playing // sounds anyway. - //llinfos << "Finished the vorbis decode, now what?" << llendl; + //LL_INFOS("AudioEngine") << "Finished the vorbis decode, now what?" << LL_ENDL; } else { adp->setHasCompletedDecode(true); - llinfos << "Vorbis decode failed for " << mCurrentDecodep->getUUID() << llendl; + LL_INFOS("AudioEngine") << "Vorbis decode failed for " << mCurrentDecodep->getUUID() << LL_ENDL; } mCurrentDecodep = NULL; } @@ -631,7 +631,7 @@ void LLAudioDecodeMgr::Impl::processQueue(const F32 num_secs) continue; } - lldebugs << "Decoding " << uuid << " from audio queue!" << llendl; + lldebugs << "Decoding " << uuid << " from audio queue!" << LL_ENDL; std::string uuid_str; std::string d_path; @@ -674,19 +674,19 @@ BOOL LLAudioDecodeMgr::addDecodeRequest(const LLUUID &uuid) if (gAudiop->hasDecodedFile(uuid)) { // Already have a decoded version, don't need to decode it. - //llinfos << "addDecodeRequest for " << uuid << " has decoded file already" << llendl; + LL_DEBUGS("AudioEngine") << "addDecodeRequest for " << uuid << " has decoded file already" << LL_ENDL; return TRUE; } if (gAssetStorage->hasLocalAsset(uuid, LLAssetType::AT_SOUND)) { // Just put it on the decode queue. - //llinfos << "addDecodeRequest for " << uuid << " has local asset file already" << llendl; + LL_DEBUGS("AudioEngine") << "addDecodeRequest for " << uuid << " has local asset file already" << LL_ENDL; mImpl->mDecodeQueue.push(uuid); return TRUE; } - //llinfos << "addDecodeRequest for " << uuid << " no file available" << llendl; + LL_DEBUGS("AudioEngine") << "addDecodeRequest for " << uuid << " no file available" << LL_ENDL; return FALSE; } diff --git a/indra/llaudio/llaudioengine.cpp b/indra/llaudio/llaudioengine.cpp index 06e752cf34..ca614f5395 100755 --- a/indra/llaudio/llaudioengine.cpp +++ b/indra/llaudio/llaudioengine.cpp @@ -123,7 +123,7 @@ bool LLAudioEngine::init(const S32 num_channels, void* userdata) // Initialize the decode manager gAudioDecodeMgrp = new LLAudioDecodeMgr; - llinfos << "LLAudioEngine::init() AudioEngine successfully initialized" << llendl; + LL_INFOS("AudioEngine") << "LLAudioEngine::init() AudioEngine successfully initialized" << LL_ENDL; return true; } @@ -308,7 +308,7 @@ void LLAudioEngine::idle(F32 max_decode_time) LLAudioChannel *channelp = getFreeChannel(max_priority); if (channelp) { - //llinfos << "Replacing source in channel due to priority!" << llendl; + LL_DEBUGS("AudioEngine") << "Replacing source in channel due to priority!" << LL_ENDL; max_sourcep->setChannel(channelp); channelp->setSource(max_sourcep); if (max_sourcep->isSyncSlave()) @@ -479,7 +479,7 @@ void LLAudioEngine::idle(F32 max_decode_time) { if (!mBuffers[i]->mInUse && mBuffers[i]->mLastUseTimer.getElapsedTimeF32() > 30.f) { - //llinfos << "Flushing unused buffer!" << llendl; + LL_DEBUGS("AudioEngine") << "Flushing unused buffer!" << LL_ENDL; mBuffers[i]->mAudioDatap->mBufferp = NULL; delete mBuffers[i]; mBuffers[i] = NULL; @@ -591,8 +591,8 @@ LLAudioBuffer * LLAudioEngine::getFreeBuffer() if (buffer_id >= 0) { - lldebugs << "Taking over unused buffer " << buffer_id << llendl; - //llinfos << "Flushing unused buffer!" << llendl; + lldebugs << "Taking over unused buffer " << buffer_id << LL_ENDL; + LL_DEBUGS("AudioEngine") << "Flushing unused buffer!" << LL_ENDL; mBuffers[buffer_id]->mAudioDatap->mBufferp = NULL; delete mBuffers[buffer_id]; mBuffers[buffer_id] = createBuffer(); @@ -673,6 +673,8 @@ void LLAudioEngine::cleanupBuffer(LLAudioBuffer *bufferp) bool LLAudioEngine::preloadSound(const LLUUID &uuid) { + LL_DEBUGS("AudioEngine")<<"( "<getAudioData(uuid); // We don't care about the return value, this is just to make sure // that we have an entry, which will mean that the audio engine knows about this @@ -684,7 +686,7 @@ bool LLAudioEngine::preloadSound(const LLUUID &uuid) // At some point we need to have the audio/asset system check the static VFS // before it goes off and fetches stuff from the server. - //llwarns << "Used internal preload for non-local sound" << llendl; + LL_DEBUGS("AudioEngine") << "Used internal preload for non-local sound "<< uuid << LL_ENDL; return false; } @@ -815,7 +817,7 @@ void LLAudioEngine::triggerSound(const LLUUID &audio_uuid, const LLUUID& owner_i const S32 type, const LLVector3d &pos_global) { // Create a new source (since this can't be associated with an existing source. - //llinfos << "Localized: " << audio_uuid << llendl; + LL_DEBUGS("AudioEngine") << "Localized: " << audio_uuid << LL_ENDL; if (mMuted) { @@ -982,11 +984,14 @@ void LLAudioEngine::cleanupAudioSource(LLAudioSource *asp) iter = mAllSources.find(asp->getID()); if (iter == mAllSources.end()) { - llwarns << "Cleaning up unknown audio source!" << llendl; - return; + LL_WARNS("AudioEngine") << "Cleaning up unknown audio source!" << LL_ENDL; + } + else + { + LL_DEBUGS("AudioEngine") << "Cleaning up audio sources for "<< asp->getID() <getExists(uuid, LLAssetType::AT_SOUND); + bool have_local = gVFS->getExists(uuid, LLAssetType::AT_SOUND); + LL_DEBUGS("AudioEngine") << "sound uuid "<mCurrentTransfer = asset_id; gAudiop->mCurrentTransferTimer.reset(); gAssetStorage->getAssetData(asset_id, LLAssetType::AT_SOUND, @@ -1211,7 +1218,7 @@ void LLAudioEngine::startNextTransfer() } else { - //llinfos << "No pending transfers?" << llendl; + //LL_DEBUGS("AudioEngine") << "No pending transfers?" << LL_ENDL; } } @@ -1221,7 +1228,7 @@ void LLAudioEngine::assetCallback(LLVFS *vfs, const LLUUID &uuid, LLAssetType::E { if (result_code) { - llinfos << "Boom, error in audio file transfer: " << LLAssetStorage::getErrorString( result_code ) << " (" << result_code << ")" << llendl; + LL_INFOS("AudioEngine") << "Boom, error in audio file transfer: " << LLAssetStorage::getErrorString( result_code ) << " (" << result_code << ")" << LL_ENDL; // Need to mark data as bad to avoid constant rerequests. LLAudioData *adp = gAudiop->getAudioData(uuid); if (adp) @@ -1238,11 +1245,11 @@ void LLAudioEngine::assetCallback(LLVFS *vfs, const LLUUID &uuid, LLAssetType::E if (!adp) { // Should never happen - llwarns << "Got asset callback without audio data for " << uuid << llendl; + LL_WARNS("AudioEngine") << "Got asset callback without audio data for " << uuid << LL_ENDL; } else { - // llinfos << "Got asset callback with good audio data for " << uuid << ", making decode request" << llendl; + LL_DEBUGS("AudioEngine") << "Got asset callback with good audio data for " << uuid << ", making decode request" << LL_ENDL; adp->setHasValidData(true); adp->setHasLocalData(true); gAudioDecodeMgrp->addDecodeRequest(uuid); @@ -1321,7 +1328,7 @@ void LLAudioSource::update() } else if (adp->hasCompletedDecode()) // Only mark corrupted after decode is done { - llwarns << "Marking LLAudioSource corrupted for " << adp->getID() << llendl; + LL_WARNS("AudioEngine") << "Marking LLAudioSource corrupted for " << adp->getID() << LL_ENDL; mCorrupted = true ; } } @@ -1357,7 +1364,6 @@ bool LLAudioSource::setupChannel() if (!adp->getBuffer()) { // We're not ready to play back the sound yet, so don't try and allocate a channel for it. - //llwarns << "Aborting, no buffer" << llendl; return false; } @@ -1375,7 +1381,7 @@ bool LLAudioSource::setupChannel() // Ugh, we don't have any free channels. // Now we have to reprioritize. // For now, just don't play the sound. - //llwarns << "Aborting, no free channels" << llendl; + //llwarns << "Aborting, no free channels" << LL_ENDL; return false; } @@ -1474,7 +1480,7 @@ bool LLAudioSource::isDone() const { // We don't have a channel assigned, and it's been // over 15 seconds since we tried to play it. Don't bother. - //llinfos << "No channel assigned, source is done" << llendl; + LL_DEBUGS("AudioEngine") << "No channel assigned, source is done" << LL_ENDL; return true; } else @@ -1640,7 +1646,7 @@ LLAudioChannel::LLAudioChannel() : LLAudioChannel::~LLAudioChannel() { // Need to disconnect any sources which are using this channel. - //llinfos << "Cleaning up audio channel" << llendl; + LL_DEBUGS("AudioEngine") << "Cleaning up audio channel" << LL_ENDL; if (mCurrentSourcep) { mCurrentSourcep->setChannel(NULL); @@ -1651,29 +1657,29 @@ LLAudioChannel::~LLAudioChannel() void LLAudioChannel::setSource(LLAudioSource *sourcep) { - //llinfos << this << ": setSource(" << sourcep << ")" << llendl; - if (!sourcep) { // Clearing the source for this channel, don't need to do anything. - //llinfos << "Clearing source for channel" << llendl; + LL_DEBUGS("AudioEngine") << "Clearing source" << ( mCurrentSourcep ? mCurrentSourcep->getID() : LLUUID::null ) << LL_ENDL; cleanup(); mCurrentSourcep = NULL; mWaiting = false; - return; } - - if (sourcep == mCurrentSourcep) + else { - // Don't reallocate the channel, this will make FMOD goofy. - //llinfos << "Calling setSource with same source!" << llendl; + LL_DEBUGS("AudioEngine") << "( id: " << sourcep->getID() << ")" << LL_ENDL; + + if (sourcep == mCurrentSourcep) + { + // Don't reallocate the channel, this will make FMOD goofy. + //LL_DEBUGS("AudioEngine") << "Calling setSource with same source!" << LL_ENDL; + } + + mCurrentSourcep = sourcep; + + updateBuffer(); + update3DPosition(); } - - mCurrentSourcep = sourcep; - - - updateBuffer(); - update3DPosition(); } @@ -1768,7 +1774,7 @@ bool LLAudioData::load() if (mBufferp) { // We already have this sound in a buffer, don't do anything. - llinfos << "Already have a buffer for this sound, don't bother loading!" << llendl; + LL_INFOS("AudioEngine") << "Already have a buffer for this sound, don't bother loading!" << LL_ENDL; return true; } @@ -1776,7 +1782,7 @@ bool LLAudioData::load() if (!mBufferp) { // No free buffers, abort. - llinfos << "Not able to allocate a new audio buffer, aborting." << llendl; + LL_INFOS("AudioEngine") << "Not able to allocate a new audio buffer, aborting." << LL_ENDL; return true; } diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 8eb0c6249d..8767616a70 100755 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -43,6 +43,7 @@ set(llcommon_SOURCE_FILES llcriticaldamp.cpp llcursortypes.cpp lldate.cpp + lldeadmantimer.cpp lldependencies.cpp lldictionary.cpp llerror.cpp @@ -79,6 +80,7 @@ set(llcommon_SOURCE_FILES llptrto.cpp llprocess.cpp llprocessor.cpp + llprocinfo.cpp llqueuedthread.cpp llrand.cpp llrefcount.cpp @@ -146,6 +148,7 @@ set(llcommon_HEADER_FILES lldarray.h lldarrayptr.h lldate.h + lldeadmantimer.h lldefs.h lldependencies.h lldeleteutils.h @@ -207,6 +210,7 @@ set(llcommon_HEADER_FILES llpriqueuemap.h llprocess.h llprocessor.h + llprocinfo.h llptrskiplist.h llptrskipmap.h llptrto.h @@ -324,12 +328,14 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(bitpack "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llbase64 "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lldate "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(lldeadmantimer "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lldependencies "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llerror "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llframetimer "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llinstancetracker "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lllazy "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(llprocinfo "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llsingleton "" "${test_libs}") diff --git a/indra/llcommon/lldeadmantimer.cpp b/indra/llcommon/lldeadmantimer.cpp new file mode 100644 index 0000000000..7d9097e344 --- /dev/null +++ b/indra/llcommon/lldeadmantimer.cpp @@ -0,0 +1,188 @@ +/** +* @file lldeadmantimer.cpp +* @brief Simple deadman-switch timer. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +* $/LicenseInfo$ +*/ + + +#include "lldeadmantimer.h" + + +// *TODO: Currently, this uses lltimer functions for its time +// aspects and this leaks into the apis in the U64s/F64s. Would +// like to perhaps switch this over to TSC register-based timers +// sometime and drop the overhead some more. + + +// Flag states and their meaning: +// mActive mDone Meaning +// false false Nothing running, no result available +// true false Timer running, no result available +// false true Timer finished, result can be read once +// true true Not allowed +// +LLDeadmanTimer::LLDeadmanTimer(F64 horizon, bool inc_cpu) + : mHorizon(time_type(llmax(horizon, F64(0.0)) * gClockFrequency)), + mActive(false), // If true, a timer is running. + mDone(false), // If true, timer has completed and can be read (once) + mStarted(U64L(0)), + mExpires(U64L(0)), + mStopped(U64L(0)), + mCount(U64L(0)), + mIncCPU(inc_cpu), + mUStartCPU(LLProcInfo::time_type(U64L(0))), + mUEndCPU(LLProcInfo::time_type(U64L(0))), + mSStartCPU(LLProcInfo::time_type(U64L(0))), + mSEndCPU(LLProcInfo::time_type(U64L(0))) +{} + + +// static +LLDeadmanTimer::time_type LLDeadmanTimer::getNow() +{ + return LLTimer::getCurrentClockCount(); +} + + +void LLDeadmanTimer::start(time_type now) +{ + // *TODO: If active, let's complete an existing timer and save + // the result to the side. I think this will be useful later. + // For now, wipe out anything in progress, start fresh. + + if (! now) + { + now = LLTimer::getCurrentClockCount(); + } + mActive = true; + mDone = false; + mStarted = now; + mExpires = now + mHorizon; + mStopped = now; + mCount = U64L(0); + if (mIncCPU) + { + LLProcInfo::getCPUUsage(mUStartCPU, mSStartCPU); + } +} + + +void LLDeadmanTimer::stop(time_type now) +{ + if (! mActive) + { + return; + } + + if (! now) + { + now = getNow(); + } + mStopped = now; + mActive = false; + mDone = true; + if (mIncCPU) + { + LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); + } +} + + +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, + U64 & user_cpu, U64 & sys_cpu) +{ + const bool status(isExpired(now, started, stopped, count)); + if (status) + { + user_cpu = U64(mUEndCPU - mUStartCPU); + sys_cpu = U64(mSEndCPU - mSStartCPU); + } + return status; +} + + +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count) +{ + if (mActive && ! mDone) + { + if (! now) + { + now = getNow(); + } + + if (now >= mExpires) + { + // mStopped from ringBell() is the value we want + mActive = false; + mDone = true; + } + } + + if (! mDone) + { + return false; + } + + started = mStarted * gClockFrequencyInv; + stopped = mStopped * gClockFrequencyInv; + count = mCount; + mDone = false; + + return true; +} + + +void LLDeadmanTimer::ringBell(time_type now, unsigned int count) +{ + if (! mActive) + { + return; + } + + if (! now) + { + now = getNow(); + } + + if (now >= mExpires) + { + // Timer has expired, this event will be dropped + mActive = false; + mDone = true; + } + else + { + // Timer renewed, keep going + mStopped = now; + mExpires = now + mHorizon; + mCount += count; + if (mIncCPU) + { + LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); + } + } + + return; +} + diff --git a/indra/llcommon/lldeadmantimer.h b/indra/llcommon/lldeadmantimer.h new file mode 100644 index 0000000000..980976e176 --- /dev/null +++ b/indra/llcommon/lldeadmantimer.h @@ -0,0 +1,214 @@ +/** +* @file lldeadmantimer.h +* @brief Interface to a simple event timer with a deadman's switch +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +* $/LicenseInfo$ +*/ + +#ifndef LL_DEADMANTIMER_H +#define LL_DEADMANTIMER_H + + +#include "linden_common.h" + +#include "lltimer.h" +#include "llprocinfo.h" + + +/// @file lldeadmantimer.h +/// +/// There are interesting user-experienced events in the viewer that +/// would seem to have well-defined start and stop points but which +/// actually lack such milestones in the code. Such events (like +/// time to load meshes after logging in, initial inventory load, +/// display name fetch) can be defined somewhat after-the-fact by +/// noticing when we no longer perform operations towards their +/// completion. This class is intended to help in such applications. +/// +/// What it implements is a deadman's switch (also known as a +/// keepalive switch and a doorbell switch). The basic operation is +/// as follows: +/// +/// * LLDeadmanTimer is instantiated with a horizon value in seconds, +/// one for each event of interest. +/// * When an event starts, @see start() is invoked to begin a +/// timing operation. +/// * As operations are performed in service of the event (issuing +/// HTTP requests, receiving responses), @see ringBell() is invoked +/// to inform the timer that the operation is still active. +/// * If the operation is canceled or otherwise terminated, @see +/// stop() can be called to end the timing operation. +/// * Concurrent with the ringBell() calls, the program makes +/// periodic (shorter than the horizon but not too short) calls +/// to @see isExpired() to see if the event has expired due to +/// either a stop() call or lack of activity (defined as a ringBell() +/// call in the previous 'horizon' seconds). If it has expired, +/// the caller also receives start, stop and count values for the +/// event which the application can then report in whatever manner +/// it sees fit. +/// * The timer becomes passive after an isExpired() call that returns +/// true. It can then be restarted with a new start() call. +/// +/// Threading: Instances are not thread-safe. They also use +/// timing code from lltimer.h which is also unsafe. +/// +/// Allocation: Not refcounted, may be stack or heap allocated. +/// + +class LL_COMMON_API LLDeadmanTimer +{ +public: + /// Public types + + /// Low-level time type chosen for compatibility with + /// LLTimer::getCurrentClockCount() which is the basis + /// of time operations in this class. This is likely + /// to change in a future version in a move to TSC-based + /// timing. + typedef U64 time_type; + +public: + /// Construct and initialize an LLDeadmanTimer + /// + /// @param horizon Time, in seconds, after the last @see ringBell() + /// call at which point the timer will consider itself + /// expired. + /// + /// @param inc_cpu If true, gather system and user cpu stats while + /// running the timer. This does require more syscalls + /// during updates. If false, cpu usage data isn't + /// collected and will be zero if queried. + LLDeadmanTimer(F64 horizon, bool inc_cpu); + + ~LLDeadmanTimer() + {} + +private: + LLDeadmanTimer(const LLDeadmanTimer &); // Not defined + void operator=(const LLDeadmanTimer &); // Not defined + +public: + /// Get the current time. Zero-basis for this time + /// representation is not defined and is different on + /// different platforms. Do not attempt to compute + /// negative times relative to the first value returned, + /// there may not be enough 'front porch' on the range + /// to prevent wraparound. + /// + /// Note: Implementation is expected to change in a + /// future release as well. + /// + static time_type getNow(); + + /// Begin timing. If the timer is already active, it is reset + /// and timing begins now. + /// + /// @param now Current time as returned by @see + /// LLTimer::getCurrentClockCount(). If zero, + /// method will lookup current time. + /// + void start(time_type now); + + /// End timing. Actively declare the end of the event independent + /// of the deadman's switch operation. @see isExpired() will return + /// true and appropriate values will be returned. + /// + /// @param now Current time as returned by @see + /// LLTimer::getCurrentClockCount(). If zero, + /// method will lookup current time. + /// + void stop(time_type now); + + /// Declare that something interesting happened. This has two + /// effects on an unexpired-timer. 1) The expiration time + /// is extended for 'horizon' seconds after the 'now' value. + /// 2) An internal counter associated with the event is incremented + /// by the @ref count parameter. This count is returned via the + /// @see isExpired() method. + /// + /// @param now Current time as returned by @see + /// LLTimer::getCurrentClockCount(). If zero, + /// method will lookup current time. + /// + /// @param count Count of events to be associated with + /// this bell ringing. + /// + void ringBell(time_type now, unsigned int count); + + /// Checks the status of the timer. If the timer has expired, + /// also returns various timer-related stats. Unlike ringBell(), + /// does not extend the horizon, it only checks for expiration. + /// + /// @param now Current time as returned by @see + /// LLTimer::getCurrentClockCount(). If zero, + /// method will lookup current time. + /// + /// @param started If expired, the starting time of the event is + /// returned to the caller via this reference. + /// + /// @param stopped If expired, the ending time of the event is + /// returned to the caller via this reference. + /// Ending time will be that provided in the + /// stop() method or the last ringBell() call + /// leading to expiration, whichever (stop() call + /// or notice of expiration) happened first. + /// + /// @param count If expired, the number of ringBell() calls + /// made prior to expiration. + /// + /// @param user_cpu Amount of CPU spent in user mode by the process + /// during the event. Value in microseconds and will + /// read zero if not enabled by the constructor. + /// + /// @param sys_cpu Amount of CPU spent in system mode by the process. + /// + /// @return true if the timer has expired, false otherwise. + /// If true, it also returns the started, + /// stopped and count values otherwise these are + /// left unchanged. + /// + bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, + U64 & user_cpu, U64 & sys_cpu); + + /// Identical to the six-arugment form except it does without the + /// CPU time return if the caller isn't interested in it. + bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count); + +protected: + time_type mHorizon; + bool mActive; + bool mDone; + time_type mStarted; + time_type mExpires; + time_type mStopped; + time_type mCount; + + const bool mIncCPU; // Include CPU metrics in timer + LLProcInfo::time_type mUStartCPU; + LLProcInfo::time_type mUEndCPU; + LLProcInfo::time_type mSStartCPU; + LLProcInfo::time_type mSEndCPU; +}; + + +#endif // LL_DEADMANTIMER_H diff --git a/indra/llcommon/llprocinfo.cpp b/indra/llcommon/llprocinfo.cpp new file mode 100644 index 0000000000..c00f979b0b --- /dev/null +++ b/indra/llcommon/llprocinfo.cpp @@ -0,0 +1,94 @@ +/** +* @file llprocinfo.cpp +* @brief Process, cpu and resource usage information APIs. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +* $/LicenseInfo$ +*/ + + +#include "llprocinfo.h" + +#if LL_WINDOWS + +#define PSAPI_VERSION 1 +#include "windows.h" +#include "psapi.h" + +#elif LL_DARWIN + +#include +#include + +#else + +#include +#include + +#endif // LL_WINDOWS/LL_DARWIN + + +// static +void LLProcInfo::getCPUUsage(time_type & user_time, time_type & system_time) +{ +#if LL_WINDOWS + + HANDLE self(GetCurrentProcess()); // Does not have to be closed + FILETIME ft_dummy, ft_system, ft_user; + + GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); + ULARGE_INTEGER uli; + uli.u.LowPart = ft_system.dwLowDateTime; + uli.u.HighPart = ft_system.dwHighDateTime; + system_time = uli.QuadPart / U64L(10); // Convert to uS + uli.u.LowPart = ft_user.dwLowDateTime; + uli.u.HighPart = ft_user.dwHighDateTime; + user_time = uli.QuadPart / U64L(10); + +#elif LL_DARWIN + + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage)) + { + user_time = system_time = time_type(0U); + return; + } + user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; + system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; + +#else // Linux + + struct rusage usage; + + if (getrusage(RUSAGE_SELF, &usage)) + { + user_time = system_time = time_type(0U); + return; + } + user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; + system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; + +#endif // LL_WINDOWS/LL_DARWIN/Linux +} + + diff --git a/indra/llcommon/llprocinfo.h b/indra/llcommon/llprocinfo.h new file mode 100644 index 0000000000..e78bcf490a --- /dev/null +++ b/indra/llcommon/llprocinfo.h @@ -0,0 +1,68 @@ +/** +* @file llprocinfo.h +* @brief Interface to process/cpu/resource information services. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +* $/LicenseInfo$ +*/ + +#ifndef LL_PROCINFO_H +#define LL_PROCINFO_H + + +#include "linden_common.h" + +/// @file llprocinfo.h +/// +/// Right now, this is really a namespace disguised as a class. +/// It wraps some types and functions to return information about +/// process resource consumption in a non-OS-specific manner. +/// +/// Threading: No instances so that's thread-safe. Implementations +/// of static functions should be thread-safe, they mostly involve +/// direct syscall invocations. +/// +/// Allocation: Not instantiatable. + +class LL_COMMON_API LLProcInfo +{ +public: + /// Public types + + typedef U64 time_type; /// Relative microseconds + +private: + LLProcInfo(); // Not defined + ~LLProcInfo(); // Not defined + LLProcInfo(const LLProcInfo &); // Not defined + void operator=(const LLProcInfo &); // Not defined + +public: + /// Get accumulated system and user CPU time in + /// microseconds. Syscalls involved in every invocation. + /// + /// Threading: expected to be safe. + static void getCPUUsage(time_type & user_time, time_type & system_time); +}; + + +#endif // LL_PROCINFO_H diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 60adeeaeb7..e67d1bc57b 100755 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -3,7 +3,7 @@ * * $LicenseInfo:firstyear=2004&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -373,6 +373,36 @@ void LLMutex::lock() #endif } +bool LLMutex::trylock() +{ + if(isSelfLocked()) + { //redundant lock + mCount++; + return true; + } + + apr_status_t status(apr_thread_mutex_trylock(mAPRMutexp)); + if (APR_STATUS_IS_EBUSY(status)) + { + return false; + } + +#if MUTEX_DEBUG + // Have to have the lock before we can access the debug info + U32 id = LLThread::currentID(); + if (mIsLocked[id] != FALSE) + llerrs << "Already locked in Thread: " << id << llendl; + mIsLocked[id] = TRUE; +#endif + +#if LL_DARWIN + mLockingThread = LLThread::currentID(); +#else + mLockingThread = sThreadID; +#endif + return true; +} + void LLMutex::unlock() { if (mCount > 0) diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index f51d985b5f..8c7143304f 100755 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2004&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -156,7 +156,8 @@ public: virtual ~LLMutex(); void lock(); // blocks - void unlock(); + bool trylock(); // non-blocking, returns true if lock held. + void unlock(); // undefined behavior when called on mutex not being held bool isLocked(); // non-blocking, but does do a lock/unlock so not free bool isSelfLocked(); //return true if locked in a same thread U32 lockingThread() const; //get ID of locking thread @@ -174,6 +175,8 @@ protected: #endif }; +//============================================================================ + // Actually a condition/mutex pair (since each condition needs to be associated with a mutex). class LL_COMMON_API LLCondition : public LLMutex { @@ -189,6 +192,8 @@ protected: apr_thread_cond_t *mAPRCondp; }; +//============================================================================ + class LLMutexLock { public: @@ -210,6 +215,43 @@ private: //============================================================================ +// Scoped locking class similar in function to LLMutexLock but uses +// the trylock() method to conditionally acquire lock without +// blocking. Caller resolves the resulting condition by calling +// the isLocked() method and either punts or continues as indicated. +// +// Mostly of interest to callers needing to avoid stalls who can +// guarantee another attempt at a later time. + +class LLMutexTrylock +{ +public: + LLMutexTrylock(LLMutex* mutex) + : mMutex(mutex), + mLocked(false) + { + if (mMutex) + mLocked = mMutex->trylock(); + } + + ~LLMutexTrylock() + { + if (mMutex && mLocked) + mMutex->unlock(); + } + + bool isLocked() const + { + return mLocked; + } + +private: + LLMutex* mMutex; + bool mLocked; +}; + +//============================================================================ + void LLThread::lockData() { mDataLock->lock(); diff --git a/indra/llcommon/lltimer.h b/indra/llcommon/lltimer.h index 513de0605d..e73741217c 100755 --- a/indra/llcommon/lltimer.h +++ b/indra/llcommon/lltimer.h @@ -146,6 +146,13 @@ static inline time_t time_max() } } +// These are really statics but they've been global for awhile +// and they're material to other timing classes. If you are +// not implementing a timer class, do not use these directly. +extern LL_COMMON_API F64 gClockFrequency; +extern LL_COMMON_API F64 gClockFrequencyInv; +extern LL_COMMON_API F64 gClocksToMicroseconds; + // Correction factor used by time_corrected() above. extern LL_COMMON_API S32 gUTCOffset; diff --git a/indra/llcommon/tests/lldeadmantimer_test.cpp b/indra/llcommon/tests/lldeadmantimer_test.cpp new file mode 100644 index 0000000000..7fd2dde6e0 --- /dev/null +++ b/indra/llcommon/tests/lldeadmantimer_test.cpp @@ -0,0 +1,628 @@ +/** + * @file lldeadmantimer_test.cpp + * @brief Tests for the LLDeadmanTimer class. + * + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "../lldeadmantimer.h" +#include "../llsd.h" +#include "../lltimer.h" + +#include "../test/lltut.h" + +// Convert between floating point time deltas and U64 time deltas. +// Reflects an implementation detail inside lldeadmantimer.cpp + +static LLDeadmanTimer::time_type float_time_to_u64(F64 delta) +{ + return LLDeadmanTimer::time_type(delta * gClockFrequency); +} + +static F64 u64_time_to_float(LLDeadmanTimer::time_type delta) +{ + return delta * gClockFrequencyInv; +} + + +namespace tut +{ + +struct deadmantimer_test +{ + deadmantimer_test() + { + // LLTimer internals updating + update_clock_frequencies(); + } +}; + +typedef test_group deadmantimer_group_t; +typedef deadmantimer_group_t::object deadmantimer_object_t; +tut::deadmantimer_group_t deadmantimer_instance("LLDeadmanTimer"); + +// Basic construction test and isExpired() call +template<> template<> +void deadmantimer_object_t::test<1>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(10.0, false); + + ensure_equals("WOCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count), false); + ensure_approximately_equals("WOCM t1 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WOCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WOCM t1 - isExpired() does not modify count", count, U64L(8)); + } + + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(10.0, true); + + ensure_equals("WCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); + ensure_approximately_equals("WCM t1 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WCM t1 - isExpired() does not modify count", count, U64L(8)); + ensure_equals("WCM t1 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); + ensure_equals("WCM t1 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); + } +} + + +// Construct with zero horizon - not useful generally but will be useful in testing +template<> template<> +void deadmantimer_object_t::test<2>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(0.0, false); // Zero is pre-expired + + ensure_equals("WOCM isExpired() still returns false with 0.0 time ctor()", + timer.isExpired(0, started, stopped, count), false); + } + + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(0.0, true); // Zero is pre-expired + + ensure_equals("WCM isExpired() still returns false with 0.0 time ctor()", + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); + } +} + + +// "pre-expired" timer - starting a timer with a 0.0 horizon will result in +// expiration on first test. +template<> template<> +void deadmantimer_object_t::test<3>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(0.0, false); + + timer.start(0); + ensure_equals("WOCM isExpired() returns true with 0.0 horizon time", + timer.isExpired(0, started, stopped, count), true); + ensure_approximately_equals("WOCM expired timer with no bell ringing has stopped == started", started, stopped, 8); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(0.0, true); + + timer.start(0); + ensure_equals("WCM isExpired() returns true with 0.0 horizon time", + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WCM expired timer with no bell ringing has stopped == started", started, stopped, 8); + } +} + + +// "pre-expired" timer - bell rings are ignored as we're already expired. +template<> template<> +void deadmantimer_object_t::test<4>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(0.0, false); + + timer.start(0); + timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); + ensure_equals("WOCM isExpired() returns true with 0.0 horizon time after bell ring", + timer.isExpired(0, started, stopped, count), true); + ensure_approximately_equals("WOCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(0.0, true); + + timer.start(0); + timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); + ensure_equals("WCM isExpired() returns true with 0.0 horizon time after bell ring", + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); + } +} + + +// start(0) test - unexpired timer reports unexpired +template<> template<> +void deadmantimer_object_t::test<5>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(10.0, false); + + timer.start(0); + ensure_equals("WOCM isExpired() returns false after starting with 10.0 horizon time", + timer.isExpired(0, started, stopped, count), false); + ensure_approximately_equals("WOCM t5 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WOCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WOCM t5 - isExpired() does not modify count", count, U64L(8)); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(10.0, true); + + timer.start(0); + ensure_equals("WCM isExpired() returns false after starting with 10.0 horizon time", + timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); + ensure_approximately_equals("WCM t5 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WCM t5 - isExpired() does not modify count", count, U64L(8)); + ensure_equals("WCM t5 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); + ensure_equals("WCM t5 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); + } +} + + +// start() test - start in the past but not beyond 1 horizon +template<> template<> +void deadmantimer_object_t::test<6>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(10.0, false); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); + timer.start(the_past); + ensure_equals("WOCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", + timer.isExpired(now, started, stopped, count), false); + ensure_approximately_equals("WOCM t6 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WOCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WOCM t6 - isExpired() does not modify count", count, U64L(8)); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(10.0, true); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); + timer.start(the_past); + ensure_equals("WCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + ensure_approximately_equals("WCM t6 - isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("t6 - isExpired() does not modify count", count, U64L(8)); + ensure_equals("WCM t6 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); + ensure_equals("WCM t6 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); + } +} + + +// start() test - start in the past but well beyond 1 horizon +template<> template<> +void deadmantimer_object_t::test<7>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(10.0, false); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); + timer.start(the_past); + ensure_equals("WOCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", + timer.isExpired(now,started, stopped, count), true); + ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(10.0, true); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); + timer.start(the_past); + ensure_equals("WCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", + timer.isExpired(now,started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); + } +} + + +// isExpired() test - results are read-once. Probes after first true are false. +template<> template<> +void deadmantimer_object_t::test<8>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(10.0, false); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); + timer.start(the_past); + ensure_equals("WOCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", + timer.isExpired(now, started, stopped, count), true); + + started = 42.0; + stopped = 97.0; + count = U64L(8); + ensure_equals("WOCM t8 - second isExpired() returns false after true", + timer.isExpired(now, started, stopped, count), false); + ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WOCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(10.0, true); + + // Would like to do subtraction on current time but can't because + // the implementation on Windows is zero-based. We wrap around + // the backside resulting in a large U64 number. + + LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); + LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); + timer.start(the_past); + ensure_equals("WCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); + + started = 42.0; + stopped = 97.0; + count = U64L(8); + user_cpu = 29000; + sys_cpu = 57000; + ensure_equals("WCM t8 - second isExpired() returns false after true", + timer.isExpired(now, started, stopped, count), false); + ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); + ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); + ensure_equals("WCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); + ensure_equals("WCM t8 - 2nd isExpired() does not modify user_cpu", user_cpu, U64L(29000)); + ensure_equals("WCM t8 - 2nd isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); + } +} + + +// ringBell() test - see that we can keep a timer from expiring +template<> template<> +void deadmantimer_object_t::test<9>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(5.0, false); + + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); + F64 real_start(u64_time_to_float(now)); + timer.start(0); + + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WOCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", + timer.isExpired(now, started, stopped, count), false); + F64 last_good_ring(u64_time_to_float(now)); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WOCM t9 - 5.0 horizon timer expires on 10-second jump", + timer.isExpired(now, started, stopped, count), true); + ensure_approximately_equals("WOCM t9 - started matches start() time", started, real_start, 4); + ensure_approximately_equals("WOCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WOCM t9 - 10 good ringBell()s", count, U64L(10)); + ensure_equals("WOCM t9 - single read only", timer.isExpired(now, started, stopped, count), false); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + LLDeadmanTimer timer(5.0, true); + + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); + F64 real_start(u64_time_to_float(now)); + timer.start(0); + + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + F64 last_good_ring(u64_time_to_float(now)); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WCM t9 - 5.0 horizon timer expires on 10-second jump", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WCM t9 - started matches start() time", started, real_start, 4); + ensure_approximately_equals("WCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WCM t9 - 10 good ringBell()s", count, U64L(10)); + ensure_equals("WCM t9 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + } +} + + +// restart after expiration test - verify that restarts behave well +template<> template<> +void deadmantimer_object_t::test<10>() +{ + { + // Without cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)); + LLDeadmanTimer timer(5.0, false); + + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); + F64 real_start(u64_time_to_float(now)); + timer.start(0); + + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", + timer.isExpired(now, started, stopped, count), false); + F64 last_good_ring(u64_time_to_float(now)); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WOCM t10 - 5.0 horizon timer expires on 10-second jump", + timer.isExpired(now, started, stopped, count), true); + ensure_approximately_equals("WOCM t10 - started matches start() time", started, real_start, 4); + ensure_approximately_equals("WOCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WOCM t10 - 10 good ringBell()s", count, U64L(10)); + ensure_equals("WOCM t10 - single read only", timer.isExpired(now, started, stopped, count), false); + + // Jump forward and restart + now += float_time_to_u64(1.0); + real_start = u64_time_to_float(now); + timer.start(now); + + // Run a modified bell ring sequence + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", + timer.isExpired(now, started, stopped, count), false); + last_good_ring = u64_time_to_float(now); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WOCM t10 - 5.0 horizon timer expires on 8-second jump", + timer.isExpired(now, started, stopped, count), true); + ensure_approximately_equals("WOCM t10 - 2nd started matches start() time", started, real_start, 4); + ensure_approximately_equals("WOCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WOCM t10 - 8 good ringBell()s", count, U64L(8)); + ensure_equals("WOCM t10 - single read only - 2nd start", + timer.isExpired(now, started, stopped, count), false); + } + { + // With cpu metrics + F64 started(42.0), stopped(97.0); + U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + + LLDeadmanTimer timer(5.0, true); + + LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); + F64 real_start(u64_time_to_float(now)); + timer.start(0); + + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + F64 last_good_ring(u64_time_to_float(now)); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WCM t10 - 5.0 horizon timer expires on 10-second jump", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WCM t10 - started matches start() time", started, real_start, 4); + ensure_approximately_equals("WCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WCM t10 - 10 good ringBell()s", count, U64L(10)); + ensure_equals("WCM t10 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + + // Jump forward and restart + now += float_time_to_u64(1.0); + real_start = u64_time_to_float(now); + timer.start(now); + + // Run a modified bell ring sequence + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + now += float_time_to_u64(1.0); + timer.ringBell(now, 1); + ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + last_good_ring = u64_time_to_float(now); + + // Jump forward and expire + now += float_time_to_u64(10.0); + ensure_equals("WCM t10 - 5.0 horizon timer expires on 8-second jump", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); + ensure_approximately_equals("WCM t10 - 2nd started matches start() time", started, real_start, 4); + ensure_approximately_equals("WCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); + ensure_equals("WCM t10 - 8 good ringBell()s", count, U64L(8)); + ensure_equals("WCM t10 - single read only - 2nd start", + timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); + } +} + + + +} // end namespace tut diff --git a/indra/llcommon/tests/llprocinfo_test.cpp b/indra/llcommon/tests/llprocinfo_test.cpp new file mode 100644 index 0000000000..12d5a695ee --- /dev/null +++ b/indra/llcommon/tests/llprocinfo_test.cpp @@ -0,0 +1,91 @@ +/** + * @file llprocinfo_test.cpp + * @brief Tests for the LLProcInfo class. + * + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2013, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "../llprocinfo.h" + +#include "../test/lltut.h" +#include "../lltimer.h" + + +static const LLProcInfo::time_type bad_user(289375U), bad_system(275U); + + +namespace tut +{ + +struct procinfo_test +{ + procinfo_test() + { + } +}; + +typedef test_group procinfo_group_t; +typedef procinfo_group_t::object procinfo_object_t; +tut::procinfo_group_t procinfo_instance("LLProcInfo"); + + +// Basic invocation works +template<> template<> +void procinfo_object_t::test<1>() +{ + LLProcInfo::time_type user(bad_user), system(bad_system); + + set_test_name("getCPUUsage() basic function"); + + LLProcInfo::getCPUUsage(user, system); + + ensure_not_equals("getCPUUsage() writes to its user argument", user, bad_user); + ensure_not_equals("getCPUUsage() writes to its system argument", system, bad_system); +} + + +// Time increases +template<> template<> +void procinfo_object_t::test<2>() +{ + LLProcInfo::time_type user(bad_user), system(bad_system); + LLProcInfo::time_type user2(bad_user), system2(bad_system); + + set_test_name("getCPUUsage() increases over time"); + + LLProcInfo::getCPUUsage(user, system); + + for (int i(0); i < 100000; ++i) + { + ms_sleep(0); + } + + LLProcInfo::getCPUUsage(user2, system2); + + ensure_equals("getCPUUsage() user value doesn't decrease over time", user2 >= user, true); + ensure_equals("getCPUUsage() system value doesn't decrease over time", system2 >= system, true); +} + + +} // end namespace tut diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden new file mode 100644 index 0000000000..eb6ccab3bc --- /dev/null +++ b/indra/llcorehttp/README.Linden @@ -0,0 +1,671 @@ + + + +1. HTTP Fetching in 15 Minutes + + Let's start with a trivial working example. You'll need a throwaway + build of the viewer. And we'll use indra/newview/llappviewer.cpp as + the host module for these hacks. + + First, add some headers: + + + #include "httpcommon.h" + #include "httprequest.h" + #include "httphandler.h" + + + You'll need to derive a class from HttpHandler (not HttpHandle). + This is used to deliver notifications of HTTP completion to your + code. Place it near the top, before LLDeferredTaskList, say: + + + class MyHandler : public LLCore::HttpHandler + { + public: + MyHandler() + : LLCore::HttpHandler() + {} + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */) + { + LL_INFOS("Hack") << "It is happening again." << LL_ENDL; + + delete this; // Last statement + } + }; + + + Add some statics up there as well: + + + // Our request object. Allocate during initialiation. + static LLCore::HttpRequest * my_request(NULL); + + // The policy class for HTTP traffic. + // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!! + static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + + // Priority for HTTP requests. Use 0U. + static LLCore::HttpRequest::priority_t my_priority(0U); + + + In LLAppViewer::init() after mAppCoreHttp.init(), create a request object: + + + my_request = new LLCore::HttpRequest(); + + + In LLAppViewer::mainLoop(), just before entering the while loop, + we'll kick off one HTTP request: + + + // Construct a handler object (we'll use the heap this time): + MyHandler * my_handler = new MyHandler; + + // Issue a GET request to 'http://www.example.com/' kicking off + // all the I/O, retry logic, etc. + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + NULL, + NULL, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LL_WARNS("Hack") << "Failed to launch HTTP request. Try again." + << LL_ENDL; + } + + + Finally, arrange to periodically call update() on the request object + to find out when the request completes. This will be done by + calling the onCompleted() method with status information and + response data from the HTTP operation. Add this to the + LLAppViewer::idle() method after the ping: + + + my_request->update(0); + + + That's it. Build it, run it and watch the log file. You should get + the "It is happening again." message indicating that the HTTP + operation completed in some manner. + + +2. What Does All That Mean + + MyHandler/HttpHandler. This class replaces the Responder-style in + legacy code. One method is currently defined. It is used for all + request completions, successful or failed: + + + void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * /* response */); + + + The onCompleted() method is invoked as a callback during calls to + HttpRequest::update(). All I/O is completed asynchronously in + another thread. But notifications are polled by calling update() + and invoking a handler for completed requests. + + In this example, the invocation also deletes the handler (which is + never referenced by the llcorehttp code again). But other + allocation models are possible including handlers shared by many + requests, stack-based handlers and handlers mixed in with other, + unrelated classes. + + LLCore::HttpRequest(). Instances of this class are used to request + all major functions of the library. Initialization, starting + requests, delivering final notification of completion and various + utility operations are all done via instances. There is one very + important rule for instances: + + Request objects may NOT be shared between threads. + + my_priority. The APIs support the idea of priority ordering of + requests but it hasn't been implemented and the hope is that this + will become useless and removed from the interface. Use 0U except + as noted. + + my_policy. This is an important one. This library attempts to + manage TCP connection usage more rigorously than in the past. This + is done by issuing requests to a queue that has various settable + properties. These establish connection usage for the queue as well + as how queues compete with one another. (This is patterned after + class-based queueing used in various networking stacks.) Several + classes are pre-defined. Deciding when to use an existing class and + when to create a new one will determine what kind of experience + users have. We'll pick up this question in detail below. + + requestGet(). Issues an ordinary HTTP GET request to a given URL + and associating the request with a policy class, a priority and an + response handler. Two additional arguments, not used here, allow + for additional headers on the request and for per-request options. + If successful, the call returns a handle whose value is other than + LLCORE_HTTP_HANDLE_INVALID. The HTTP operation is then performed + asynchronously by another thread without any additional work by the + caller. If the handle returned is invalid, you can get the status + code by calling my_request->getStatus(). + + update(). To get notification that the request has completed, a + call to update() will invoke onCompleted() methods. + + +3. Refinements, Necessary and Otherwise + + MyHandler::onCompleted(). You'll want to do something useful with + your response. Distinguish errors from successes and getting the + response body back in some form. + + Add a new header: + + + #include "bufferarray.h" + + + Replace the existing MyHandler::onCompleted() definition with: + + + virtual void onCompleted(LLCore::HttpHandle /* handle */, + LLCore::HttpResponse * response) + { + LLCore::HttpStatus status = response->getStatus(); + if (status) + { + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + + if (data && data->size()) + { + // There's some data. A BufferArray is a linked list + // of buckets. We'll create a linear buffer and copy + // the data into it. + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + data_blob[data_len] = '\0'; + + // Process the data now in NUL-terminated string. + // Needs more scrubbing but this will do. + LL_INFOS("Hack") << "Received: " << data_blob << LL_ENDL; + + // Free the temporary data + delete [] data_blob; + } + } + else + { + // Something went wrong. Translate the status to + // a meaningful message. + LL_WARNS("Hack") << "HTTP GET failed. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + } + + delete this; // Last statement + } + + + HttpHeaders. The header file "httprequest.h" documents the expected + important headers that will go out with the request. You can add to + these by including an HttpHeaders object with the requestGet() call. + These are typically setup once as part of init rather than + dynamically created. + + Add another header: + + + #include "httpheaders.h" + + + In LLAppViewer::mainLoop(), add this alongside the allocation of + my_handler: + + + // Additional headers for all requests + LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders(); + my_headers->append("Accept", "text/html, application/llsd+xml"); + + + HttpOptions. Options are similar and include a mix of value types. + One interesting per-request option is the trace setting. This + enables various debug-type messages in the log file that show the + progress of the request through the library. It takes values from + zero to three with higher values giving more verbose logging. We'll + use '2' and this will also give us a chance to verify that + HttpHeaders works as expected. + + Same as above, a new header: + + + #include "httpoptions.h" + + + And in LLAppView::mainLoop(): + + + // Special options for requests + LLCore::HttpOptions * my_options = new LLCore::HttpOptions(); + my_options->setTrace(2); + + + Now let's put that all together into a more complete requesting + sequence. Replace the existing invocation of requestGet() with this + slightly more elaborate block: + + + LLCore::HttpHandle handle; + handle = my_request->requestGet(my_policy, + my_priority, + "http://www.example.com/", + my_options, + my_headers, + my_handler); + if (LLCORE_HTTP_HANDLE_INVALID == handle) + { + LLCore::HttpStatus status = my_request->getStatus(); + + LL_WARNS("Hack") << "Failed to request HTTP GET. Status: " + << status.toTerseString() + << ", Reason: " << status.toString() + << LL_ENDL; + + delete my_handler; // No longer needed. + my_handler = NULL; + } + + + Build, run and examine the log file. You'll get some new data with + this run. First, you should get the www.example.com home page + content: + + +---------------------------------------------------------------------------- +2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received: + + + Example Domain + + + + + + + + +
+

Example Domain

+

This domain is established to be used for illustrative examples in documents. You may use this + domain in examples without prior coordination or asking for permission.

+

More information...

+
+ + +---------------------------------------------------------------------------- + + + You'll also get a detailed trace of the HTTP operation itself. Note + the HEADEROUT line which shows the additional header added to the + request. + + +---------------------------------------------------------------------------- +HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle: 086D3148 +HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle: 086D3148, Actives: 0, Readies: 0 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: About to connect() to www.example.com port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Trying 93.184.216.119... +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connected to www.example.com (93.184.216.119) port 80 (#0) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADEROUT, Data: GET / HTTP/1.1 Host: www.example.com Accept-Encoding: deflate, gzip Connection: keep-alive Keep-alive: 300 Accept: text/html, application/llsd+xml +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: HTTP/1.1 200 OK +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Accept-Ranges: bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Cache-Control: max-age=604800 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Type: text/html +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Date: Tue, 17 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Etag: "3012602696" +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Expires: Tue, 24 Sep 2013 20:26:56 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Server: ECS (ewr/1590) +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: X-Cache: HIT +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: x-ec-custom-error: 1 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: Content-Length: 1270 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: HEADERIN, Data: +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: DATAIN, Data: 256 Bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle: 086D3148, Type: TEXT, Data: Connection #0 to host www.example.com left intact +HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle: 086D3148, Status: Http_200 +HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 +---------------------------------------------------------------------------- + + +4. What Does All That Mean, Part 2 + + HttpStatus. The HttpStatus object encodes errors from libcurl, the + library itself and HTTP status values. It does this to avoid + collapsing all non-HTTP error into a single '499' HTTP status and to + make errors distinct. + + To aid programming, the usual bool conversions are available so that + you can write 'if (status)' and the expected thing will happen + whether it's an HTTP, libcurl or library error. There's also + provision to override the treatment of HTTP errors (making 404 a + success, say). + + Share data, don't copy it. The library was started with the goal of + avoiding data copies as much as possible. Instead, read-only data + sharing across threads with atomic reference counts is used for a + number of data types. These currently are: + + * BufferArray. Linked list of data blocks/HTTP bodies. + * HttpHeaders. Shared headers for both requests and responses. + * HttpOptions. Request-only data modifying HTTP behavior. + * HttpResponse. HTTP response description given to onCompleted. + + Using objects of these types requires a few rules: + + * Constructor always gives a reference to caller. + * References are dropped with release() not delete. + * Additional references may be taken out with addRef(). + * Unless otherwise stated, once an object is shared with another + thread it should be treated as read-only. There's no + synchronization on the objects themselves. + + HttpResponse. You'll encounter this mainly in onCompleted() methods. + Commonly-used interfaces on this object: + + * getStatus() to return the final status of the request. + * getBody() to retrieve the response body which may be NULL or + zero-length. + * getContentType() to return the value of the 'Content-Type' + header or an empty string if none was sent. + + This is a reference-counted object so you can call addRef() on it + and hold onto the response for an arbitrary time. But you'll + usually just call a few methods and return from onCompleted() whose + caller will release the object. + + BufferArray. The core data representation for request and response + bodies. In HTTP responses, it's fetched with the getBody() method + and may be NULL or non-NULL with zero length. All successful data + handling should check both conditions before attempting to fetch + data from the object. Data access model uses simple read/write + semantics: + + * append() + * size() + * read() + * write() + + (There is a more sophisticated stream adapter that extends these + methods and will be covered below.) So, one way to retrieve data + from a request is as follows: + + + LLCore::BufferArray * data = response->getBody(); + if (data && data->size()) + { + size_t data_len = data->size(); + char * data_blob = new char [data_len + 1]; + data->read(0, data_blob, data_len); + + + HttpOptions and HttpResponse. Really just simple containers of POD + and std::string pairs. But reference counted and the rule about not + modifying after sharing must be followed. You'll have the urge to + change options dynamically at some point. And you'll try to do that + by just writing new values to the shared object. And in tests + everything will appear to work. Then you ship and people in the + real world start hitting read/write races in strings and then crash. + Don't be lazy. + + HttpHandle. Uniquely identifies a request and can be used to + identify it in an onCompleted() method or cancel it if it's still + queued. But as soon as a request's onCompleted() invocation + returns, the handle becomes invalid and may be reused immediately + for new requests. Don't hold on to handles after notification. + + +5. And Still More Refinements + + (Note: The following refinements are just code fragments. They + don't directly fit into the working example above. But they + demonstrate several idioms you'll want to copy.) + + LLSD, std::streambuf, std::iostream. The read(), write() and + append() methods may be adequate for your purposes. But we use a + lot of LLSD. Its interfaces aren't particularly compatible with + BufferArray. And so two adapters are available to give + stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream, + which implement the std::streambuf and std::iostream interfaces, + respectively. + + A std::streambuf interface isn't something you'll want to use + directly. Instead, you'll use the much friendlier std::iostream + interface found in BufferArrayStream. This adapter gives you all + the '>>' and '<<' operators you'll want as well as working + directly with the LLSD conversion operators. + + Some new headers: + + + #include "bufferstream.h" + #include "llsdserialize.h" + + + And an updated fragment based on onCompleted() above: + + + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + LLSD resp_llsd; + + if (data && data->size()) + { + // There's some data and we expect this to be + // LLSD. Checking of content type and validation + // during parsing would be admirable additions. + // But we'll forgo that now. + LLCore::BufferArrayStream data_stream(data); + LLSDSerialize::fromXML(resp_llsd, data_stream); + } + LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL; + } + else + { + + + Converting an LLSD object into an XML stream stored in a + BufferArray is just the reverse of the above: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + LLSD src_llsd; + src_llsd["foo"] = "bar"; + + LLSDSerialize::toXML(src_llsd, data_stream); + + // 'data' now contains an XML payload and can be sent + // to a web service using the requestPut() or requestPost() + // methods. + ... requestPost(...); + + // And don't forget to release the BufferArray. + data->release(); + data = NULL; + + + LLSD will often go hand-in-hand with BufferArray and data + transport. But you can also do all the streaming I/O you'd expect + of a std::iostream object: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + data_stream << "Hello, World!" << 29.4 << '\n'; + std::string str; + data_stream >> str; + std::cout << str << std::endl; + + data->release(); + // Actual delete will occur when 'data_stream' + // falls out of scope and is destructed. + + + Scoping objects and cleaning up. The examples haven't bothered + with cleanup of objects that are no longer needed. Instead, most + objects have been allocated as if they were global and eternal. + You'll put the objects in more appropriate feature objects and + clean them up as a group. Here's a checklist for actions you may + need to take on cleanup: + + * Call delete on: + o HttpHandlers created on the heap + o HttpRequest objects + * Call release() on: + o BufferArray objects + o HttpHeaders objects + o HttpOptions objects + o HttpResponse objects + + On program exit, as threads wind down, the library continues to + operate safely. Threads don't interact via the library and even + dangling references to HttpHandler objects are safe. If you don't + call HttpRequest::update(), handler references are never + dereferenced. + + You can take a more thorough approach to wind-down. Keep a list + of HttpHandles (not HttpHandlers) of outstanding requests. For + each of these, call HttpRequest::requestCancel() to cancel the + operation. (Don't add the cancel requests' handled to the list.) + This will cancel the outstanding requests that haven't completed. + Canceled or completed, all requests will queue notifications. You + can now cycle calling update() discarding responses. Continue + until all requests notify or a few seconds have passed. + + Global startup and shutdown is handled in the viewer. But you can + learn about it in the code or in the documentation in the headers. + + +6. Choosing a Policy Class + + Now it's time to get rid of the default policy class. Take a look + at the policy class definitions in newview/llappcorehttp.h. + Ideally, you'll find one that's compatible with what you're doing. + Some of the compatibility guidelines are: + + * Destination: Pair of host and port. Mixing requests with + different destinations may cause more connection setup and tear + down. + + * Method: http or https. Usually moot given destination. But + mixing these may also cause connection churn. + + * Transfer size: If you're moving 100MB at a time and you make your + requests to the same policy class as a lot of small, fast event + information that fast traffic is going to get stuck behind you + and someone's experience is going to be miserable. + + * Long poll requests: These are long-lived, must- do operations. + They have a special home called AP_LONG_POLL. + + * Concurrency: High concurrency (5 or more) and large transfer + sizes are incompatible. Another head-of-the-line problem. High + concurrency is tolerated when it's desired to get maximal + throughput. Mesh and texture downloads, for example. + + * Pipelined: If your requests are not idempotent, stay away from + anything marked 'soon' or 'yes'. Hidden retries may be a + problem for you. For now, would also recommend keeping PUT and + POST requests out of classes that may be pipelined. Support for + that is still a bit new. + + If you haven't found a compatible match, you can either create a + new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall + class when all else fails. Inventory query operations might be a + candidate for a new class that supported pipelining on https:. + Same with display name lookups and other bursty-at-login + operations. For other things, AP_DEFAULT will do what it can and + will, in some way or another, tolerate any usage. Whether the + users' experiences are good are for you to determine. + + +7. FAQ + + Q1. What do these policy classes achieve? + + A1. Previously, HTTP-using code in the viewer was written as if + it were some isolated, local operation that didn't have to + consider resources, contention or impact on services and the + larger environment. The result was an application with on the + order of 100 HTTP launch points in its codebase that could create + dozens or even 100's of TCP connections zeroing in on grid + services and disrupting networking equipment, web services and + innocent users. The use of policy classes (modeled on + http://en.wikipedia.org/wiki/Class-based_queueing) is a means to + restrict connection concurrency, good and necessary in itself. In + turn, that reduces demands on an expensive resource (connection + setup and concurrency) which relieves strain on network points. + That enables connection keepalive and opportunites for true + improvements in throughput and user experience. + + Another aspect of the classes is that they give some control over + how competing demands for the network will be apportioned. If + mesh fetches, texture fetches and inventory queries are all being + made at once, the relative weights of their classes' concurrency + limits established that apportioning. We now have an opportunity + to balance the entire viewer system. + + Q2. How's that data sharing with refcounts working for you? + + A2. Meh. It does reduce memory churn and the frequency at which + free blocks must be moved between threads. But it's also a design + for static configuration and dynamic reconfiguration (not + requiring a restart) is favored. Creating new options for every + request isn't too bad, it a sequence of "new, fill, request, + release" for each requested operation. That in contrast to doing + the "new, fill, release" at startup. The bad comes in getting at + the source data. One rule in this work was "no new thread + problems." And one source for those is pulling setting values out + of gSettings in threads. None of that is thread safe though we + tend to get away with it. + + Q3. What needs to be done? + + A3. There's a To-Do list in _httpinternal.h. It has both large + and small projects here if someone would like to try changes. diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 008e4fd95c..f80d7f60f5 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,8 @@ // General library to-do list // // - Implement policy classes. Structure is mostly there just didn't -// need it for the first consumer. +// need it for the first consumer. [Classes are there. More +// advanced features, like borrowing, aren't there yet.] // - Consider Removing 'priority' from the request interface. Its use // in an always active class can lead to starvation of low-priority // requests. Requires coodination of priority values across all @@ -46,6 +47,7 @@ // may not really need it. // - Set/get for global policy and policy classes is clumsy. Rework // it heading in a direction that allows for more dynamic behavior. +// [Mostly fixed] // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the // pedantic. // - Update downloader and other long-duration services are going to @@ -64,6 +66,12 @@ // This won't help in the face of the router problems we've looked // at, however. Detect starvation due to UDP activity and provide // feedback to it. +// - Change the transfer timeout scheme. We're less interested in +// absolute time, in most cases, than in continuous progress. +// - Many of the policy class settings are currently applied to the +// entire class. Some, like connection limits, would be better +// applied to each destination target making multiple targets +// independent. // // Integration to-do list // - LLTextureFetch still needs a major refactor. The use of @@ -73,7 +81,6 @@ // the main source file. // - Expand areas of usage eventually leading to the removal of LLCurl. // Rough order of expansion: -// . Mesh fetch // . Avatar names // . Group membership lists // . Caps access in general @@ -97,8 +104,8 @@ namespace LLCore { // Maxium number of policy classes that can be defined. -// *TODO: Currently limited to the default class, extend. -const int HTTP_POLICY_CLASS_LIMIT = 1; +// *TODO: Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 8; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -129,6 +136,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10; // Retries and time-on-queue are not included and aren't // accounted for. const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; const long HTTP_REQUEST_TIMEOUT_MIN = 0L; const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; @@ -137,6 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; + // Tuning parameters // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 6fe0bfc7d1..fc257fb0c1 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,7 +41,8 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service), mPolicyCount(0), - mMultiHandles(NULL) + mMultiHandles(NULL), + mActiveHandles(NULL) {} @@ -77,6 +78,9 @@ void HttpLibcurl::shutdown() delete [] mMultiHandles; mMultiHandles = NULL; + + delete [] mActiveHandles; + mActiveHandles = NULL; } mPolicyCount = 0; @@ -90,9 +94,12 @@ void HttpLibcurl::start(int policy_count) mPolicyCount = policy_count; mMultiHandles = new CURLM * [mPolicyCount]; + mActiveHandles = new int [mPolicyCount]; + for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { mMultiHandles[policy_class] = curl_multi_init(); + mActiveHandles[policy_class] = 0; } } @@ -110,8 +117,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() // Give libcurl some cycles to do I/O & callbacks for (int policy_class(0); policy_class < mPolicyCount; ++policy_class) { - if (! mMultiHandles[policy_class]) + if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) + { continue; + } int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); @@ -132,12 +141,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - if (completeRequest(mMultiHandles[policy_class], handle, result)) - { - // Request is still active, don't get too sleepy - ret = HttpService::NORMAL; - } - handle = NULL; // No longer valid on return + completeRequest(mMultiHandles[policy_class], handle, result); + handle = NULL; // No longer valid on return + ret = HttpService::NORMAL; // If anything completes, we may have a free slot. + // Turning around quickly reduces connection gap by 7-10mS. } else if (CURLMSG_NONE == msg->msg) { @@ -193,6 +200,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) // On success, make operation active mActiveOps.insert(op); + ++mActiveHandles[op->mReqPolicy]; } @@ -214,6 +222,7 @@ bool HttpLibcurl::cancel(HttpHandle handle) // Drop references mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->release(); return true; @@ -240,7 +249,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op) { LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle: " << static_cast(op) - << ", Status: " << op->mStatus.toHex() + << ", Status: " << op->mStatus.toTerseString() << LL_ENDL; } @@ -275,6 +284,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode // Deactivate request mActiveOps.erase(it); + --mActiveHandles[op->mReqPolicy]; op->mCurlActive = false; // Set final status of request if it hasn't failed by other mechanisms yet @@ -316,7 +326,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode { LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " << static_cast(op) - << ", Status: " << op->mStatus.toHex() + << ", Status: " << op->mStatus.toTerseString() << LL_ENDL; } @@ -336,19 +346,9 @@ int HttpLibcurl::getActiveCount() const int HttpLibcurl::getActiveCountInClass(int policy_class) const { - int count(0); - - for (active_set_t::const_iterator iter(mActiveOps.begin()); - mActiveOps.end() != iter; - ++iter) - { - if ((*iter)->mReqPolicy == policy_class) - { - ++count; - } - } - - return count; + llassert_always(policy_class < mPolicyCount); + + return mActiveHandles ? mActiveHandles[policy_class] : 0; } @@ -359,12 +359,17 @@ int HttpLibcurl::getActiveCountInClass(int policy_class) const struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) { - for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); - - headers->mHeaders.end() != it; - ++it) + const HttpHeaders::const_iterator end(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); end != it; ++it) { - slist = curl_slist_append(slist, (*it).c_str()); + static const char sep[] = ": "; + std::string header; + header.reserve((*it).first.size() + (*it).second.size() + sizeof(sep)); + header.append((*it).first); + header.append(sep); + header.append((*it).second); + + slist = curl_slist_append(slist, header.c_str()); } return slist; } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 611f029ef5..67f98dd4f0 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -71,16 +71,22 @@ public: /// /// @return Indication of how long this method is /// willing to wait for next service call. + /// + /// Threading: called by worker thread. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have /// provided us with a reference count on the op to hold the /// request. (No additional references will be added.) + /// + /// Threading: called by worker thread. void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. + /// + /// Threading: called by init thread. void start(int policy_count); /// Synchronously stop libcurl operations. All active requests @@ -91,9 +97,13 @@ public: /// respective reply queues. /// /// Can be restarted with a start() call. + /// + /// Threading: called by worker thread. void shutdown(); /// Return global and per-class counts of active requests. + /// + /// Threading: called by worker thread. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; @@ -103,6 +113,7 @@ public: /// /// @return True if handle was found and operation canceled. /// + /// Threading: called by worker thread. bool cancel(HttpHandle handle); protected: @@ -121,7 +132,8 @@ protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; + CURLM ** mMultiHandles; // One handle per policy class + int * mActiveHandles; // Active count per policy class }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..5bb0654652 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation() mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U), - mTracing(0) + mTracing(HTTP_TRACE_OFF) { mMetricCreated = totalTime(); } @@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called." << LL_ENDL; } @@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called." << LL_ENDL; } @@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." + LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called." << LL_ENDL; } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 914627fad0..937a61187d 100755 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -72,7 +72,7 @@ class HttpService; class HttpOperation : public LLCoreInt::RefCounted { public: - /// Threading: called by a consumer/application thread. + /// Threading: called by consumer thread. HttpOperation(); protected: @@ -108,7 +108,7 @@ public: /// by the worker thread. This is passible data /// until notification is performed. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// void setReplyPath(HttpReplyQueue * reply_queue, HttpHandler * handler); @@ -141,7 +141,7 @@ public: /// call to HttpRequest::update(). This method does the necessary /// dispatching. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// virtual void visitNotifier(HttpRequest *); diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 207ed8e1e4..926031501e 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and // escape and format it for a tracing line in logging. Absolutely @@ -74,14 +83,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); - -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// Error testing and reporting for libcurl status codes +void check_curl_easy_code(CURLcode code); +void check_curl_easy_code(CURLcode code, int curl_setopt_option); } // end anonymous namespace @@ -104,12 +115,15 @@ HttpOpRequest::HttpOpRequest() mCurlService(NULL), mCurlHeaders(NULL), mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), + mPolicy503Retries(0), mPolicyRetryAt(HttpTime(0)), mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { @@ -153,6 +167,10 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + if (mReplyBody) { mReplyBody->release(); @@ -208,6 +226,11 @@ void HttpOpRequest::stageFromActive(HttpService * service) mCurlHeaders = NULL; } + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + addAsReply(); } @@ -226,6 +249,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); mUserHandler->onCompleted(static_cast(this), response); @@ -335,6 +359,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, { mProcFlags |= PF_SAVE_HEADERS; } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } mPolicyRetryLimit = options->getRetries(); mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -350,6 +378,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, // HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { + CURLcode code; + // Scrub transport and result data for retried op case mCurlActive = false; mCurlHandle = NULL; @@ -379,64 +409,108 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get policy options + // Get global policy options HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); mCurlHandle = LLCurl::createStandardCurlHandle(); + if (! mCurlHandle) + { + // We're in trouble. We'll continue but it won't go well. + LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle. Continuing." + << LL_ENDL; + return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + check_curl_easy_code(code, CURLOPT_IPRESOLVE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); + check_curl_easy_code(code, CURLOPT_NOSIGNAL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); + check_curl_easy_code(code, CURLOPT_NOPROGRESS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); + check_curl_easy_code(code, CURLOPT_URL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); + check_curl_easy_code(code, CURLOPT_PRIVATE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); + // The Linksys WRT54G V5 router has an issue with frequent + // DNS lookups from LAN machines. If they happen too often, + // like for every HTTP request, the router gets annoyed after + // about 700 or so requests and starts issuing TCP RSTs to + // new connections. Reuse the DNS lookups for even a few + // seconds and no RSTs. + code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); + check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); + check_curl_easy_code(code, CURLOPT_AUTOREFERER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); + check_curl_easy_code(code, CURLOPT_MAXREDIRS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); + check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); + check_curl_easy_code(code, CURLOPT_WRITEDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + check_curl_easy_code(code, CURLOPT_READFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); + check_curl_easy_code(code, CURLOPT_READDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); + code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); - const std::string * opt_value(NULL); - long opt_long(0L); - policy.get(HttpRequest::GP_LLPROXY, &opt_long); - if (opt_long) + if (policy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) + else if (policy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); - curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); + check_curl_easy_code(code, CURLOPT_PROXY); + code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + check_curl_easy_code(code, CURLOPT_PROXYTYPE); } - if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) + if (policy.mCAPath.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); + check_curl_easy_code(code, CURLOPT_CAPATH); } - if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + if (policy.mCAFile.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); + check_curl_easy_code(code, CURLOPT_CAINFO); } switch (mReqMethod) { case HOR_GET: - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + check_curl_easy_code(code, CURLOPT_HTTPGET); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { - curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + check_curl_easy_code(code, CURLOPT_POST); + code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + check_curl_easy_code(code, CURLOPT_ENCODING); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); @@ -445,14 +519,17 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) case HOR_PUT: { - curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + check_curl_easy_code(code, CURLOPT_UPLOAD); long data_size(0); if (mReqBody) { data_size = mReqBody->size(); } - curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); - curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + check_curl_easy_code(code, CURLOPT_INFILESIZE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); + check_curl_easy_code(code, CURLOPT_POSTFIELDS); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); @@ -469,9 +546,12 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Tracing if (mTracing >= HTTP_TRACE_CURL_HEADERS) { - curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); - curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); + check_curl_easy_code(code, CURLOPT_VERBOSE); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); + check_curl_easy_code(code, CURLOPT_DEBUGDATA); + code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); + check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION); } // There's a CURLOPT for this now... @@ -499,13 +579,22 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) - { + { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); } - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); - curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; + } + code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); + check_curl_easy_code(code, CURLOPT_TIMEOUT); + code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); + check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT); // Request headers if (mReqHeaders) @@ -513,12 +602,15 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Caller's headers last to override mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); + check_curl_easy_code(code, CURLOPT_HTTPHEADER); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); - curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); + check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); + code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); + check_curl_easy_code(code, CURLOPT_HEADERDATA); } if (status) @@ -559,7 +651,7 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void { // Warn but continue if the read position moves beyond end-of-body // for some reason. - LL_WARNS("HttpCore") << "Request body position beyond body size. Truncating request body." + LL_WARNS("CoreHttp") << "Request body position beyond body size. Truncating request body." << LL_ENDL; } return 0; @@ -576,15 +668,15 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { static const char status_line[] = "HTTP/"; static const size_t status_line_len = sizeof(status_line) - 1; - - static const char con_ran_line[] = "content-range:"; - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + HttpOpRequest * op(static_cast(userdata)); const size_t hdr_size(size * nmemb); const char * hdr_data(static_cast(data)); // Not null terminated - + bool is_header(true); + if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over @@ -592,11 +684,13 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; op->mStatus = HttpStatus(); if (op->mReplyHeaders) { - op->mReplyHeaders->mHeaders.clear(); + op->mReplyHeaders->clear(); } + is_header = false; } // Nothing in here wants a final CR/LF combination. Remove @@ -609,52 +703,109 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi --wanted_hdr_size; } } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } // Save header if caller wants them in the response - if (op->mProcFlags & PF_SAVE_HEADERS) + if (is_header && op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response if (! op->mReplyHeaders) { op->mReplyHeaders = new HttpHeaders; } - op->mReplyHeaders->mHeaders.push_back(std::string(hdr_data, wanted_hdr_size)); + op->mReplyHeaders->append(name, value ? value : ""); } + // From this point, header-specific processors are free to + // modify the header value. + // Detect and parse 'Content-Range' headers - if (op->mProcFlags & PF_SCAN_RANGE_HEADER) + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) { - char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); - - memcpy(hdr_buffer, hdr_data, frag_size); - hdr_buffer[frag_size] = '\0'; - if (frag_size > con_ran_line_len && - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) - { - unsigned int first(0), last(0), length(0); - int status; + unsigned int first(0), last(0), length(0); + int status; - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) - { - // Success, record the fragment position - op->mReplyOffset = first; - op->mReplyLength = last - first + 1; - op->mReplyFullLength = length; - } - else if (-1 == status) - { - // Response is badly formed and shouldn't be accepted - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); - } - else - { - // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, frag_size) - << "'. Ignoring." - << LL_ENDL; - } + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) + { + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } + + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; } } @@ -769,14 +920,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length) { + static const char * const hdr_whitespace(" \t"); + char * tok_state(NULL), * tok(NULL); bool match(true); - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) match = false; - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) - match = 0 == os_strcasecmp("bytes", tok); - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) match = false; if (match) { @@ -815,6 +968,25 @@ int parse_content_range_header(char * buffer, } +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { std::string out; @@ -851,15 +1023,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS - return _strnicmp(s1, s2, n); -#else - return strncasecmp(s1, s2, n); -#endif // LL_WINDOWS -} - int os_strcasecmp(const char *s1, const char *s2) { @@ -881,6 +1044,73 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate) } -} // end anonymous namespace +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} - + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + char * rstr(lstr + strlen(lstr)); + while (lstr < rstr && *--rstr) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + llassert(lstr <= rstr); + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + +void check_curl_easy_code(CURLcode code, int curl_setopt_option) +{ + if (CURLE_OK != code) + { + // Comment from old llcurl code which may no longer apply: + // + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) + << ", curl_easy_setopt option: " << curl_setopt_option + << LL_ENDL; + } +} + + +void check_curl_easy_code(CURLcode code) +{ + if (CURLE_OK != code) + { + // Comment from old llcurl code which may no longer apply: + // + // linux appears to throw a curl error once per session for a bad initialization + // at a pretty random time (when enabling cookies). + LL_WARNS("CoreHttp") << "libcurl error detected: " << curl_easy_strerror(code) + << LL_ENDL; + } +} + +} // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 74a349b0bf..2f628b5aba 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -157,6 +157,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; public: // Request data @@ -174,6 +175,8 @@ public: HttpService * mCurlService; curl_slist * mCurlHeaders; size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; // Result data HttpStatus mStatus; @@ -183,9 +186,11 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; + int mReplyRetryAfter; // Policy data int mPolicyRetries; + int mPolicy503Retries; HttpTime mPolicyRetryAt; int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 8198528a9b..a5363f9170 100755 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ #include "_httpopsetget.h" #include "httpcommon.h" +#include "httprequest.h" #include "_httpservice.h" #include "_httppolicy.h" @@ -43,10 +44,11 @@ namespace LLCore HttpOpSetGet::HttpOpSetGet() : HttpOperation(), - mIsGlobal(false), - mDoSet(false), - mSetting(-1), // Nothing requested - mLongValue(0L) + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), + mReqClass(HttpRequest::INVALID_POLICY_ID), + mReqDoSet(false), + mReqLongValue(0L), + mReplyLongValue(0L) {} @@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet() {} -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) { - mIsGlobal = true; - mSetting = setting; + HttpStatus status; + + mReqOption = opt; + mReqClass = pclass; + return status; } -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) { - mIsGlobal = true; - mDoSet = true; - mSetting = setting; - mStrValue = value; + HttpStatus status; + + if (! HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqLongValue = value; + + return status; +} + + +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) +{ + HttpStatus status; + + if (HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqStrValue = value; + + return status; } void HttpOpSetGet::stageFromRequest(HttpService * service) { - HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); - HttpRequest::EGlobalPolicy setting(static_cast(mSetting)); - - if (mDoSet) + if (mReqDoSet) { - mStatus = pol_opt.set(setting, mStrValue); - } - if (mStatus) - { - const std::string * value(NULL); - if ((mStatus = pol_opt.get(setting, &value))) + if (HttpService::sOptionDesc[mReqOption].mIsLong) { - mStrValue = *value; + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqLongValue, &mReplyLongValue); + } + else + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqStrValue, &mReplyStrValue); + } + } + else + { + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); + } + else + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); } } diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index 6966b9d94e..a1e76dd429 100755 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -46,7 +46,10 @@ namespace LLCore /// configuration settings. /// /// *NOTE: Expect this to change. Don't really like it yet. - +/// +/// *TODO: Can't return values to caller yet. Need to do +/// something better with HttpResponse and visitNotifier(). +/// class HttpOpSetGet : public HttpOperation { public: @@ -61,19 +64,23 @@ private: public: /// Threading: called by application thread - void setupGet(HttpRequest::EGlobalPolicy setting); - void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); virtual void stageFromRequest(HttpService *); public: // Request data - bool mIsGlobal; - bool mDoSet; - int mSetting; - long mLongValue; - std::string mStrValue; + HttpRequest::EPolicyOption mReqOption; + HttpRequest::policy_t mReqClass; + bool mReqDoSet; + long mReqLongValue; + std::string mReqStrValue; + // Reply Data + long mReplyLongValue; + std::string mReplyStrValue; }; // end class HttpOpSetGet diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 014bd37e2e..fd5a93e192 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -41,57 +41,70 @@ namespace LLCore // Per-policy-class data for a running system. -// Collection of queues, parameters, history, metrics, etc. +// Collection of queues, options and other data // for a single policy class. // // Threading: accessed only by worker thread -struct HttpPolicy::State +struct HttpPolicy::ClassState { public: - State() - : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnMin(1), - mNextSample(0), - mErrorCount(0), - mErrorFactor(0) + ClassState() + : mThrottleEnd(0), + mThrottleLeft(0L), + mRequestCount(0L) {} HttpReadyQueue mReadyQueue; HttpRetryQueue mRetryQueue; HttpPolicyClass mOptions; - - long mConnMax; - long mConnAt; - long mConnMin; - - HttpTime mNextSample; - unsigned long mErrorCount; - unsigned long mErrorFactor; + HttpTime mThrottleEnd; + long mThrottleLeft; + long mRequestCount; }; HttpPolicy::HttpPolicy(HttpService * service) - : mActiveClasses(0), - mState(NULL), - mService(service) -{} + : mService(service) +{ + // Create default class + mClasses.push_back(new ClassState()); +} HttpPolicy::~HttpPolicy() { shutdown(); + + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) + { + delete (*it); + } + mClasses.clear(); mService = NULL; } +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return HttpRequest::INVALID_POLICY_ID; + } + mClasses.push_back(new ClassState()); + return policy_class; +} + + void HttpPolicy::shutdown() { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + ClassState & state(*mClasses[policy_class]); + + HttpRetryQueue & retryq(state.mRetryQueue); while (! retryq.empty()) { HttpOpRequest * op(retryq.top()); @@ -101,7 +114,7 @@ void HttpPolicy::shutdown() op->release(); } - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + HttpReadyQueue & readyq(state.mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); @@ -111,28 +124,11 @@ void HttpPolicy::shutdown() op->release(); } } - delete [] mState; - mState = NULL; - mActiveClasses = 0; } -void HttpPolicy::start(const HttpPolicyGlobal & global, - const std::vector & classes) -{ - llassert_always(! mState); - - mGlobalOptions = global; - mActiveClasses = classes.size(); - mState = new State [mActiveClasses]; - for (int i(0); i < mActiveClasses; ++i) - { - mState[i].mOptions = classes[i]; - mState[i].mConnMax = classes[i].mConnectionLimit; - mState[i].mConnAt = mState[i].mConnMax; - mState[i].mConnMin = 2; - } -} +void HttpPolicy::start() +{} void HttpPolicy::addOp(HttpOpRequest * op) @@ -140,7 +136,8 @@ void HttpPolicy::addOp(HttpOpRequest * op) const int policy_class(op->mReqPolicy); op->mPolicyRetries = 0; - mState[policy_class].mReadyQueue.push(op); + op->mPolicy503Retries = 0; + mClasses[policy_class]->mReadyQueue.push(op); } @@ -155,25 +152,39 @@ void HttpPolicy::retryOp(HttpOpRequest * op) 5000000 // ... to every 5.0 S. }; static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); - + static const HttpStatus error_503(503); + const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); - - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + bool external_delta(false); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + external_delta = true; + } op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; - LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) - << " retry " << op->mPolicyRetries - << " scheduled for +" << (delta / HttpTime(1000)) - << " mS. Status: " << op->mStatus.toHex() - << LL_ENDL; - if (op->mTracing > 0) + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) + << " retry " << op->mPolicyRetries + << " scheduled in " << (delta / HttpTime(1000)) + << " mS (" << (external_delta ? "external" : "internal") + << "). Status: " << op->mStatus.toTerseString() + << LL_ENDL; + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " << static_cast(op) + << ", Delta: " << (delta / HttpTime(1000)) + << ", Retries: " << op->mPolicyRetries << LL_ENDL; } - mState[policy_class].mRetryQueue.push(op); + mClasses[policy_class]->mRetryQueue.push(op); } @@ -188,21 +199,43 @@ void HttpPolicy::retryOp(HttpOpRequest * op) // the worker thread may sleep hard otherwise will ask for // normal polling frequency. // +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases. This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +// HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { const HttpTime now(totalTime()); HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); - int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mConnAt - active); // Expect negatives here - + ClassState & state(*mClasses[policy_class]); HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); + + if (retryq.empty() && readyq.empty()) + { + continue; + } + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + + if (throttle_current && state.mThrottleLeft <= 0) + { + // Throttled condition, don't serve this class but don't sleep hard. + result = HttpService::NORMAL; + continue; + } + + int active(transport.getActiveCountInClass(policy_class)); + int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here + if (needed > 0) { // First see if we have any retries... @@ -216,10 +249,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - + + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } - + // Now go on to the new requests... while (needed > 0 && ! readyq.empty()) { @@ -229,10 +279,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } } - + + throttle_on: + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... @@ -246,9 +315,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // We don't scan retry queue because a priority change there // is meaningless. The request will be issued based on retry // intervals not priority value, which is now moot. @@ -276,9 +345,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior bool HttpPolicy::cancel(HttpHandle handle) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // Scan retry queue HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); @@ -337,14 +406,14 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) << " failed after " << op->mPolicyRetries << " retries. Reason: " << op->mStatus.toString() - << " (" << op->mStatus.toHex() << ")" + << " (" << op->mStatus.toTerseString() << ")" << LL_ENDL; } else if (op->mPolicyRetries) { - LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) - << " succeeded on retry " << op->mPolicyRetries << "." - << LL_ENDL; + LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast(op) + << " succeeded on retry " << op->mPolicyRetries << "." + << LL_ENDL; } op->stageFromActive(mService); @@ -352,13 +421,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) return false; // not active } + +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ + llassert_always(pclass >= 0 && pclass < mClasses.size()); + + return mClasses[pclass]->mOptions; +} + int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { - if (policy_class < mActiveClasses) + if (policy_class < mClasses.size()) { - return (mState[policy_class].mReadyQueue.size() - + mState[policy_class].mRetryQueue.size()); + return (mClasses[policy_class]->mReadyQueue.size() + + mClasses[policy_class]->mRetryQueue.size()); } return 0; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 03d92c0b8e..bf1aa74267 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,9 @@ private: void operator=(const HttpPolicy &); // Not defined public: + /// Threading: called by init thread. + HttpRequest::policy_t createPolicyClass(); + /// Cancel all ready and retry requests sending them to /// their notification queues. Release state resources /// making further request handling impossible. @@ -71,9 +74,8 @@ public: /// requests. One-time call invoked before starting /// the worker thread. /// - /// Threading: called by application thread - void start(const HttpPolicyGlobal & global, - const std::vector & classes); + /// Threading: called by init thread + void start(); /// Give the policy layer some cycles to scan the ready /// queue promoting higher-priority requests to active @@ -93,7 +95,7 @@ public: /// and should not be modified by anyone until retrieved /// from queue. /// - /// Threading: called by any thread + /// Threading: called by worker thread void addOp(HttpOpRequest *); /// Similar to addOp, used when a caller wants to retry a @@ -130,30 +132,39 @@ public: /// Threading: called by worker thread bool stageAfterCompletion(HttpOpRequest * op); - // Get pointer to global policy options. Caller is expected - // to do context checks like no setting once running. + /// Get a reference to global policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. /// /// Threading: called by any thread *but* the object may /// only be modified by the worker thread once running. - /// HttpPolicyGlobal & getGlobalOptions() { return mGlobalOptions; } + /// Get a reference to class policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running and + /// read accesses by other threads are exposed to races at + /// that point. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); + /// Get ready counts for a particular policy class /// /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; protected: - struct State; - - int mActiveClasses; - State * mState; - HttpService * mService; // Naked pointer, not refcounted, not owner - HttpPolicyGlobal mGlobalOptions; + struct ClassState; + typedef std::vector class_list_t; + HttpPolicyGlobal mGlobalOptions; + class_list_t mClasses; + HttpService * mService; // Naked pointer, not refcounted, not owner }; // end class HttpPolicy } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..f34a8e9f1e 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,10 +34,10 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(0) + mPipelining(HTTP_PIPELINING_DEFAULT), + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) {} @@ -49,75 +49,75 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mPerHostConnectionLimit = other.mPerHostConnectionLimit; mPipelining = other.mPipelining; + mThrottleRate = other.mThrottleRate; } return *this; } HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) - : mSetMask(other.mSetMask), - mConnectionLimit(other.mConnectionLimit), + : mConnectionLimit(other.mConnectionLimit), mPerHostConnectionLimit(other.mPerHostConnectionLimit), - mPipelining(other.mPipelining) + mPipelining(other.mPipelining), + mThrottleRate(other.mThrottleRate) {} -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::CP_ENABLE_PIPELINING: + case HttpRequest::PO_ENABLE_PIPELINING: mPipelining = llclamp(value, 0L, 1L); break; + case HttpRequest::PO_THROTTLE_RATE: + mThrottleRate = llclamp(value, 0L, 1000000L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: - src = &mPerHostConnectionLimit; + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + *value = mPerHostConnectionLimit; break; - case HttpRequest::CP_ENABLE_PIPELINING: - src = &mPipelining; + case HttpRequest::PO_ENABLE_PIPELINING: + *value = mPipelining; + break; + + case HttpRequest::PO_THROTTLE_RATE: + *value = mThrottleRate; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index d175413cbd..38f1194ded 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyClass { public: @@ -44,14 +56,14 @@ public: HttpPolicyClass(const HttpPolicyClass &); // Not defined public: - HttpStatus set(HttpRequest::EClassPolicy opt, long value); - HttpStatus get(HttpRequest::EClassPolicy opt, long * value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; public: - unsigned long mSetMask; long mConnectionLimit; long mPerHostConnectionLimit; long mPipelining; + long mThrottleRate; }; // end class HttpPolicyClass } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 72f409d3b1..1dc95f3dce 100755 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,8 +34,7 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mTrace(HTTP_TRACE_OFF), mUseLLProxy(0) {} @@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mCAPath = other.mCAPath; mCAFile = other.mCAFile; @@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::GP_TRACE: + case HttpRequest::PO_TRACE: mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); break; - case HttpRequest::GP_LLPROXY: + case HttpRequest::PO_LLPROXY: mUseLLProxy = llclamp(value, 0L, 1L); break; @@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) { switch (opt) { - case HttpRequest::GP_CA_PATH: + case HttpRequest::PO_CA_PATH: mCAPath = value; break; - case HttpRequest::GP_CA_FILE: + case HttpRequest::PO_CA_FILE: mCAFile = value; break; - case HttpRequest::GP_HTTP_PROXY: + case HttpRequest::PO_HTTP_PROXY: mCAFile = value; break; @@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::GP_TRACE: - src = &mTrace; + case HttpRequest::PO_TRACE: + *value = mTrace; break; - case HttpRequest::GP_LLPROXY: - src = &mUseLLProxy; + case HttpRequest::PO_LLPROXY: + *value = mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - const std::string * src(NULL); - switch (opt) { - case HttpRequest::GP_CA_PATH: - src = &mCAPath; + case HttpRequest::PO_CA_PATH: + *value = mCAPath; break; - case HttpRequest::GP_CA_FILE: - src = &mCAFile; + case HttpRequest::PO_CA_FILE: + *value = mCAFile; break; - case HttpRequest::GP_HTTP_PROXY: - src = &mHttpProxy; + case HttpRequest::PO_HTTP_PROXY: + *value = mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index a50d0e4188..67c4ba9481 100755 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyGlobal { public: @@ -46,13 +58,12 @@ private: HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined public: - HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); - HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; public: - unsigned long mSetMask; long mConnectionLimit; std::string mCAPath; std::string mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..c94249dc2d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,6 +43,18 @@ namespace LLCore { +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ // isLong isDynamic isGlobal isClass + { true, true, true, true }, // PO_CONNECTION_LIMIT + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT + { false, false, true, false }, // PO_CA_PATH + { false, false, true, false }, // PO_CA_FILE + { false, true, true, false }, // PO_HTTP_PROXY + { true, true, true, false }, // PO_LLPROXY + { true, true, true, false }, // PO_TRACE + { true, true, false, true }, // PO_ENABLE_PIPELINING + { true, true, false, true } // PO_THROTTLE_RATE +}; HttpService * HttpService::sInstance(NULL); volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); @@ -51,15 +63,9 @@ HttpService::HttpService() mExitRequested(0U), mThread(NULL), mPolicy(NULL), - mTransport(NULL) -{ - // Create the default policy class - HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); - mPolicyClasses.push_back(pol_class); -} + mTransport(NULL), + mLastPolicy(0) +{} HttpService::~HttpService() @@ -149,13 +155,8 @@ void HttpService::term() HttpRequest::policy_t HttpService::createPolicyClass() { - const HttpRequest::policy_t policy_class(mPolicyClasses.size()); - if (policy_class >= HTTP_POLICY_CLASS_LIMIT) - { - return 0; - } - mPolicyClasses.push_back(HttpPolicyClass()); - return policy_class; + mLastPolicy = mPolicy->createPolicyClass(); + return mLastPolicy; } @@ -188,8 +189,8 @@ void HttpService::startThread() } // Push current policy definitions, enable policy & transport components - mPolicy->start(mPolicyGlobal, mPolicyClasses); - mTransport->start(mPolicyClasses.size()); + mPolicy->start(); + mTransport->start(mLastPolicy + 1); mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); sState = RUNNING; @@ -322,7 +323,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { // Setup for subsequent tracing long tracing(HTTP_TRACE_OFF); - mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); if (op->mTracing > HTTP_TRACE_OFF) @@ -345,4 +346,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) } +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long * ret_value) +{ + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + } + + HttpStatus status; + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return status; + } + + // Only global has string values + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long value, long * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + const std::string & value, std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + // Only string values are global at this time + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ffe0349d4d..cf23f3ab61 100755 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,6 +53,7 @@ namespace LLCore class HttpRequestQueue; class HttpPolicy; class HttpLibcurl; +class HttpOpSetGet; /// The HttpService class does the work behind the request queue. It @@ -106,7 +107,7 @@ public: NORMAL, ///< continuous polling of request, ready, active queues REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write }; - + static void init(HttpRequestQueue *); static void term(); @@ -136,7 +137,7 @@ public: /// acquires its weaknesses. static bool isStopped(); - /// Threading: callable by consumer thread *once*. + /// Threading: callable by init thread *once*. void startThread(); /// Threading: callable by worker thread. @@ -180,28 +181,39 @@ public: return *mRequestQueue; } - /// Threading: callable by consumer thread. - HttpPolicyGlobal & getGlobalOptions() - { - return mPolicyGlobal; - } - /// Threading: callable by consumer thread. HttpRequest::policy_t createPolicyClass(); - /// Threading: callable by consumer thread. - HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) - { - llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); - return mPolicyClasses[policy_class]; - } - protected: void threadRun(LLCoreInt::HttpThread * thread); ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + friend class HttpOpSetGet; + friend class HttpRequest; + + // Used internally to describe what operations are allowed + // on each policy option. + struct OptionDescriptor + { + bool mIsLong; + bool mIsDynamic; + bool mIsGlobal; + bool mIsClass; + }; + + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long * ret_value); + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + std::string * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long value, long * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + const std::string & value, std::string * ret_value); protected: + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; static HttpService * sInstance; // === shared data === @@ -210,13 +222,13 @@ protected: LLAtomicU32 mExitRequested; LLCoreInt::HttpThread * mThread; - // === consumer-thread-only data === - HttpPolicyGlobal mPolicyGlobal; - std::vector mPolicyClasses; - // === working-thread-only data === HttpPolicy * mPolicy; // Simple pointer, has ownership HttpLibcurl * mTransport; // Simple pointer, has ownership + + // === main-thread-only data === + HttpRequest::policy_t mLastPolicy; + }; // end class HttpService } // end namespace LLCore diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 40ad4f047d..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,6 +39,7 @@ #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" +#include "httpoptions.h" #include "httpheaders.h" #include "bufferarray.h" #include "_mutex.h" @@ -57,6 +58,7 @@ void usage(std::ostream & out); // Default command line settings static int concurrency_limit(40); +static int highwater(100); static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture"; #if defined(WIN32) @@ -79,11 +81,11 @@ public: WorkingSet(); ~WorkingSet(); - bool reload(LLCore::HttpRequest *); + bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *); virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - void loadTextureUuids(FILE * in); + void loadAssetUuids(FILE * in); public: struct Spec @@ -93,24 +95,27 @@ public: int mLength; }; typedef std::set handle_set_t; - typedef std::vector texture_list_t; + typedef std::vector asset_list_t; public: bool mVerbose; bool mRandomRange; - int mMaxConcurrency; + int mRequestLowWater; + int mRequestHighWater; handle_set_t mHandles; int mRemaining; int mLimit; int mAt; std::string mUrl; - texture_list_t mTextures; + asset_list_t mAssets; int mErrorsApi; int mErrorsHttp; int mErrorsHttp404; int mErrorsHttp416; int mErrorsHttp500; int mErrorsHttp503; + int mRetries; + int mRetriesHttp503; int mSuccesses; long mByteCount; LLCore::HttpHeaders * mHeaders; @@ -158,7 +163,7 @@ int main(int argc, char** argv) bool do_verbose(false); int option(-1); - while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) + while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:"))) { switch (option) { @@ -182,6 +187,21 @@ int main(int argc, char** argv) } break; + case 'H': + { + unsigned long value; + char * end; + + value = strtoul(optarg, &end, 10); + if (value < 1 || value > 100 || *end != '\0') + { + usage(std::cerr); + return 1; + } + highwater = value; + } + break; + case 'R': do_random = true; break; @@ -216,25 +236,32 @@ int main(int argc, char** argv) // Initialization init_curl(); LLCore::HttpRequest::createService(); - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concurrency_limit); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); LLCore::HttpRequest::startThread(); // Get service point LLCore::HttpRequest * hr = new LLCore::HttpRequest(); + // Get request options + LLCore::HttpOptions * opt = new LLCore::HttpOptions(); + opt->setRetries(12); + opt->setUseRetryAfter(true); + // Get a handler/working set WorkingSet ws; // Fill the working set with work ws.mUrl = url_format; - ws.loadTextureUuids(uuids); + ws.loadAssetUuids(uuids); ws.mRandomRange = do_random; ws.mVerbose = do_verbose; - ws.mMaxConcurrency = 100; + ws.mRequestHighWater = highwater; + ws.mRequestLowWater = ws.mRequestHighWater / 2; - if (! ws.mTextures.size()) + if (! ws.mAssets.size()) { std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl; return 1; @@ -246,9 +273,9 @@ int main(int argc, char** argv) // Run it int passes(0); - while (! ws.reload(hr)) + while (! ws.reload(hr, opt)) { - hr->update(5000000); + hr->update(0); ms_sleep(2); if (0 == (++passes % 200)) { @@ -265,6 +292,8 @@ int main(int argc, char** argv) std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << " HTTP 416 errors: " << ws.mErrorsHttp416 << " HTTP 500 errors: " << ws.mErrorsHttp500 << " HTTP 503 errors: " << ws.mErrorsHttp503 << std::endl; + std::cout << "Retries: " << ws.mRetries << " Retries on 503: " << ws.mRetriesHttp503 + << std::endl; std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime) << " uS System CPU: " << (metrics.mEndSTime - metrics.mStartSTime) << " uS Wall Time: " << (metrics.mEndWallTime - metrics.mStartWallTime) @@ -275,6 +304,8 @@ int main(int argc, char** argv) // Clean up hr->requestStopThread(NULL); ms_sleep(1000); + opt->release(); + opt = NULL; delete hr; LLCore::HttpRequest::destroyService(); term_curl(); @@ -300,8 +331,10 @@ void usage(std::ostream & out) " -u printf-style format string for URL generation\n" " Default: " << url_format << "\n" " -R Issue GETs with random Range: headers\n" - " -c Maximum request concurrency. Range: [1..100]\n" + " -c Maximum connection concurrency. Range: [1..100]\n" " Default: " << concurrency_limit << "\n" + " -H HTTP request highwater (requests fed to llcorehttp).\n" + " Range: [1..100] Default: " << highwater << "\n" " -v Verbose mode. Issue some chatter while running\n" " -h print this help\n" "\n" @@ -322,13 +355,15 @@ WorkingSet::WorkingSet() mErrorsHttp416(0), mErrorsHttp500(0), mErrorsHttp503(0), + mRetries(0), + mRetriesHttp503(0), mSuccesses(0), mByteCount(0L) { - mTextures.reserve(30000); + mAssets.reserve(30000); mHeaders = new LLCore::HttpHeaders; - mHeaders->mHeaders.push_back("Accept: image/x-j2c"); + mHeaders->append("Accept", "image/x-j2c"); } @@ -342,29 +377,35 @@ WorkingSet::~WorkingSet() } -bool WorkingSet::reload(LLCore::HttpRequest * hr) +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt) { - int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); + if (mRequestLowWater <= mHandles.size()) + { + // Haven't fallen below low-water level yet. + return false; + } + + int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size()))); for (int i(0); i < to_do; ++i) { char buffer[1024]; #if defined(WIN32) - _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + _snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #else - snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); + snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str()); #endif - int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); - int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); + int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); + int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength); LLCore::HttpHandle handle; if (offset || length) { - handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); + handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this); } else { - handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); + handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this); } if (! handle) { @@ -410,7 +451,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r { // More success LLCore::BufferArray * data(response->getBody()); - mByteCount += data->size(); + mByteCount += data ? data->size() : 0; ++mSuccesses; } else @@ -446,6 +487,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r ++mErrorsApi; } } + unsigned int retry(0U), retry_503(0U); + response->getRetries(&retry, &retry_503); + mRetries += int(retry); + mRetriesHttp503 += int(retry_503); mHandles.erase(it); } @@ -459,21 +504,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r } -void WorkingSet::loadTextureUuids(FILE * in) +void WorkingSet::loadAssetUuids(FILE * in) { char buffer[1024]; while (fgets(buffer, sizeof(buffer), in)) { - WorkingSet::Spec texture; + WorkingSet::Spec asset; char * state(NULL); char * token = strtok_r(buffer, " \t\n,", &state); if (token && 36 == strlen(token)) { // Close enough for this function - texture.mUuid = token; - texture.mOffset = 0; - texture.mLength = 0; + asset.mUuid = token; + asset.mOffset = 0; + asset.mLength = 0; token = strtok_r(buffer, " \t\n,", &state); if (token) { @@ -482,14 +527,14 @@ void WorkingSet::loadTextureUuids(FILE * in) if (token) { int length(atoi(token)); - texture.mOffset = offset; - texture.mLength = length; + asset.mOffset = offset; + asset.mLength = length; } } - mTextures.push_back(texture); + mAssets.push_back(asset); } } - mRemaining = mLimit = mTextures.size(); + mRemaining = mLimit = mAssets.size(); } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 0738760763..c2f15155ac 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -70,7 +70,8 @@ std::string HttpStatus::toString() const "Invalid datatype for argument or option", "Option has not been explicitly set", "Option is not dynamic and must be set early", - "Invalid HTTP status code received from server" + "Invalid HTTP status code received from server", + "Could not allocate required resource" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -177,6 +178,44 @@ std::string HttpStatus::toString() const } +std::string HttpStatus::toTerseString() const +{ + std::ostringstream result; + + unsigned int error_value((unsigned short) mStatus); + + switch (mType) + { + case EXT_CURL_EASY: + result << "Easy_"; + break; + + case EXT_CURL_MULTI: + result << "Multi_"; + break; + + case LLCORE: + result << "Core_"; + break; + + default: + if (isHttpStatus()) + { + result << "Http_"; + error_value = mType; + } + else + { + result << "Unknown_"; + } + break; + } + + result << error_value; + return result.str(); +} + + // Pass true on statuses that might actually be cleared by a // retry. Library failures, calling problems, etc. aren't // going to be fixed by squirting bits all over the Net. @@ -206,6 +245,5 @@ bool HttpStatus::isRetryable() const *this == inv_cont_range); // Short data read disagrees with content-range } - } // end namespace LLCore diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 41fb5164cf..9601f94125 100755 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -29,9 +29,9 @@ /// @package LLCore::HTTP /// -/// This library implements a high-level, Indra-code-free client interface to -/// HTTP services based on actual patterns found in the viewer and simulator. -/// Interfaces are similar to those supplied by the legacy classes +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator. Interfaces are similar to those supplied by the legacy classes /// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that /// allows an application to specify connection behaviors: limits on /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. @@ -52,7 +52,7 @@ /// - "llcorehttp/httprequest.h" /// - "llcorehttp/httpresponse.h" /// -/// The library is still under early development and particular users +/// The library is still under development and particular users /// may need access to internal implementation details that are found /// in the _*.h header files. But this is a crutch to be avoided if at /// all possible and probably indicates some interface work is neeeded. @@ -66,6 +66,8 @@ /// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +/// begin servicing requests. /// /// An HTTP consumer in an application, and an application may have many /// consumers, does a few things: @@ -91,10 +93,12 @@ /// objects. /// - Do completion processing in your onCompletion() method. /// -/// Code fragments: -/// Rather than a poorly-maintained example in comments, look in the -/// example subdirectory which is a minimal yet functional tool to do -/// GET request performance testing. With four calls: +/// Code fragments. +/// +/// Initialization. Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls: /// /// init_curl(); /// LLCore::HttpRequest::createService(); @@ -103,7 +107,85 @@ /// /// the program is basically ready to issue requests. /// - +/// HttpHandler. Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses. This is a non-reference-counted entity +/// which can be used as a base class or a mixin. An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests. Your choice but expect to +/// code something like the following: +/// +/// class AppHandler : public LLCore::HttpHandler +/// { +/// public: +/// virtual void onCompleted(HttpHandle handle, +/// HttpResponse * response) +/// { +/// ... +/// } +/// ... +/// }; +/// ... +/// handler = new handler(...); +/// +/// +/// Issuing requests. Using 'hr' above, +/// +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +/// 0, // Priority, not used yet +/// url, +/// NULL, // options +/// NULL, // additional headers +/// handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler. If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications. Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions. Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +/// hr->update(0); +/// // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library. The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer Any thread that has instanced HttpRequest and is +/// issuing requests. A particular instance can only +/// be used by one consumer thread but a consumer may +/// have many instances available to it. +/// init Special consumer thread, usually the main thread, +/// involved in setting up the library at startup. +/// worker Thread used internally by the library to perform +/// HTTP operations. Consumers will not have to deal +/// with this thread directly but some APIs are reserved +/// to it. +/// any Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely. However, there is a tremendous +/// amount of sharing between threads of read-only data. So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization. During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +/// #include "linden_common.h" // Modifies curl/curl.h interfaces @@ -164,7 +246,10 @@ enum HttpError HE_OPT_NOT_DYNAMIC = 8, // Invalid HTTP status code returned by server - HE_INVALID_HTTP_STATUS = 9 + HE_INVALID_HTTP_STATUS = 9, + + // Couldn't allocate resource, typically libcurl handle + HE_BAD_ALLOC = 10 }; // end enum HttpError @@ -239,9 +324,10 @@ struct HttpStatus return *this; } - static const type_enum_t EXT_CURL_EASY = 0; - static const type_enum_t EXT_CURL_MULTI = 1; - static const type_enum_t LLCORE = 2; + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code + ///< 100-999 directly represent HTTP status codes type_enum_t mType; short mStatus; @@ -297,6 +383,14 @@ struct HttpStatus /// LLCore itself). std::string toString() const; + /// Convert status to a compact string representation + /// of the form: "_". The will be + /// one of: Core, Http, Easy, Multi, Unknown. And + /// will be an unsigned integer. More easily + /// interpreted than the hex representation, it's still + /// compact and easily searched. + std::string toTerseString() const; + /// Returns true if the status value represents an /// HTTP response status (100 - 999). bool isHttpStatus() const diff --git a/indra/llcorehttp/httpheaders.cpp b/indra/llcorehttp/httpheaders.cpp index 2832696271..23ebea361c 100755 --- a/indra/llcorehttp/httpheaders.cpp +++ b/indra/llcorehttp/httpheaders.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,8 @@ #include "httpheaders.h" +#include "llstring.h" + namespace LLCore { @@ -40,5 +42,142 @@ HttpHeaders::~HttpHeaders() {} +void +HttpHeaders::clear() +{ + mHeaders.clear(); +} + + +void HttpHeaders::append(const std::string & name, const std::string & value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::append(const char * name, const char * value) +{ + mHeaders.push_back(value_type(name, value)); +} + + +void HttpHeaders::appendNormal(const char * header, size_t size) +{ + std::string name; + std::string value; + + int col_pos(0); + for (; col_pos < size; ++col_pos) + { + if (':' == header[col_pos]) + break; + } + + if (col_pos < size) + { + // Looks like a header, split it and normalize. + // Name is everything before the colon, may be zero-length. + name.assign(header, col_pos); + + // Value is everything after the colon, may also be zero-length. + const size_t val_len(size - col_pos - 1); + if (val_len) + { + value.assign(header + col_pos + 1, val_len); + } + + // Clean the strings + LLStringUtil::toLower(name); + LLStringUtil::trim(name); + LLStringUtil::trimHead(value); + } + else + { + // Uncertain what this is, we'll pack it as + // a name without a value. Won't clean as we don't + // know what it is... + name.assign(header, size); + } + + mHeaders.push_back(value_type(name, value)); +} + + +// Find from end to simulate a tradition of using single-valued +// std::map for this in the past. +const std::string * HttpHeaders::find(const char * name) const +{ + const_reverse_iterator iend(rend()); + for (const_reverse_iterator iter(rbegin()); iend != iter; ++iter) + { + if ((*iter).first == name) + { + return &(*iter).second; + } + } + return NULL; +} + + +// Standard Iterators +HttpHeaders::iterator HttpHeaders::begin() +{ + return mHeaders.begin(); +} + + +HttpHeaders::const_iterator HttpHeaders::begin() const +{ + return mHeaders.begin(); +} + + +HttpHeaders::iterator HttpHeaders::end() +{ + return mHeaders.end(); +} + + +HttpHeaders::const_iterator HttpHeaders::end() const +{ + return mHeaders.end(); +} + + +// Standard Reverse Iterators +HttpHeaders::reverse_iterator HttpHeaders::rbegin() +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rbegin() const +{ + return mHeaders.rbegin(); +} + + +HttpHeaders::reverse_iterator HttpHeaders::rend() +{ + return mHeaders.rend(); +} + + +HttpHeaders::const_reverse_iterator HttpHeaders::rend() const +{ + return mHeaders.rend(); +} + + +// Return the raw container to the caller. +// +// To be used FOR UNIT TESTS ONLY. +// +HttpHeaders::container_t & HttpHeaders::getContainerTESTONLY() +{ + return mHeaders; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/httpheaders.h b/indra/llcorehttp/httpheaders.h index 3449daa3a1..f70cd898f3 100755 --- a/indra/llcorehttp/httpheaders.h +++ b/indra/llcorehttp/httpheaders.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,13 +43,26 @@ namespace LLCore /// caller has asked that headers be returned (not the default /// option). /// -/// @note -/// This is a minimally-functional placeholder at the moment -/// to fill out the class hierarchy. The final class will be -/// something else, probably more pair-oriented. It's also -/// an area where shared values are desirable so refcounting is -/// already specced and a copy-on-write scheme imagined. -/// Expect changes here. +/// Class is mostly a thin wrapper around a vector of pairs +/// of strings. Methods provided are few and intended to +/// reflect actual use patterns. These include: +/// - Clearing the list +/// - Appending a name/value pair to the vector +/// - Processing a raw byte string into a normalized name/value +/// pair and appending the result. +/// - Simple case-sensitive find-last-by-name search +/// - Forward and reverse iterators over all pairs +/// +/// Container is ordered and multi-valued. Headers are +/// written in the order in which they are appended and +/// are stored in the order in which they're received from +/// the wire. The same header may appear two or more times +/// in any container. Searches using the simple find() +/// interface will find only the last occurrence (somewhat +/// simulates the use of std::map). Fuller searches require +/// the use of an iterator. Headers received from the wire +/// are only returned from the last request when redirections +/// are involved. /// /// Threading: Not intrinsically thread-safe. It *is* expected /// that callers will build these objects and then share them @@ -63,6 +76,16 @@ namespace LLCore class HttpHeaders : public LLCoreInt::RefCounted { +public: + typedef std::pair header_t; + typedef std::vector container_t; + typedef container_t::iterator iterator; + typedef container_t::const_iterator const_iterator; + typedef container_t::reverse_iterator reverse_iterator; + typedef container_t::const_reverse_iterator const_reverse_iterator; + typedef container_t::value_type value_type; + typedef container_t::size_type size_type; + public: /// @post In addition to the instance, caller has a refcount /// to the instance. A call to @see release() will destroy @@ -76,7 +99,78 @@ protected: void operator=(const HttpHeaders &); // Not defined public: - typedef std::vector container_t; + // Empty the list of headers. + void clear(); + + // Append a name/value pair supplied as either std::strings + // or NUL-terminated char * to the header list. No normalization + // is performed on the strings. No conformance test is + // performed (names may contain spaces, colons, etc.). + // + void append(const std::string & name, const std::string & value); + void append(const char * name, const char * value); + + // Extract a name/value pair from a raw byte array using + // the first colon character as a separator. Input string + // does not need to be NUL-terminated. Resulting name/value + // pair is appended to the header list. + // + // Normalization is performed on the name/value pair as + // follows: + // - name is lower-cased according to mostly ASCII rules + // - name is left- and right-trimmed of spaces and tabs + // - value is left-trimmed of spaces and tabs + // - either or both of name and value may be zero-length + // + // By convention, headers read from the wire will be normalized + // in this fashion prior to delivery to any HttpHandler code. + // Headers to be written to the wire are left as appended to + // the list. + void appendNormal(const char * header, size_t size); + + // Perform a simple, case-sensitive search of the header list + // returning a pointer to the value of the last matching header + // in the header list. If none is found, a NULL pointer is returned. + // + // Any pointer returned references objects in the container itself + // and will have the same lifetime as this class. If you want + // the value beyond the lifetime of this instance, make a copy. + // + // @arg name C-style string giving the name of a header + // to search. The comparison is case-sensitive + // though list entries may have been normalized + // to lower-case. + // + // @return NULL if the header wasn't found otherwise + // a pointer to a std::string in the container. + // Pointer is valid only for the lifetime of + // the container or until container is modifed. + // + const std::string * find(const char * name) const; + + // Count of headers currently in the list. + size_type size() const + { + return mHeaders.size(); + } + + // Standard std::vector-based forward iterators. + iterator begin(); + const_iterator begin() const; + iterator end(); + const_iterator end() const; + + // Standard std::vector-based reverse iterators. + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + reverse_iterator rend(); + const_reverse_iterator rend() const; + +public: + // For unit tests only - not a public API + container_t & getContainerTESTONLY(); + +protected: container_t mHeaders; }; // end class HttpHeaders diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ HttpOptions::HttpOptions() mWantHeaders(false), mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), - mRetries(HTTP_RETRY_COUNT_DEFAULT) + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) {} @@ -64,10 +66,21 @@ void HttpOptions::setTimeout(unsigned int timeout) } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + void HttpOptions::setRetries(unsigned int retries) { mRetries = retries; } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..4ab5ff18c4 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -68,36 +68,55 @@ protected: void operator=(const HttpOptions &); // Not defined public: + // Default: false void setWantHeaders(bool wanted); bool getWantHeaders() const { return mWantHeaders; } - + + // Default: 0 void setTrace(int long); int getTrace() const { return mTracing; } + // Default: 30 void setTimeout(unsigned int timeout); unsigned int getTimeout() const { return mTimeout; } + // Default: 0 + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + + // Default: 8 void setRetries(unsigned int retries); unsigned int getRetries() const { return mRetries; } + + // Default: true + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; + unsigned int mTransferTimeout; unsigned int mRetries; - + bool mUseRetryAfter; }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 9b739a8825..7b1888e3eb 100755 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -54,12 +54,8 @@ namespace LLCore // ==================================== -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); - - HttpRequest::HttpRequest() - : //HttpHandler(), - mReplyQueue(NULL), + : mReplyQueue(NULL), mRequestQueue(NULL) { mRequestQueue = HttpRequestQueue::instanceOf(); @@ -90,26 +86,6 @@ HttpRequest::~HttpRequest() // ==================================== -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) -{ - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) - { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); - } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); -} - - -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) -{ - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) - { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); - } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); -} - - HttpRequest::policy_t HttpRequest::createPolicyClass() { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) @@ -120,15 +96,81 @@ HttpRequest::policy_t HttpRequest::createPolicyClass() } -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value) +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); +} + + +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value) +{ + if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + long value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; } @@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode) return handle; } -// ==================================== -// Dynamic Policy Methods -// ==================================== - -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) -{ - HttpStatus status; - HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpSetGet * op = new HttpOpSetGet(); - op->setupSet(GP_HTTP_PROXY, proxy); - op->setReplyPath(mReplyQueue, handler); - if (! (status = mRequestQueue->addOp(op))) // transfers refcount - { - op->release(); - mLastReqStatus = status; - return handle; - } - - mLastReqStatus = status; - handle = static_cast(op); - - return handle; -} - } // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index ab2f302d34..651654844a 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -56,6 +56,9 @@ class BufferArray; /// The class supports the current HTTP request operations: /// /// - requestGetByteRange: GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut: /// /// Policy Classes /// @@ -100,9 +103,26 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. - static const int DEFAULT_POLICY_ID = 0; + static const policy_t DEFAULT_POLICY_ID = 0; + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; - enum EGlobalPolicy + /// Create a new policy class into which requests can be made. + /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// + static policy_t createPolicyClass(); + + enum EPolicyOption { /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying @@ -113,24 +133,40 @@ public: /// a somewhat soft value. There may be an additional five /// connections per policy class depending upon runtime /// behavior. - GP_CONNECTION_LIMIT, + /// + /// Both global and per-class + PO_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + /// + /// Per-class only + PO_PER_HOST_CONNECTION_LIMIT, /// String containing a system-appropriate directory name /// where SSL certs are stored. - GP_CA_PATH, + /// + /// Global only + PO_CA_PATH, /// String giving a full path to a file containing SSL certs. - GP_CA_FILE, + /// + /// Global only + PO_CA_FILE, /// String of host/port to use as simple HTTP proxy. This is /// going to change in the future into something more elaborate /// that may support richer schemes. - GP_HTTP_PROXY, + /// + /// Global only + PO_HTTP_PROXY, /// Long value that if non-zero enables the use of the /// traditional LLProxy code for http/socks5 support. If - /// enabled, has priority over GP_HTTP_PROXY. - GP_LLPROXY, + // enabled, has priority over GP_HTTP_PROXY. + /// + /// Global only + PO_LLPROXY, /// Long value setting the logging trace level for the /// library. Possible values are: @@ -143,50 +179,59 @@ public: /// These values are also used in the trace modes for /// individual requests in HttpOptions. Also be aware that /// tracing tends to impact performance of the viewer. - GP_TRACE - }; - - /// Set a parameter on a global policy option. Calls - /// made after the start of the servicing thread are - /// not honored and return an error status. - /// - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); - - /// Create a new policy class into which requests can be made. - /// - /// @return If positive, the policy_id used to reference - /// the class in other methods. If 0, an error - /// occurred and @see getStatus() may provide more - /// detail on the reason. - static policy_t createPolicyClass(); - - enum EClassPolicy - { - /// Limits the number of connections used for the class. - CP_CONNECTION_LIMIT, - - /// Limits the number of connections used for a single - /// literal address/port pair within the class. - CP_PER_HOST_CONNECTION_LIMIT, + /// + /// Global only + PO_TRACE, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - CP_ENABLE_PIPELINING + /// + /// Per-class only + PO_ENABLE_PIPELINING, + + /// Controls whether client-side throttling should be + /// performed on this policy class. Positive values + /// enable throttling and specify the request rate + /// (requests per second) that should be targetted. + /// A value of zero, the default, specifies no throttling. + /// + /// Per-class only + PO_THROTTLE_RATE, + + PO_LAST // Always at end }; - + + /// Set a policy option for a global or class parameter at + /// startup time (prior to thread start). + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. For globals, specify GLOBAL_POLICY_ID. + /// @param value Desired value of option. + /// @param ret_value Pointer to receive effective set value + /// if successful. May be NULL if effective + /// value not wanted. + /// @return Standard status code. + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value); + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value); + /// Set a parameter on a class-based policy option. Calls /// made after the start of the servicing thread are /// not honored and return an error status. /// - /// @param policy_id ID of class as returned by @see createPolicyClass(). - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. Ignored for globals but recommend + /// using INVALID_POLICY_ID in this case. + /// @param value Desired value of option. + /// @return Handle of dynamic request. Use @see getStatus() if + /// the returned handle is invalid. + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, + HttpHandler * handler); + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, + HttpHandler * handler); /// @} @@ -488,16 +533,6 @@ public: /// @} - /// @name DynamicPolicyMethods - /// - /// @{ - - /// Request that a running transport pick up a new proxy setting. - /// An empty string will indicate no proxy is to be used. - HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); - - /// @} - protected: void generateNotification(HttpOperation * op); @@ -519,7 +554,6 @@ private: /// Must be established before any threading is allowed to /// start. /// - static policy_t sNextPolicyID; /// @} // End Global State diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse() mReplyLength(0U), mReplyFullLength(0U), mBufferArray(NULL), - mHeaders(NULL) + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index f19b521fbf..aee64e2878 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -149,6 +149,25 @@ public: mContentType = con_type; } + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + if (retries) + { + *retries = mRetries; + } + if (retries_503) + { + *retries_503 = m503Retries; + } + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + protected: // Response data here HttpStatus mStatus; @@ -158,6 +177,8 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; }; diff --git a/indra/llcorehttp/tests/test_httpheaders.hpp b/indra/llcorehttp/tests/test_httpheaders.hpp index ce0d19b058..668c36dc66 100755 --- a/indra/llcorehttp/tests/test_httpheaders.hpp +++ b/indra/llcorehttp/tests/test_httpheaders.hpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,6 @@ using namespace LLCoreInt; - namespace tut { @@ -63,7 +62,7 @@ void HttpHeadersTestObjectType::test<1>() HttpHeaders * headers = new HttpHeaders(); ensure("One ref on construction of HttpHeaders", headers->getRefCount() == 1); ensure("Memory being used", mMemTotal < GetMemTotal()); - ensure("Nothing in headers", 0 == headers->mHeaders.size()); + ensure("Nothing in headers", 0 == headers->size()); // release the implicit reference, causing the object to be released headers->release(); @@ -85,14 +84,340 @@ void HttpHeadersTestObjectType::test<2>() { // Append a few strings - std::string str1("Pragma:"); - headers->mHeaders.push_back(str1); - std::string str2("Accept: application/json"); - headers->mHeaders.push_back(str2); + std::string str1n("Pragma"); + std::string str1v(""); + headers->append(str1n, str1v); + std::string str2n("Accept"); + std::string str2v("application/json"); + headers->append(str2n, str2v); - ensure("Headers retained", 2 == headers->mHeaders.size()); - ensure("First is first", headers->mHeaders[0] == str1); - ensure("Second is second", headers->mHeaders[1] == str2); + ensure("Headers retained", 2 == headers->size()); + HttpHeaders::container_t & c(headers->getContainerTESTONLY()); + + ensure("First name is first name", c[0].first == str1n); + ensure("First value is first value", c[0].second == str1v); + ensure("Second name is second name", c[1].first == str2n); + ensure("Second value is second value", c[1].second == str2v); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<3>() +{ + set_test_name("HttpHeaders basic find"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + // Append a few strings + std::string str1n("Uno"); + std::string str1v("1"); + headers->append(str1n, str1v); + std::string str2n("doS"); + std::string str2v("2-2-2-2"); + headers->append(str2n, str2v); + std::string str3n("TRES"); + std::string str3v("trois gymnopedie"); + headers->append(str3n, str3v); + + ensure("Headers retained", 3 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("TRES"); + ensure("Found the last item", result != NULL); + ensure("Last item is a nice", result != NULL && str3v == *result); + + // appends above are raw and find is case sensitive + result = headers->find("TReS"); + ensure("Last item not found due to case", result == NULL); + + result = headers->find("TRE"); + ensure("Last item not found due to prefixing (1)", result == NULL); + + result = headers->find("TRESS"); + ensure("Last item not found due to prefixing (2)", result == NULL); + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +template <> template <> +void HttpHeadersTestObjectType::test<4>() +{ + set_test_name("HttpHeaders normalized header entry"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + ensure("First append worked in some fashion", 1 == headers->size()); + + const std::string * result(NULL); + + // Find a header + result = headers->find("accept"); + ensure("Found 'accept'", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Left-clean on value + static char line2[] = " next : \t\tlinejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + ensure("Second append worked", 2 == headers->size()); + result = headers->find("next"); + ensure("Found 'next'", result != NULL); + ensure("next value is left-clean", result != NULL && + *result == "linejunk \t"); + + // First value unmolested + result = headers->find("accept"); + ensure("Found 'accept' again", result != NULL); + ensure("accept value has face", result != NULL && *result == line1v); + + // Colons in value are okay + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + ensure("Third append worked", 3 == headers->size()); + result = headers->find("fancy-pants"); + ensure("Found 'fancy-pants'", result != NULL); + ensure("fancy-pants value has colons", result != NULL && *result == line3v); + + // Zero-length value + static char line4[] = "all-talk-no-walk:"; + headers->appendNormal(line4, sizeof(line4) - 1); + ensure("Fourth append worked", 4 == headers->size()); + result = headers->find("all-talk-no-walk"); + ensure("Found 'all-talk'", result != NULL); + ensure("al-talk value is zero-length", result != NULL && result->size() == 0); + + // Zero-length name + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + ensure("Fifth append worked", 5 == headers->size()); + result = headers->find(""); + ensure("Found no-name", result != NULL); + ensure("no-name value is something", result != NULL && *result == line5v); + + // Lone colon is still something + headers->clear(); + static char line6[] = " :"; + headers->appendNormal(line6, sizeof(line6) - 1); + ensure("Sixth append worked", 1 == headers->size()); + result = headers->find(""); + ensure("Found 2nd no-name", result != NULL); + ensure("2nd no-name value is nothing", result != NULL && result->size() == 0); + + // Line without colons is taken as-is and unstripped in name + static char line7[] = " \toskdgioasdghaosdghoowg28342908tg8902hg0hwedfhqew890v7qh0wdebv78q0wdevbhq>?M>BNM?NZ? \t"; + headers->appendNormal(line7, sizeof(line7) - 1); + ensure("Seventh append worked", 2 == headers->size()); + result = headers->find(line7); + ensure("Found whatsit line", result != NULL); + ensure("Whatsit line has no value", result != NULL && result->size() == 0); + + // Normaling interface heeds the byte count, doesn't look for NUL-terminator + static char line8[] = "binary:ignorestuffontheendofthis"; + headers->appendNormal(line8, 13); + ensure("Eighth append worked", 3 == headers->size()); + result = headers->find("binary"); + ensure("Found 'binary'", result != NULL); + ensure("binary value was limited to 'ignore'", result != NULL && + *result == "ignore"); + + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Verify forward iterator finds everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<5>() +{ + set_test_name("HttpHeaders iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::iterator end(headers->end()), begin(headers->begin()); + ensure("Empty container has equal begin/end const iterators", end == begin); + HttpHeaders::const_iterator cend(headers->end()), cbegin(headers->begin()); + ensure("Empty container has equal rbegin/rend const iterators", cend == cbegin); + + ensure("Empty container has equal begin/end iterators", headers->end() == headers->begin()); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line1v, + line2v, + line3v, + line4v, + line5v, + line6v + }; + + int i(0); + HttpHeaders::const_iterator cend(headers->end()); + for (HttpHeaders::const_iterator it(headers->begin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::iterator end(headers->end()); + for (HttpHeaders::iterator it(headers->begin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + } + + // release the implicit reference, causing the object to be released + headers->release(); + + // make sure we didn't leak any memory + ensure(mMemTotal == GetMemTotal()); +} + +// Reverse iterators find everything as expected +template <> template <> +void HttpHeadersTestObjectType::test<6>() +{ + set_test_name("HttpHeaders reverse iterator tests"); + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + + // create a new ref counted object with an implicit reference + HttpHeaders * headers = new HttpHeaders(); + + HttpHeaders::reverse_iterator rend(headers->rend()), rbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", rend == rbegin); + HttpHeaders::const_reverse_iterator crend(headers->rend()), crbegin(headers->rbegin()); + ensure("Empty container has equal rbegin/rend const iterators", crend == crbegin); + + { + static char line1[] = " AcCePT : image/yourfacehere"; + static char line1v[] = "image/yourfacehere"; + headers->appendNormal(line1, sizeof(line1) - 1); + + static char line2[] = " next : \t\tlinejunk \t"; + static char line2v[] = "linejunk \t"; + headers->appendNormal(line2, sizeof(line2) - 1); + + static char line3[] = "FancY-PANTs::plop:-neuf-=vleem="; + static char line3v[] = ":plop:-neuf-=vleem="; + headers->appendNormal(line3, sizeof(line3) - 1); + + static char line4[] = "all-talk-no-walk:"; + static char line4v[] = ""; + headers->appendNormal(line4, sizeof(line4) - 1); + + static char line5[] = ":all-talk-no-walk"; + static char line5v[] = "all-talk-no-walk"; + headers->appendNormal(line5, sizeof(line5) - 1); + + static char line6[] = " :"; + static char line6v[] = ""; + headers->appendNormal(line6, sizeof(line6) - 1); + + ensure("All entries accounted for", 6 == headers->size()); + + static char * values[] = { + line6v, + line5v, + line4v, + line3v, + line2v, + line1v + }; + + int i(0); + HttpHeaders::const_reverse_iterator cend(headers->rend()); + for (HttpHeaders::const_reverse_iterator it(headers->rbegin()); + cend != it; + ++it, ++i) + { + std::ostringstream str; + str << "Const Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } + + // Rewind, do non-consts + i = 0; + HttpHeaders::reverse_iterator end(headers->rend()); + for (HttpHeaders::reverse_iterator it(headers->rbegin()); + end != it; + ++it, ++i) + { + std::ostringstream str; + str << "Iterator value # " << i << " was " << values[i]; + ensure(str.str(), (*it).second == values[i]); + } } // release the implicit reference, causing the object to be released diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 900a699887..43f7e36da5 100755 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -69,6 +69,8 @@ void usleep(unsigned long usec); namespace tut { +typedef std::vector > regex_container_t; + struct HttpRequestTestData { // the test objects inherit from this so the member functions and variables @@ -118,11 +120,17 @@ public: for (int i(0); i < mHeadersRequired.size(); ++i) { bool found = false; - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersRequired[i])) + // std::cerr << "Header: " << (*iter).first + // << ": " << (*iter).second << std::endl; + + if (boost::regex_match((*iter).first, + mHeadersRequired[i].first) && + boost::regex_match((*iter).second, + mHeadersRequired[i].second)) { found = true; break; @@ -138,11 +146,14 @@ public: { for (int i(0); i < mHeadersDisallowed.size(); ++i) { - for (HttpHeaders::container_t::const_iterator iter(header->mHeaders.begin()); - header->mHeaders.end() != iter; + for (HttpHeaders::const_iterator iter(header->begin()); + header->end() != iter; ++iter) { - if (boost::regex_match(*iter, mHeadersDisallowed[i])) + if (boost::regex_match((*iter).first, + mHeadersDisallowed[i].first) && + boost::regex_match((*iter).second, + mHeadersDisallowed[i].second)) { std::ostringstream str; str << "Disallowed header # " << i << " not found in response"; @@ -168,8 +179,8 @@ public: std::string mName; HttpHandle mExpectHandle; std::string mCheckContentType; - std::vector mHeadersRequired; - std::vector mHeadersDisallowed; + regex_container_t mHeadersRequired; + regex_container_t mHeadersDisallowed; }; typedef test_group HttpRequestTestGroupType; @@ -1211,7 +1222,7 @@ void HttpRequestTestObjectType::test<12>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1329,7 +1340,7 @@ void HttpRequestTestObjectType::test<13>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1344,7 +1355,9 @@ void HttpRequestTestObjectType::test<13>() // Issue a GET that succeeds mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-LL-Special:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type(boost::regex("X-LL-Special", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base, @@ -1711,18 +1724,54 @@ void HttpRequestTestObjectType::test<16>() // Issue a GET that *can* connect mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1744,23 +1793,60 @@ void HttpRequestTestObjectType::test<16>() // Do a texture-style fetch headers = new HttpHeaders; - headers->mHeaders.push_back("Accept: image/x-j2c"); + headers->append("Accept", "image/x-j2c"); mStatus = HttpStatus(200); handler.mHeadersRequired.clear(); handler.mHeadersDisallowed.clear(); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*image/x-j2c", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\W*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("image/x-j2c", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("\\W*X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -1901,20 +1987,63 @@ void HttpRequestTestObjectType::test<17>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer_encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2061,20 +2190,64 @@ void HttpRequestTestObjectType::test<18>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*chunked.*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2215,27 +2388,73 @@ void HttpRequestTestObjectType::test<19>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("Keep-Alive: 120"); - headers->mHeaders.push_back("Accept-encoding: deflate"); - headers->mHeaders.push_back("Accept: text/plain"); + headers->append("Keep-Alive", "120"); + headers->append("Accept-encoding", "deflate"); + headers->append("Accept", "text/plain"); // Issue a GET with modified headers mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/plain", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*deflate", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-type:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("deflate", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); HttpHandle handle = req->requestGet(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2368,10 +2587,10 @@ void HttpRequestTestObjectType::test<20>() // headers headers = new HttpHeaders(); - headers->mHeaders.push_back("keep-Alive: 120"); - headers->mHeaders.push_back("Accept: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); - headers->mHeaders.push_back("cache-control: no-store"); + headers->append("keep-Alive", "120"); + headers->append("Accept", "text/html"); + headers->append("content-type", "application/llsd+xml"); + headers->append("cache-control", "no-store"); // And a buffer array const char * msg("It was the best of times, it was the worst of times."); @@ -2380,23 +2599,76 @@ void HttpRequestTestObjectType::test<20>() // Issue a default POST mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*text/html", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*120", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("\\s*X-Reflect-cache-control:\\s*no-store", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*application/x-www-form-urlencoded", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-keep-alive:\\s*300", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("120", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex("no-store", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/x-www-form-urlencoded", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("300", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + HttpHandle handle = req->requestPost(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2538,9 +2810,9 @@ void HttpRequestTestObjectType::test<21>() // headers headers = new HttpHeaders; - headers->mHeaders.push_back("content-type: text/plain"); - headers->mHeaders.push_back("content-type: text/html"); - headers->mHeaders.push_back("content-type: application/llsd+xml"); + headers->append("content-type", "text/plain"); + headers->append("content-type", "text/html"); + headers->append("content-type", "application/llsd+xml"); // And a buffer array const char * msg("It was the best of times, it was the worst of times."); @@ -2549,22 +2821,71 @@ void HttpRequestTestObjectType::test<21>() // Issue a default PUT mStatus = HttpStatus(200); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-connection:\\s*keep-alive", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept:\\s*\\*/\\*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-accept-encoding:\\s*((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase)); // close enough - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-keep-alive:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-host:\\s*.*", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-length:\\s*\\d+", boost::regex::icase)); - handler.mHeadersRequired.push_back(boost::regex("X-Reflect-content-type:\\s*application/llsd\\+xml", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-cache-control:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-pragma:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-range:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-referer:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-content-encoding:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-expect:.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("\\s*X-Reflect-transfer-encoding:\\s*.*chunked.*", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/plain", boost::regex::icase)); - handler.mHeadersDisallowed.push_back(boost::regex("X-Reflect-content-type:\\s*text/html", boost::regex::icase)); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-connection", boost::regex::icase), + boost::regex("keep-alive", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept", boost::regex::icase), + boost::regex("\\*/\\*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-accept-encoding", boost::regex::icase), + boost::regex("((gzip|deflate),\\s*)+(gzip|deflate)", boost::regex::icase))); // close enough + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-keep-alive", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-host", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-length", boost::regex::icase), + boost::regex("\\d+", boost::regex::icase))); + handler.mHeadersRequired.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("application/llsd\\+xml", boost::regex::icase))); + + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-cache-control", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-pragma", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-range", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-referer", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-expect", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-transfer-encoding", boost::regex::icase), + boost::regex(".*", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/plain", boost::regex::icase))); + handler.mHeadersDisallowed.push_back( + regex_container_t::value_type( + boost::regex("X-Reflect-content-type", boost::regex::icase), + boost::regex("text/html", boost::regex::icase))); HttpHandle handle = req->requestPut(HttpRequest::DEFAULT_POLICY_ID, 0U, url_base + "reflect/", @@ -2854,6 +3175,142 @@ void HttpRequestTestObjectType::test<22>() } } +template <> template <> +void HttpRequestTestObjectType::test<23>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET 503s with 'Retry-After'"); + + // This tests mainly that the code doesn't fall over if + // various well- and mis-formed Retry-After headers are + // sent along with the response. Direct inspection of + // the parsing result isn't supported. + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/503/"); // path to 503 generators + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Retry once only + opts->setUseRetryAfter(true); // Try to parse the retry-after header + + // Issue a GET that 503s with valid retry-after + mStatus = HttpStatus(503); + int url_limit(6); + for (int i(0); i < url_limit; ++i) + { + std::ostringstream url; + url << url_base << i << "/"; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url.str(), + 0, + 0, + opts, + NULL, + &handler); + + std::ostringstream testtag; + testtag << "Valid handle returned for 503 request #" << i; + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); + } + + + // Run the notification pump. + int count(0); + int limit(LOOP_COUNT_LONG); + while (count++ < limit && mHandlerCalls < url_limit) + { + req->update(0); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == url_limit); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = LOOP_COUNT_LONG; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = LOOP_COUNT_SHORT; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(LOOP_SLEEP_INTERVAL); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index b5538528c5..0b379836c9 100755 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -259,6 +259,65 @@ void HttpStatusTestObjectType::test<7>() ensure(msg == "Unknown error"); } + +template <> template <> +void HttpStatusTestObjectType::test<8>() +{ + set_test_name("HttpStatus toHex() nominal function"); + + HttpStatus status(404); + std::string msg = status.toHex(); + // std::cout << "Result: " << msg << std::endl; + ensure(msg == "01940001"); +} + + +template <> template <> +void HttpStatusTestObjectType::test<9>() +{ + set_test_name("HttpStatus toTerseString() nominal function"); + + HttpStatus status(404); + std::string msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 404", msg == "Http_404"); + + status = HttpStatus(200); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Normal HTTP 200", msg == "Http_200"); + + status = HttpStatus(200, HE_REPLY_ERROR); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Unsuccessful HTTP 200", msg == "Http_200"); // No distinction for error + + status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Easy couldn't connect error", msg == "Easy_7"); + + status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Multi out-of-memory error", msg == "Multi_3"); + + status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Core option not set error", msg == "Core_7"); + + status = HttpStatus(22000, 1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error", msg == "Unknown_1"); + + status = HttpStatus(22000, -1); + msg = status.toTerseString(); + // std::cout << "Result: " << msg << std::endl; + ensure("Undecodable error 65535", msg == "Unknown_65535"); +} + } // end namespace tut #endif // TEST_HTTP_STATUS_H diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3c3af8dc75..04cde651c4 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -69,6 +69,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): "Content-Range: bytes 0-75/2983", "Content-Length: 76" -- '/bug2295/inv_cont_range/0/' Generates HE_INVALID_CONTENT_RANGE error in llcorehttp. + - '/503/' Generate 503 responses with various kinds + of 'retry-after' headers + -- '/503/0/' "Retry-After: 2" + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" + -- '/503/3/' "Retry-After: " + -- '/503/4/' "Retry-After: (*#*(@*(@(")" + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" Some combinations make no sense, there's no effort to protect you from that. @@ -143,22 +152,40 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): if "/sleep/" in self.path: time.sleep(30) - if "fail" in self.path: - status = data.get("status", 500) - # self.responses maps an int status to a (short, long) pair of - # strings. We want the longer string. That's why we pass a string - # pair to get(): the [1] will select the second string, whether it - # came from self.responses or from our default pair. - reason = data.get("reason", - self.responses.get(status, - ("fail requested", - "Your request specified failure status %s " - "without providing a reason" % status))[1]) - debug("fail requested: %s: %r", status, reason) - self.send_error(status, reason) + if "/503/" in self.path: + # Tests for various kinds of 'Retry-After' header parsing + body = None + if "/503/0/" in self.path: + self.send_response(503) + self.send_header("retry-after", "2") + elif "/503/1/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") + elif "/503/2/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") + elif "/503/3/" in self.path: + self.send_response(503) + self.send_header("retry-after", "") + elif "/503/4/" in self.path: + self.send_response(503) + self.send_header("retry-after", "(*#*(@*(@(") + elif "/503/5/" in self.path: + self.send_response(503) + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") + elif "/503/6/" in self.path: + self.send_response(503) + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") + else: + # Unknown request + self.send_response(400) + body = "Unknown /503/ path in server" if "/reflect/" in self.path: self.reflect_headers() + self.send_header("Content-type", "text/plain") self.end_headers() + if body: + self.wfile.write(body) elif "/bug2295/" in self.path: # Test for https://jira.secondlife.com/browse/BUG-2295 # @@ -194,8 +221,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.end_headers() if body: self.wfile.write(body) - else: - # Normal response path + elif "fail" not in self.path: data = data.copy() # we're going to modify # Ensure there's a "reply" key in data, even if there wasn't before data["reply"] = data.get("reply", llsd.LLSD("success")) @@ -210,6 +236,22 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): self.end_headers() if withdata: self.wfile.write(response) + else: # fail requested + status = data.get("status", 500) + # self.responses maps an int status to a (short, long) pair of + # strings. We want the longer string. That's why we pass a string + # pair to get(): the [1] will select the second string, whether it + # came from self.responses or from our default pair. + reason = data.get("reason", + self.responses.get(status, + ("fail requested", + "Your request specified failure status %s " + "without providing a reason" % status))[1]) + debug("fail requested: %s: %r", status, reason) + self.send_error(status, reason) + if "/reflect/" in self.path: + self.reflect_headers() + self.end_headers() def reflect_headers(self): for name in self.headers.keys(): diff --git a/indra/llmath/llvolume.cpp b/indra/llmath/llvolume.cpp index f74c934b21..cc5742ff7a 100755 --- a/indra/llmath/llvolume.cpp +++ b/indra/llmath/llvolume.cpp @@ -6747,7 +6747,7 @@ BOOL LLVolumeFace::createSide(LLVolume* volume, BOOL partial_build) return TRUE; } -//adapted from Lengyel, Eric. “Computing Tangent Space Basis Vectors for an Arbitrary Mesh”. Terathon Software 3D Graphics Library, 2001. http://www.terathon.com/code/tangent.html +//adapted from Lengyel, Eric. "Computing Tangent Space Basis Vectors for an Arbitrary Mesh". Terathon Software 3D Graphics Library, 2001. http://www.terathon.com/code/tangent.html void CalculateTangentArray(U32 vertexCount, const LLVector4a *vertex, const LLVector4a *normal, const LLVector2 *texcoord, U32 triangleCount, const U16* index_array, LLVector4a *tangent) { diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index 081f070866..25e175f2cc 100755 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -6,7 +6,7 @@ * * $LicenseInfo:firstyear=2006&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -294,9 +294,12 @@ LLCurl::Easy* LLCurl::Easy::getEasy() return NULL; } - // set no DNS caching as default for all easy handles. This prevents them adopting a - // multi handles cache if they are added to one. - CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + // Enable a brief cache period for now. This was zero for the longest time + // which caused some routers grief and generated unneeded traffic. For the + // threaded resolver, we're using system resolution libraries and non-zero values + // are preferred. The c-ares resolver is another matter and it might not + // track server changes as well. + CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); check_curl_code(result); result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); check_curl_code(result); diff --git a/indra/llrender/llglslshader.cpp b/indra/llrender/llglslshader.cpp index 35620bb656..1c50a51d02 100755 --- a/indra/llrender/llglslshader.cpp +++ b/indra/llrender/llglslshader.cpp @@ -717,7 +717,14 @@ BOOL LLGLSLShader::mapUniforms(const vector * uniforms) BOOL LLGLSLShader::link(BOOL suppress_errors) { - return LLShaderMgr::instance()->linkProgramObject(mProgramObject, suppress_errors); + BOOL success = LLShaderMgr::instance()->linkProgramObject(mProgramObject, suppress_errors); + + if (!suppress_errors) + { + LLShaderMgr::instance()->dumpObjectLog(mProgramObject, !success, mName); + } + + return success; } void LLGLSLShader::bind() diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp index d230574752..6e04fc82df 100755 --- a/indra/llrender/llshadermgr.cpp +++ b/indra/llrender/llshadermgr.cpp @@ -505,9 +505,25 @@ static std::string get_object_log(GLhandleARB ret) return res; } -void LLShaderMgr::dumpObjectLog(GLhandleARB ret, BOOL warns) +void LLShaderMgr::dumpObjectLog(GLhandleARB ret, BOOL warns, const std::string& filename) { std::string log = get_object_log(ret); + + if (log.length() > 0 || warns) + { + if (!filename.empty()) + { + if (warns) + { + LL_WARNS("ShaderLoading") << "From " << filename << ":" << LL_ENDL; + } + else + { + LL_INFOS("ShaderLoading") << "From " << filename << ":" << LL_ENDL; + } + } + } + if ( log.length() > 0 ) { if (warns) @@ -558,7 +574,7 @@ GLhandleARB LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shade file = LLFile::fopen(fname.str(), "r"); /* Flawfinder: ignore */ if (file) { - LL_INFOS("ShaderLoading") << "Loading file: shaders/class" << gpu_class << "/" << filename << " (Want class " << gpu_class << ")" << LL_ENDL; + LL_DEBUGS("ShaderLoading") << "Loading file: shaders/class" << gpu_class << "/" << filename << " (Want class " << gpu_class << ")" << LL_ENDL; break; // done } } @@ -812,8 +828,8 @@ GLhandleARB LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shade if (error != GL_NO_ERROR || success == GL_FALSE) { //an error occured, print log - LL_WARNS("ShaderLoading") << "GLSL Compilation Error: (" << error << ") in " << filename << LL_ENDL; - dumpObjectLog(ret); + LL_WARNS("ShaderLoading") << "GLSL Compilation Error:" << LL_ENDL; + dumpObjectLog(ret, TRUE, filename); #if LL_WINDOWS std::stringstream ostr; //dump shader source for debugging @@ -938,11 +954,6 @@ BOOL LLShaderMgr::linkProgramObject(GLhandleARB obj, BOOL suppress_errors) suppress_errors = FALSE; } #endif - if (!suppress_errors) - { - dumpObjectLog(obj, !success); - } - return success; } @@ -1146,6 +1157,7 @@ void LLShaderMgr::initAttribsAndUniforms() mReservedUniforms.push_back("env_intensity"); mReservedUniforms.push_back("matrixPalette"); + mReservedUniforms.push_back("translationPalette"); mReservedUniforms.push_back("screenTex"); mReservedUniforms.push_back("screenDepth"); diff --git a/indra/llrender/llshadermgr.h b/indra/llrender/llshadermgr.h index 51c27fc8b6..394b38f832 100755 --- a/indra/llrender/llshadermgr.h +++ b/indra/llrender/llshadermgr.h @@ -176,6 +176,7 @@ public: ENVIRONMENT_INTENSITY, AVATAR_MATRIX, + AVATAR_TRANSLATION, WATER_SCREENTEX, WATER_SCREENDEPTH, @@ -224,7 +225,7 @@ DISPLAY_GAMMA, virtual void initAttribsAndUniforms(void); BOOL attachShaderFeatures(LLGLSLShader * shader); - void dumpObjectLog(GLhandleARB ret, BOOL warns = TRUE); + void dumpObjectLog(GLhandleARB ret, BOOL warns = TRUE, const std::string& filename = ""); BOOL linkProgramObject(GLhandleARB obj, BOOL suppress_errors = FALSE); BOOL validateProgramObject(GLhandleARB obj); GLhandleARB loadShaderFile(const std::string& filename, S32 & shader_level, GLenum type, boost::unordered_map* defines = NULL, S32 texture_index_channels = -1); diff --git a/indra/llui/llbutton.cpp b/indra/llui/llbutton.cpp index 50ac511d18..3cfe5ac57f 100755 --- a/indra/llui/llbutton.cpp +++ b/indra/llui/llbutton.cpp @@ -641,7 +641,7 @@ void LLButton::draw() bool use_glow_effect = FALSE; LLColor4 highlighting_color = LLColor4::white; - LLColor4 glow_color; + LLColor4 glow_color = LLColor4::white; LLRender::eBlendType glow_type = LLRender::BT_ADD_WITH_ALPHA; LLUIImage* imagep = NULL; diff --git a/indra/llui/llcommandmanager.cpp b/indra/llui/llcommandmanager.cpp index 49cfb2255e..625fb8e870 100755 --- a/indra/llui/llcommandmanager.cpp +++ b/indra/llui/llcommandmanager.cpp @@ -50,8 +50,6 @@ const LLCommandId LLCommandId::null = LLCommandId("null command"); LLCommand::Params::Params() : available_in_toybox("available_in_toybox", false) , icon("icon") - , hover_icon_unselected("hover_icon_unselected") - , hover_icon_selected("hover_icon_selected") , label_ref("label_ref") , name("name") , tooltip_ref("tooltip_ref") @@ -73,8 +71,6 @@ LLCommand::LLCommand(const LLCommand::Params& p) : mIdentifier(p.name) , mAvailableInToybox(p.available_in_toybox) , mIcon(p.icon) - , mHoverIconUnselected(p.hover_icon_unselected) - , mHoverIconSelected(p.hover_icon_selected) , mLabelRef(p.label_ref) , mName(p.name) , mTooltipRef(p.tooltip_ref) diff --git a/indra/llui/llcommandmanager.h b/indra/llui/llcommandmanager.h index 9f276f712d..ff5a8a3257 100755 --- a/indra/llui/llcommandmanager.h +++ b/indra/llui/llcommandmanager.h @@ -96,9 +96,6 @@ public: Mandatory name; Mandatory tooltip_ref; - Optional hover_icon_selected; - Optional hover_icon_unselected; - Mandatory execute_function; Optional execute_parameters; @@ -127,8 +124,6 @@ public: const std::string& labelRef() const { return mLabelRef; } const std::string& name() const { return mName; } const std::string& tooltipRef() const { return mTooltipRef; } - const std::string& hoverIconUnselected() const {return mHoverIconUnselected; } - const std::string& hoverIconSelected() const {return mHoverIconSelected; } const std::string& executeFunctionName() const { return mExecuteFunction; } const LLSD& executeParameters() const { return mExecuteParameters; } @@ -155,8 +150,6 @@ private: std::string mLabelRef; std::string mName; std::string mTooltipRef; - std::string mHoverIconUnselected; - std::string mHoverIconSelected; std::string mExecuteFunction; LLSD mExecuteParameters; diff --git a/indra/llui/lltabcontainer.cpp b/indra/llui/lltabcontainer.cpp index 6fd2bb1b36..76ba53ec32 100755 --- a/indra/llui/lltabcontainer.cpp +++ b/indra/llui/lltabcontainer.cpp @@ -193,15 +193,12 @@ LLTabContainer::TabParams::TabParams() : tab_top_image_unselected("tab_top_image_unselected"), tab_top_image_selected("tab_top_image_selected"), tab_top_image_flash("tab_top_image_flash"), - tab_top_image_hovered("tab_top_image_hovered"), tab_bottom_image_unselected("tab_bottom_image_unselected"), tab_bottom_image_selected("tab_bottom_image_selected"), tab_bottom_image_flash("tab_bottom_image_flash"), - tab_bottom_image_hovered("tab_bottom_image_hovered"), tab_left_image_unselected("tab_left_image_unselected"), tab_left_image_selected("tab_left_image_selected"), - tab_left_image_flash("tab_left_image_flash"), - tab_left_image_hovered("tab_left_image_hovered") + tab_left_image_flash("tab_left_image_flash") {} LLTabContainer::Params::Params() @@ -221,8 +218,7 @@ LLTabContainer::Params::Params() open_tabs_on_drag_and_drop("open_tabs_on_drag_and_drop", false), tab_icon_ctrl_pad("tab_icon_ctrl_pad", 0), use_ellipses("use_ellipses"), - font_halign("halign"), - use_highlighting_on_hover("use_highlighting_on_hover",false) + font_halign("halign") {} LLTabContainer::LLTabContainer(const LLTabContainer::Params& p) @@ -258,8 +254,7 @@ LLTabContainer::LLTabContainer(const LLTabContainer::Params& p) mCustomIconCtrlUsed(p.use_custom_icon_ctrl), mOpenTabsOnDragAndDrop(p.open_tabs_on_drag_and_drop), mTabIconCtrlPad(p.tab_icon_ctrl_pad), - mUseTabEllipses(p.use_ellipses), - mUseHighlightingOnHover(p.use_highlighting_on_hover) + mUseTabEllipses(p.use_ellipses) { static LLUICachedControl tabcntr_vert_tab_min_width ("UITabCntrVertTabMinWidth", 0); @@ -908,30 +903,18 @@ void LLTabContainer::update_images(LLTabTuple* tuple, TabParams params, LLTabCon tuple->mButton->setImageUnselected(static_cast(params.tab_top_image_unselected)); tuple->mButton->setImageSelected(static_cast(params.tab_top_image_selected)); tuple->mButton->setImageFlash(static_cast(params.tab_top_image_flash)); - if(mUseHighlightingOnHover) - { - tuple->mButton->setImageHoverUnselected(static_cast(params.tab_top_image_hovered)); - } } else if (pos == LLTabContainer::BOTTOM) { tuple->mButton->setImageUnselected(static_cast(params.tab_bottom_image_unselected)); tuple->mButton->setImageSelected(static_cast(params.tab_bottom_image_selected)); tuple->mButton->setImageFlash(static_cast(params.tab_bottom_image_flash)); - if(mUseHighlightingOnHover) - { - tuple->mButton->setImageHoverUnselected(static_cast(params.tab_bottom_image_hovered)); - } } else if (pos == LLTabContainer::LEFT) { tuple->mButton->setImageUnselected(static_cast(params.tab_left_image_unselected)); tuple->mButton->setImageSelected(static_cast(params.tab_left_image_selected)); tuple->mButton->setImageFlash(static_cast(params.tab_left_image_flash)); - if(mUseHighlightingOnHover) - { - tuple->mButton->setImageHoverUnselected(static_cast(params.tab_left_image_hovered)); - } } } } diff --git a/indra/llui/lltabcontainer.h b/indra/llui/lltabcontainer.h index 7e7d4ac6e6..57862fc626 100755 --- a/indra/llui/lltabcontainer.h +++ b/indra/llui/lltabcontainer.h @@ -62,15 +62,12 @@ public: Optional tab_top_image_unselected, tab_top_image_selected, tab_top_image_flash, - tab_top_image_hovered, tab_bottom_image_unselected, tab_bottom_image_selected, tab_bottom_image_flash, - tab_bottom_image_hovered, tab_left_image_unselected, tab_left_image_selected, - tab_left_image_flash, - tab_left_image_hovered; + tab_left_image_flash; TabParams(); }; @@ -117,11 +114,6 @@ public: */ Optional tab_icon_ctrl_pad; - /** - * This variable is used to found out should we highlight tab button on hover - */ - Optional use_highlighting_on_hover; - Params(); }; @@ -315,7 +307,6 @@ private: bool mOpenTabsOnDragAndDrop; S32 mTabIconCtrlPad; bool mUseTabEllipses; - bool mUseHighlightingOnHover; }; #endif // LL_TABCONTAINER_H diff --git a/indra/llui/lltoolbar.cpp b/indra/llui/lltoolbar.cpp index 6bfe113933..928e82cb8c 100755 --- a/indra/llui/lltoolbar.cpp +++ b/indra/llui/lltoolbar.cpp @@ -928,8 +928,6 @@ LLToolBarButton* LLToolBar::createButton(const LLCommandId& id) button_p.label = LLTrans::getString(commandp->labelRef()); button_p.tool_tip = LLTrans::getString(commandp->tooltipRef()); button_p.image_overlay = LLUI::getUIImage(commandp->icon()); - button_p.image_hover_unselected = LLUI::getUIImage(commandp->hoverIconUnselected()); - button_p.image_hover_selected = LLUI::getUIImage(commandp->hoverIconSelected()); button_p.button_flash_enable = commandp->isFlashingAllowed(); button_p.overwriteFrom(mButtonParams[mButtonType]); LLToolBarButton* button = LLUICtrlFactory::create(button_p); diff --git a/indra/llwindow/llopenglview-objc.mm b/indra/llwindow/llopenglview-objc.mm index 7415c9d8dc..b393a3796d 100644 --- a/indra/llwindow/llopenglview-objc.mm +++ b/indra/llwindow/llopenglview-objc.mm @@ -376,13 +376,6 @@ attributedStringInfo getSegments(NSAttributedString *str) [[self inputContext] handleEvent:theEvent]; } - if ([[theEvent charactersIgnoringModifiers] characterAtIndex:0] == NSCarriageReturnCharacter || - [[theEvent charactersIgnoringModifiers] characterAtIndex:0] == NSEnterCharacter) - { - // callKeyDown won't return the value we expect for enter or return. Handle them as a separate case. - [[self inputContext] handleEvent:theEvent]; - } - // OS X intentionally does not send us key-up information on cmd-key combinations. // This behaviour is not a bug, and only applies to cmd-combinations (no others). // Since SL assumes we receive those, we fake it here. diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index c5e1cde4e6..17e340d136 100755 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -267,6 +267,7 @@ set(viewer_SOURCE_FILES llfloaterregiondebugconsole.cpp llfloaterregioninfo.cpp llfloaterreporter.cpp + llfloaterregionrestarting.cpp llfloaterscriptdebug.cpp llfloaterscriptlimits.cpp llfloatersearch.cpp @@ -855,6 +856,7 @@ set(viewer_HEADER_FILES llfloaterregiondebugconsole.h llfloaterregioninfo.h llfloaterreporter.h + llfloaterregionrestarting.h llfloaterscriptdebug.h llfloaterscriptlimits.h llfloatersearch.h diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 3609cf7707..c1e43e6d45 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.6.13 +3.7.3 diff --git a/indra/newview/app_settings/commands.xml b/indra/newview/app_settings/commands.xml index ce878f156b..60c942094a 100755 --- a/indra/newview/app_settings/commands.xml +++ b/indra/newview/app_settings/commands.xml @@ -3,8 +3,6 @@ Value 16 - + Mesh2MaxConcurrentRequests + + Comment + Number of connections to use for loading meshes. + Persist + 1 + Type + U32 + Value + 8 + MeshMaxConcurrentRequests Comment - Number of threads to use for loading meshes. + Number of connections to use for loading meshes (legacy system). Persist 1 Type U32 Value 32 + + MeshUseHttpRetryAfter + + Comment + If TRUE, use Retry-After response headers when rescheduling a mesh request that fails with an HTTP 503 status. Static. + Persist + 1 + Type + Boolean + Value + 1 + + MeshUseGetMesh1 + + Comment + If TRUE, use the legacy GetMesh capability for mesh download requests. Semi-dynamic (read at region crossings). + Persist + 1 + Type + Boolean + Value + 0 RunMultipleThreads @@ -12643,6 +12675,17 @@ Value 00000000-0000-0000-0000-000000000000 + UISndRestart + + Comment + Sound file for region restarting (uuid for sound asset) + Persist + 1 + Type + String + Value + b92a0f64-7709-8811-40c5-16afd624a45f + UISndSnapshot Comment @@ -14882,17 +14925,6 @@ Value 0 - DisablePrecacheDelayAfterTeleporting - - Comment - Disables the artificial delay in the viewer that precaches some incoming assets - Persist - 0 - Type - Boolean - Value - 0 - FMODExProfilerEnable Comment diff --git a/indra/newview/app_settings/shaders/class1/avatar/objectSkinV.glsl b/indra/newview/app_settings/shaders/class1/avatar/objectSkinV.glsl index 39632d0cef..57129c3bd1 100755 --- a/indra/newview/app_settings/shaders/class1/avatar/objectSkinV.glsl +++ b/indra/newview/app_settings/shaders/class1/avatar/objectSkinV.glsl @@ -22,30 +22,46 @@ * $/LicenseInfo$ */ - - ATTRIBUTE vec4 weight4; -uniform mat4 matrixPalette[32]; +uniform mat3 matrixPalette[52]; +uniform vec3 translationPalette[52]; mat4 getObjectSkinnedTransform() { - int i; + int i; vec4 w = fract(weight4); vec4 index = floor(weight4); - index = min(index, vec4(31.0)); + index = min(index, vec4(51.0)); index = max(index, vec4( 0.0)); float scale = 1.0/(w.x+w.y+w.z+w.w); w *= scale; - mat4 mat = matrixPalette[int(index.x)]*w.x; - mat += matrixPalette[int(index.y)]*w.y; - mat += matrixPalette[int(index.z)]*w.z; - mat += matrixPalette[int(index.w)]*w.w; + int i1 = int(index.x); + int i2 = int(index.y); + int i3 = int(index.z); + int i4 = int(index.w); - return mat; + mat3 mat = matrixPalette[i1]*w.x; + mat += matrixPalette[i2]*w.y; + mat += matrixPalette[i3]*w.z; + mat += matrixPalette[i4]*w.w; + + vec3 trans = translationPalette[i1]*w.x; + trans += translationPalette[i2]*w.y; + trans += translationPalette[i3]*w.z; + trans += translationPalette[i4]*w.w; + + mat4 ret; + + ret[0] = vec4(mat[0], 0); + ret[1] = vec4(mat[1], 0); + ret[2] = vec4(mat[2], 0); + ret[3] = vec4(trans, 1.0); + + return ret; } diff --git a/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl b/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl index e5f7366b70..2b5f001873 100755 --- a/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl +++ b/indra/newview/app_settings/shaders/class1/deferred/alphaF.glsl @@ -534,6 +534,7 @@ void main() #ifdef FOR_IMPOSTOR vec4 color; color.rgb = diff.rgb; + color.a = 1.0; #ifdef USE_VERTEX_COLOR float final_alpha = diff.a * vertex_color.a; diff --git a/indra/newview/character/avatar_lad.xml b/indra/newview/character/avatar_lad.xml index e5b385f4aa..5268498d56 100755 --- a/indra/newview/character/avatar_lad.xml +++ b/indra/newview/character/avatar_lad.xml @@ -3825,7 +3825,11 @@ + pos="0.07 0 -0.02"/> + @@ -3844,7 +3848,16 @@ camera_elevation=".1" camera_distance="1" camera_angle="15"> - + + + + - + + + + - + + + + + + + + + + + + + - + + + + + + + @@ -4162,7 +4245,16 @@ value_max="1.3" camera_elevation=".3" camera_distance=".8"> - + + + + - + + + + @@ -507,6 +506,7 @@ Looking for people to hang out with? Try the [secondlife:///app/worldmap World M right="-10" top_pad="4" left="3" + use_ellipses="true" name="groupcount"> You belong to [COUNT] groups, and can join [REMAINING] more. diff --git a/indra/newview/skins/default/xui/en/widgets/location_input.xml b/indra/newview/skins/default/xui/en/widgets/location_input.xml index 4ea1aa6efb..61ec046649 100755 --- a/indra/newview/skins/default/xui/en/widgets/location_input.xml +++ b/indra/newview/skins/default/xui/en/widgets/location_input.xml @@ -150,7 +150,6 @@ + tab_left_image_selected="SegmentedBtn_Left_Selected_Over"/> + tab_left_image_selected="SegmentedBtn_Left_Selected_Over"/> + tab_left_image_selected="SegmentedBtn_Left_Selected_Over"/> diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py index 9e8623c1f9..96b4c7268c 100755 --- a/indra/newview/viewer_manifest.py +++ b/indra/newview/viewer_manifest.py @@ -38,7 +38,7 @@ viewer_dir = os.path.dirname(__file__) # Put it FIRST because some of our build hosts have an ancient install of # indra.util.llmanifest under their system Python! sys.path.insert(0, os.path.join(viewer_dir, os.pardir, "lib", "python")) -from indra.util.llmanifest import LLManifest, main, proper_windows_path, path_ancestors, CHANNEL_VENDOR_BASE, RELEASE_CHANNEL +from indra.util.llmanifest import LLManifest, main, proper_windows_path, path_ancestors, CHANNEL_VENDOR_BASE, RELEASE_CHANNEL, ManifestError try: from llbase import llsd except ImportError: @@ -136,13 +136,11 @@ class ViewerManifest(LLManifest): settings_install['CmdLineGridChoice']['Value'] = self.grid() print "Set CmdLineGridChoice in settings_install.xml to '%s'" % self.grid() - # did we actually copy anything into settings_install dict? - if settings_install: - # put_in_file(src=) need not be an actual pathname; it - # only needs to be non-empty - self.put_in_file(llsd.format_pretty_xml(settings_install), - "settings_install.xml", - src="environment") + # put_in_file(src=) need not be an actual pathname; it + # only needs to be non-empty + self.put_in_file(llsd.format_pretty_xml(settings_install), + "settings_install.xml", + src="environment") self.end_prefix("app_settings") @@ -820,11 +818,27 @@ class Darwin_i386_Manifest(ViewerManifest): keychain_pwd = open(keychain_pwd_path).read().rstrip() self.run_command('security unlock-keychain -p "%s" "%s/Library/Keychains/viewer.keychain"' % ( keychain_pwd, home_path ) ) - self.run_command('codesign --verbose --force --keychain "%(home_path)s/Library/Keychains/viewer.keychain" --sign %(identity)r %(bundle)r' % { - 'home_path' : home_path, - 'identity': identity, - 'bundle': self.get_dst_prefix() - }) + signed=False + sign_attempts=3 + sign_retry_wait=15 + while (not signed) and (sign_attempts > 0): + try: + sign_attempts-=1; + self.run_command( + 'codesign --verbose --force --keychain "%(home_path)s/Library/Keychains/viewer.keychain" --sign %(identity)r %(bundle)r' % { + 'home_path' : home_path, + 'identity': identity, + 'bundle': self.get_dst_prefix() + }) + signed=True # if no exception was raised, the codesign worked + except ManifestError, err: + if sign_attempts: + print >> sys.stderr, "codesign failed, waiting %d seconds before retrying" % sign_retry_wait + time.sleep(sign_retry_wait) + sign_retry_wait*=2 + else: + print >> sys.stderr, "Maximum codesign attempts exceeded; giving up" + raise imagename="SecondLife_" + '_'.join(self.args['version'])