SL-15572 - retry logic for codeticket.py commands

master
Brad Payne (Vir Linden) 2021-07-13 14:55:11 +01:00 committed by Andrey Lihatskiy
parent 126877bc5b
commit e3c256bdaf
1 changed files with 27 additions and 7 deletions

View File

@ -16,6 +16,29 @@
# * The special style in which python is invoked is intentional to permit
# use of a native python install on windows - which requires paths in DOS form
retry_cmd()
{
max_attempts="$1"; shift
initial_wait="$1"; shift
attempt_num=1
echo "trying" "$@"
until "$@"
do
if ((attempt_num==max_attempts))
then
echo "Last attempt $attempt_num failed"
return 1
else
wait_time=$(($attempt_num*$initial_wait))
echo "Attempt $attempt_num failed. Trying again in $wait_time seconds..."
sleep $wait_time
attempt_num=$(($attempt_num+1))
fi
done
echo "succeeded"
return 0
}
build_dir_Darwin()
{
echo build-darwin-x86_64
@ -463,7 +486,7 @@ then
succeeded=$build_coverity
else
# Upload base package.
python_cmd "$helpers/codeticket.py" addoutput Installer "$package" \
retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput Installer "$package" \
|| fatal "Upload of installer failed"
wait_for_codeticket
@ -473,7 +496,7 @@ then
package=$(installer_$arch "$package_id")
if [ x"$package" != x ]
then
python_cmd "$helpers/codeticket.py" addoutput "Installer $package_id" "$package" \
retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Installer $package_id" "$package" \
|| fatal "Upload of installer $package_id failed"
wait_for_codeticket
else
@ -487,7 +510,7 @@ then
if [ "${RELEASE_CRASH_REPORTING:-}" != "OFF" ]
then
# Upload crash reporter file
python_cmd "$helpers/codeticket.py" addoutput "Symbolfile" "$VIEWER_SYMBOL_FILE" \
retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Symbolfile" "$VIEWER_SYMBOL_FILE" \
|| fatal "Upload of symbolfile failed"
wait_for_codeticket
fi
@ -497,10 +520,7 @@ then
if [ -r "$build_dir/llphysicsextensions_package" ]
then
llphysicsextensions_package=$(cat $build_dir/llphysicsextensions_package)
# This next upload is a frequent failure; see if giving the last one some time helps
# JJ is making changes to Codeticket that we hope will eliminate this failure soon
sleep 300
python_cmd "$helpers/codeticket.py" addoutput "Physics Extensions Package" "$llphysicsextensions_package" --private \
retry_cmd 4 30 python_cmd "$helpers/codeticket.py" addoutput "Physics Extensions Package" "$llphysicsextensions_package" --private \
|| fatal "Upload of physics extensions package failed"
fi
fi