clean commit of teamcity service message related changes of unit test reporting into a clean repo

master
Aaron Terrell (Enus) 2010-08-24 15:15:14 -07:00
parent 98cc236503
commit ee5a71be10
3 changed files with 522 additions and 133 deletions

View File

@ -173,4 +173,13 @@ oz_viewer-review1_coverity.build_CYGWIN_Debug = false
oz_viewer-review1_coverity.build_CYGWIN_RelWithDebInfo = false
oz_viewer-review1_coverity.build_CYGWIN_Release = false
# ========================================
# enus
# ========================================
viewer-tut-teamcity.email = enus@lindenlab.com
viewer-tut-teamcity.build_server = false
viewer-tut-teamcity.build_server_tests = false
# eof

View File

@ -222,7 +222,7 @@ do
fi
else
begin_section "Build$variant"
build "$variant" "$build_dir" >> "$build_log" 2>&1
build "$variant" "$build_dir" 2>&1 | tee -a "$build_log" | grep --line-buffered "^##teamcity"
if `cat "$build_dir/build_ok"`
then
echo so far so good.
@ -251,7 +251,7 @@ then
begin_section "Build$variant"
build_dir=`build_dir_$arch $variant`
build_dir_stubs="$build_dir/win_setup/$variant"
cat "$build_dir/build.log" >> "$build_log"
tee -a $build_log < "$build_dir/build.log" | grep --line-buffered "^##teamcity"
if `cat "$build_dir/build_ok"`
then
echo so far so good.

View File

@ -56,62 +56,75 @@
namespace tut
{
std::string sSourceDir;
test_runner_singleton runner;
}
class LLTestCallback : public tut::callback
{
public:
LLTestCallback(bool verbose_mode, std::ostream *stream) :
mVerboseMode(verbose_mode),
mTotalTests(0),
mPassedTests(0),
mFailedTests(0),
mSkippedTests(0),
mStream(stream)
LLTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
mVerboseMode(verbose_mode),
mTotalTests(0),
mPassedTests(0),
mFailedTests(0),
mSkippedTests(0),
mStream(stream),
suite_name(suitename)
{
}
void run_started()
LLTestCallback()
{
}
virtual void run_started()
{
//std::cout << "run_started" << std::endl;
}
void test_completed(const tut::test_result& tr)
virtual void group_started(const std::string& name) {
std::cout << "group_started name=" << name << std::endl;
}
virtual void group_completed(const std::string& name) {
std::cout << "group_completed name=" << name << std::endl;
}
virtual void test_completed(const tut::test_result& tr)
{
++mTotalTests;
std::ostringstream out;
out << "[" << tr.group << ", " << tr.test << "] ";
switch(tr.result)
{
case tut::test_result::ok:
++mPassedTests;
out << "ok";
break;
case tut::test_result::fail:
++mFailedTests;
out << "fail";
break;
case tut::test_result::ex:
++mFailedTests;
out << "exception";
break;
case tut::test_result::warn:
++mFailedTests;
out << "test destructor throw";
break;
case tut::test_result::term:
++mFailedTests;
out << "abnormal termination";
break;
case tut::test_result::skip:
++mSkippedTests;
out << "skipped known failure";
break;
default:
++mFailedTests;
out << "unknown";
case tut::test_result::ok:
++mPassedTests;
out << "ok";
break;
case tut::test_result::fail:
++mFailedTests;
out << "fail";
break;
case tut::test_result::ex:
++mFailedTests;
out << "exception";
break;
case tut::test_result::warn:
++mFailedTests;
out << "test destructor throw";
break;
case tut::test_result::term:
++mFailedTests;
out << "abnormal termination";
break;
case tut::test_result::skip:
++mSkippedTests;
out << "skipped known failure";
break;
default:
++mFailedTests;
out << "unknown";
}
if(mVerboseMode || (tr.result != tut::test_result::ok))
{
@ -127,8 +140,8 @@ public:
std::cout << out.str() << std::endl;
}
}
void run_completed()
virtual void run_completed()
{
if (mStream)
{
@ -136,11 +149,11 @@ public:
}
run_completed_(std::cout);
}
int getFailedTests() const { return mFailedTests; }
private:
void run_completed_(std::ostream &stream)
virtual int getFailedTests() const { return mFailedTests; }
//private:
virtual void run_completed_(std::ostream &stream)
{
stream << "\tTotal Tests:\t" << mTotalTests << std::endl;
stream << "\tPassed Tests:\t" << mPassedTests;
@ -149,13 +162,13 @@ private:
stream << "\tYAY!! \\o/";
}
stream << std::endl;
if (mSkippedTests > 0)
{
stream << "\tSkipped known failures:\t" << mSkippedTests
<< std::endl;
<< std::endl;
}
if(mFailedTests > 0)
{
stream << "*********************************" << std::endl;
@ -164,8 +177,9 @@ private:
stream << "*********************************" << std::endl;
}
}
protected:
std::string suite_name;
bool mVerboseMode;
int mTotalTests;
int mPassedTests;
@ -174,6 +188,259 @@ protected:
std::ostream *mStream;
};
// copy of LLTestCallback which should become a subclass (commented out below). Delete this LLTCTestCallback one fixed.
// TeamCity specific class which emits service messages
// http://confluence.jetbrains.net/display/TCD3/Build+Script+Interaction+with+TeamCity;#BuildScriptInteractionwithTeamCity-testReporting
class LLTCTestCallback : public tut::callback
{
public:
LLTCTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
mVerboseMode(verbose_mode),
mTotalTests(0),
mPassedTests(0),
mFailedTests(0),
mSkippedTests(0),
mStream(stream),
suite_name(suitename)
{
}
LLTCTestCallback()
{
}
virtual void run_started()
{
//std::cout << "unit test run_started" << std::flush;
}
virtual void group_started(const std::string& name) {
std::cout << "group_started name=" << name << std::endl;
std::cout << "##teamcity[testSuiteStarted name='" << name << "']\n" << std::flush;
}
virtual void group_completed(const std::string& name) {
std::cout << "group_completed name=" << name << std::endl;
std::cout << "##teamcity[testSuiteFinished name='" << name << "']\n" << std::flush;
}
virtual void test_completed(const tut::test_result& tr)
{
++mTotalTests;
std::ostringstream out;
out << "[" << tr.group << ", " << tr.test << "] \n";
switch(tr.result)
{
case tut::test_result::ok:
++mPassedTests;
out << "ok";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::fail:
++mFailedTests;
out << "fail";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::ex:
++mFailedTests;
out << "exception";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::warn:
++mFailedTests;
out << "test destructor throw";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::term:
++mFailedTests;
out << "abnormal termination";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::skip:
++mSkippedTests;
out << "skipped known failure";
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testIgnored name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
default:
++mFailedTests;
out << "unknown";
}
if(mVerboseMode || (tr.result != tut::test_result::ok))
{
if(!tr.message.empty())
{
out << ": '" << tr.message << "'";
}
if (mStream)
{
*mStream << out.str() << std::endl;
}
std::cout << out.str() << std::endl;
}
}
virtual void run_completed()
{
if (mStream)
{
run_completed_(*mStream);
}
run_completed_(std::cout);
}
virtual int getFailedTests() const { return mFailedTests; }
//private:
virtual void run_completed_(std::ostream &stream)
{
stream << "\tTotal Tests:\t" << mTotalTests << std::endl;
stream << "\tPassed Tests:\t" << mPassedTests;
if (mPassedTests == mTotalTests)
{
stream << "\tYAY!! \\o/";
}
stream << std::endl;
if (mSkippedTests > 0)
{
stream << "\tSkipped known failures:\t" << mSkippedTests
<< std::endl;
}
if(mFailedTests > 0)
{
stream << "*********************************" << std::endl;
stream << "Failed Tests:\t" << mFailedTests << std::endl;
stream << "Please report or fix the problem." << std::endl;
stream << "*********************************" << std::endl;
}
}
protected:
std::string suite_name;
bool mVerboseMode;
int mTotalTests;
int mPassedTests;
int mFailedTests;
int mSkippedTests;
std::ostream *mStream;
};
/*
// commented out subclass which should be fixed to eliminate the duplicated LLTestCallback and LLTCTestCallaback classes
// when this is fixed, the duplicated code in the if(getenv("TEAMCITY_PROJECT_NAME") statements below
//
// currectly producing errors like thr following:
// {path}viewer-tut-teamcity2/indra/build-darwin-i386/sharedlibs/RelWithDebInfo/RelWithDebInfo/PROJECT_llmessage_TEST_llmime
// --touch={path}viewer-tut-teamcity2/indra/build-darwin-i386/llmessage/PROJECT_llmessage_TEST_llmime_ok.txt
// --{path}viewer-tut-teamcity2/indra/llmessage
//
// run_started
// group_started name=mime_index
// ##teamcity[testSuiteStarted name='mime_index']
// Segmentation fault
// TeamCity specific class which emits service messages
// http://confluence.jetbrains.net/display/TCD3/Build+Script+Interaction+with+TeamCity;#BuildScriptInteractionwithTeamCity-testReporting
class LLTCTestCallback : public LLTestCallback
{
public:
LLTCTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
mVerboseMode(verbose_mode),
mTotalTests(0),
mPassedTests(0),
mFailedTests(0),
mSkippedTests(0),
mStream(stream),
suite_name(suitename)
{
}
LLTCTestCallback()
{
}
virtual void group_started(const std::string& name) {
LLTestCallback::group_started(name);
std::cout << "##teamcity[testSuiteStarted name='" << name << "']\n" << std::flush;
}
virtual void group_completed(const std::string& name) {
LLTestCallback::group_completed(name);
std::cout << "##teamcity[testSuiteFinished name='" << name << "']\n" << std::flush;
}
virtual void test_completed(const tut::test_result& tr)
{
LLTestCallback::test_completed(tr);
switch(tr.result)
{
case tut::test_result::ok:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::fail:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::ex:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::warn:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::term:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
case tut::test_result::skip:
std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testIgnored name='" << tr.group << "." << tr.test << "']\n" << std::flush;
std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
break;
default:
break;
}
}
protected:
std::string suite_name;
bool mVerboseMode;
int mTotalTests;
int mPassedTests;
int mFailedTests;
int mSkippedTests;
std::ostream *mStream;
};
}
*/
static const apr_getopt_option_t TEST_CL_OPTIONS[] =
{
{"help", 'h', 0, "Print the help message."},
@ -185,27 +452,28 @@ static const apr_getopt_option_t TEST_CL_OPTIONS[] =
{"touch", 't', 1, "Touch the given file if all tests succeed"},
{"wait", 'w', 0, "Wait for input before exit."},
{"debug", 'd', 0, "Emit full debug logs."},
{"suitename", 'x', 1, "Run tests using this suitename"},
{0, 0, 0, 0}
};
void stream_usage(std::ostream& s, const char* app)
{
s << "Usage: " << app << " [OPTIONS]" << std::endl
<< std::endl;
<< std::endl;
s << "This application runs the unit tests." << std::endl << std::endl;
s << "Options: " << std::endl;
const apr_getopt_option_t* option = &TEST_CL_OPTIONS[0];
while(option->name)
{
s << " ";
s << " -" << (char)option->optch << ", --" << option->name
<< std::endl;
<< std::endl;
s << "\t" << option->description << std::endl << std::endl;
++option;
}
s << "Examples:" << std::endl;
s << " " << app << " --verbose" << std::endl;
s << "\tRun all the tests and report all results." << std::endl;
@ -242,13 +510,13 @@ int main(int argc, char **argv)
LLError::initForApplication(".");
LLError::setFatalFunction(wouldHaveCrashed);
LLError::setDefaultLevel(LLError::LEVEL_ERROR);
//< *TODO: should come from error config file. Note that we
// have a command line option that sets this to debug.
//< *TODO: should come from error config file. Note that we
// have a command line option that sets this to debug.
#ifdef CTYPE_WORKAROUND
ctype_workaround();
#endif
apr_initialize();
apr_pool_t* pool = NULL;
if(APR_SUCCESS != apr_pool_create(&pool, NULL))
@ -262,12 +530,13 @@ int main(int argc, char **argv)
std::cerr << "Unable to pool" << std::endl;
return 1;
}
// values used for controlling application
bool verbose_mode = false;
bool wait_at_exit = false;
std::string test_group;
std::string suite_name;
// values use for options parsing
apr_status_t apr_err;
const char* opt_arg = NULL;
@ -283,88 +552,199 @@ int main(int argc, char **argv)
{
char buf[255]; /* Flawfinder: ignore */
std::cerr << "Error parsing options: "
<< apr_strerror(apr_err, buf, 255) << std::endl;
<< apr_strerror(apr_err, buf, 255) << std::endl;
return 1;
}
switch (opt_id)
{
case 'g':
test_group.assign(opt_arg);
break;
case 'h':
stream_usage(std::cout, argv[0]);
return 0;
break;
case 'l':
stream_groups(std::cout, argv[0]);
return 0;
case 'v':
verbose_mode = true;
break;
case 'o':
output = new std::ofstream;
output->open(opt_arg);
break;
case 's': // --sourcedir
tut::sSourceDir = opt_arg;
// For convenience, so you can use tut::sSourceDir + "myfile"
tut::sSourceDir += '/';
break;
case 't':
touch = opt_arg;
break;
case 'w':
wait_at_exit = true;
break;
case 'd':
// *TODO: should come from error config file. We set it to
// ERROR by default, so this allows full debug levels.
LLError::setDefaultLevel(LLError::LEVEL_DEBUG);
break;
default:
stream_usage(std::cerr, argv[0]);
return 1;
break;
case 'g':
test_group.assign(opt_arg);
break;
case 'h':
stream_usage(std::cout, argv[0]);
return 0;
break;
case 'l':
stream_groups(std::cout, argv[0]);
return 0;
case 'v':
verbose_mode = true;
break;
case 'o':
output = new std::ofstream;
output->open(opt_arg);
break;
case 's': // --sourcedir
tut::sSourceDir = opt_arg;
// For convenience, so you can use tut::sSourceDir + "myfile"
tut::sSourceDir += '/';
break;
case 't':
touch = opt_arg;
break;
case 'w':
wait_at_exit = true;
break;
case 'd':
// *TODO: should come from error config file. We set it to
// ERROR by default, so this allows full debug levels.
LLError::setDefaultLevel(LLError::LEVEL_DEBUG);
break;
case 'x':
suite_name.assign(opt_arg);
break;
default:
stream_usage(std::cerr, argv[0]);
return 1;
break;
}
}
// run the tests
LLTestCallback callback(verbose_mode, output);
tut::runner.get().set_callback(&callback);
if(test_group.empty())
/*
// commented out test tunner logic which should be fixed when eliminate the duplicated LLTestCallback and LLTCTestCallaback classes
// become proper class:subclass
// if the Segmentation fault issue is resolved, all code in the block comments can be uncommented, and all code below can be removed.
LLTestCallback* mycallback;
if (getenv("TEAMCITY_PROJECT_NAME"))
{
mycallback = new LLTCTestCallback(verbose_mode, output, suite_name);
}
else
{
mycallback = new LLTestCallback(verbose_mode, output, suite_name);
}
tut::runner.get().set_callback(mycallback);
if(test_group.empty())
{
tut::runner.get().run_tests();
}
else
{
tut::runner.get().run_tests(test_group);
}
bool success = (mycallback->getFailedTests() == 0);
if (wait_at_exit)
{
std::cerr << "Press return to exit..." << std::endl;
std::cin.get();
}
if (output)
{
output->close();
delete output;
}
if (touch && success)
{
std::ofstream s;
s.open(touch);
s << "ok" << std::endl;
s.close();
}
apr_terminate();
int retval = (success ? 0 : 1);
return retval;
*/
// run the tests
if (getenv("TEAMCITY_PROJECT_NAME"))
{
tut::runner.get().run_tests();
LLTCTestCallback* mycallback;
mycallback = new LLTCTestCallback(verbose_mode, output, suite_name);
tut::runner.get().set_callback(mycallback);
if(test_group.empty())
{
tut::runner.get().run_tests();
}
else
{
tut::runner.get().run_tests(test_group);
}
bool success = (mycallback->getFailedTests() == 0);
if (wait_at_exit)
{
std::cerr << "Press return to exit..." << std::endl;
std::cin.get();
}
if (output)
{
output->close();
delete output;
}
if (touch && success)
{
std::ofstream s;
s.open(touch);
s << "ok" << std::endl;
s.close();
}
apr_terminate();
int retval = (success ? 0 : 1);
return retval;
}
// NOT if (getenv("TEAMCITY_PROJECT_NAME"))
else
{
tut::runner.get().run_tests(test_group);
LLTestCallback* mycallback;
mycallback = new LLTestCallback(verbose_mode, output, suite_name);
tut::runner.get().set_callback(mycallback);
if(test_group.empty())
{
tut::runner.get().run_tests();
}
else
{
tut::runner.get().run_tests(test_group);
}
bool success = (mycallback->getFailedTests() == 0);
if (wait_at_exit)
{
std::cerr << "Press return to exit..." << std::endl;
std::cin.get();
}
if (output)
{
output->close();
delete output;
}
if (touch && success)
{
std::ofstream s;
s.open(touch);
s << "ok" << std::endl;
s.close();
}
apr_terminate();
int retval = (success ? 0 : 1);
return retval;
}
bool success = (callback.getFailedTests() == 0);
if (wait_at_exit)
{
std::cerr << "Press return to exit..." << std::endl;
std::cin.get();
}
if (output)
{
output->close();
delete output;
}
if (touch && success)
{
std::ofstream s;
s.open(touch);
s << "ok" << std::endl;
s.close();
}
apr_terminate();
int retval = (success ? 0 : 1);
return retval;
}