A Discrete-Event Network Simulator
API
Loading...
Searching...
No Matches
test.cc
Go to the documentation of this file.
1/*
2 * Copyright (c) 2009 University of Washington
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include "test.h"
8
9#include "abort.h"
10#include "assert.h"
11#include "config.h"
12#include "des-metrics.h"
13#include "log.h"
14#include "rng-seed-manager.h"
15#include "singleton.h"
16#include "system-path.h"
17
18#include <cmath>
19#include <cstring>
20#include <list>
21#include <map>
22#include <vector>
23
24/**
25 * \file
26 * \ingroup testing
27 * \brief ns3::TestCase, ns3::TestSuite, ns3::TestRunner implementations,
28 */
29
30namespace ns3
31{
32
34
35bool
36TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
37{
38 NS_LOG_FUNCTION(x1 << x2 << epsilon);
39 int exponent;
40 double delta;
41 double difference;
42
43 //
44 // Find exponent of largest absolute value
45 //
46 {
47 double max = (std::fabs(x1) > std::fabs(x2)) ? x1 : x2;
48 std::frexp(max, &exponent);
49 }
50
51 //
52 // Form a neighborhood of size 2 * delta
53 //
54 delta = std::ldexp(epsilon, exponent);
55 difference = x1 - x2;
56
57 return difference <= delta && difference >= -delta;
58}
59
60/**
61 * \ingroup testingimpl
62 * Container for details of a test failure.
63 */
65{
66 /**
67 * Constructor.
68 *
69 * \param [in] _cond The name of the condition being tested.
70 * \param [in] _actual The actual value returned by the test.
71 * \param [in] _limit The expected value.
72 * \param [in] _message The associated message.
73 * \param [in] _file The source file.
74 * \param [in] _line The source line.
75 */
76 TestCaseFailure(std::string _cond,
77 std::string _actual,
78 std::string _limit,
79 std::string _message,
80 std::string _file,
81 int32_t _line);
82 std::string cond; /**< The name of the condition being tested. */
83 std::string actual; /**< The actual value returned by the test. */
84 std::string limit; /**< The expected value. */
85 std::string message; /**< The associated message. */
86 std::string file; /**< The source file. */
87 int32_t line; /**< The source line. */
88};
89
90/**
91 * Output streamer for TestCaseFailure.
92 *
93 * \param [in,out] os The output stream.
94 * \param [in] failure The TestCaseFailure to print.
95 * \returns The stream.
96 */
97std::ostream&
98operator<<(std::ostream& os, const TestCaseFailure& failure)
99{
100 os << " test=\"" << failure.cond << "\" actual=\"" << failure.actual << "\" limit=\""
101 << failure.limit << "\" in=\"" << failure.file << ":" << failure.line << "\" "
102 << failure.message;
103
104 return os;
105}
106
107/**
108 * \ingroup testingimpl
109 * Container for results from a TestCase.
110 */
112{
113 /** Constructor. */
114 Result();
115
116 /** Test running time. */
118 /** TestCaseFailure records for each child. */
119 std::vector<TestCaseFailure> failure;
120 /** \c true if any child TestCases failed. */
122};
123
124/**
125 * \ingroup testingimpl
126 * Container for all tests.
127 * \todo Move TestRunnerImpl to separate file.
128 */
129class TestRunnerImpl : public Singleton<TestRunnerImpl>
130{
131 public:
132 /** Constructor. */
134
135 /**
136 * Add a new top-level TestSuite.
137 * \param [in] testSuite The new TestSuite.
138 */
139 void AddTestSuite(TestSuite* testSuite);
140 /** \copydoc TestCase::MustAssertOnFailure() */
141 bool MustAssertOnFailure() const;
142 /** \copydoc TestCase::MustContinueOnFailure() */
143 bool MustContinueOnFailure() const;
144 /**
145 * Check if this run should update the reference data.
146 * \return \c true if we should update the reference data.
147 */
148 bool MustUpdateData() const;
149 /**
150 * Get the path to the root of the source tree.
151 *
152 * The root directory is defined by the presence of two files:
153 * "VERSION" and "LICENSE".
154 *
155 * \returns The path to the root.
156 */
157 std::string GetTopLevelSourceDir() const;
158 /**
159 * Get the path to temporary directory.
160 * \return The temporary directory path.
161 */
162 std::string GetTempDir() const;
163 /** \copydoc TestRunner::Run() */
164 int Run(int argc, char* argv[]);
165
166 private:
167 /**
168 * Check if this is the root of the source tree.
169 * \param [in] path The path to test.
170 * \returns \c true if \pname{path} is the root.
171 */
172 bool IsTopLevelSourceDir(std::string path) const;
173 /**
174 * Clean up characters not allowed in XML.
175 *
176 * XML files have restrictions on certain characters that may be present in
177 * data. We need to replace these characters with their alternate
178 * representation on the way into the XML file.
179 *
180 * Specifically, we make these replacements:
181 * Raw Source | Replacement
182 * :--------: | :---------:
183 * '<' | "&lt;"
184 * '>' | "&gt;"
185 * '&' | "&amp;"
186 * '"' | "&39;"
187 * '\' | "&quot;"
188 *
189 * \param [in] xml The raw string.
190 * \returns The sanitized string.
191 */
192 std::string ReplaceXmlSpecialCharacters(std::string xml) const;
193 /**
194 * Print the test report.
195 *
196 * \param [in] test The TestCase to print.
197 * \param [in,out] os The output stream.
198 * \param [in] xml Generate XML output if \c true.
199 * \param [in] level Indentation level.
200 */
201 void PrintReport(TestCase* test, std::ostream* os, bool xml, int level);
202 /**
203 * Print the list of all requested test suites.
204 *
205 * \param [in] begin Iterator to the first TestCase to print.
206 * \param [in] end Iterator to the end of the list.
207 * \param [in] printTestType Prepend the test type label if \c true.
208 */
209 void PrintTestNameList(std::list<TestCase*>::const_iterator begin,
210 std::list<TestCase*>::const_iterator end,
211 bool printTestType) const;
212 /** Print the list of test types. */
213 void PrintTestTypeList() const;
214 /**
215 * Print the help text.
216 * \param [in] programName The name of the invoking program.
217 */
218 void PrintHelp(const char* programName) const;
219 /**
220 * Generate the list of tests matching the constraints.
221 *
222 * Test name and type constraints are or'ed. The duration constraint
223 * is and'ed.
224 *
225 * \param [in] testName Include a specific test by name.
226 * \param [in] testType Include all tests of give type.
227 * \param [in] maximumTestDuration Restrict to tests shorter than this.
228 * \returns The list of tests matching the filter constraints.
229 */
230 std::list<TestCase*> FilterTests(std::string testName,
231 TestSuite::Type testType,
232 TestCase::Duration maximumTestDuration);
233
234 /** Container type for the test. */
235 typedef std::vector<TestSuite*> TestSuiteVector;
236
237 TestSuiteVector m_suites; //!< The list of tests.
238 std::string m_tempDir; //!< The temporary directory.
239 bool m_verbose; //!< Produce verbose output.
240 bool m_assertOnFailure; //!< \c true if we should assert on failure.
241 bool m_continueOnFailure; //!< \c true if we should continue on failure.
242 bool m_updateData; //!< \c true if we should update reference data.
243};
244
246 std::string _actual,
247 std::string _limit,
248 std::string _message,
249 std::string _file,
250 int32_t _line)
251 : cond(_cond),
252 actual(_actual),
253 limit(_limit),
254 message(_message),
255 file(_file),
256 line(_line)
257{
258 NS_LOG_FUNCTION(this << _cond << _actual << _limit << _message << _file << _line);
259}
260
262 : childrenFailed(false)
263{
264 NS_LOG_FUNCTION(this);
265}
266
267TestCase::TestCase(std::string name)
268 : m_parent(nullptr),
269 m_dataDir(""),
270 m_runner(nullptr),
271 m_result(nullptr),
272 m_name(name),
274{
275 NS_LOG_FUNCTION(this << name);
276}
277
279{
280 NS_LOG_FUNCTION(this);
281 NS_ASSERT(m_runner == nullptr);
282 m_parent = nullptr;
283 delete m_result;
284 for (auto i = m_children.begin(); i != m_children.end(); ++i)
285 {
286 delete *i;
287 }
288 m_children.clear();
289}
290
291void
293{
294 NS_LOG_FUNCTION(&testCase << duration);
295
296 // Test names are used to create temporary directories,
297 // so we test for illegal characters.
298 //
299 // Windows: <>:"/\|?*
300 // http://msdn.microsoft.com/en-us/library/aa365247(v=vs.85).aspx
301 // Mac: : (deprecated, was path separator in Mac OS Classic, pre X)
302 // Unix: / (and .. may give trouble?)
303 //
304 // The Windows list is too restrictive: we like to label
305 // tests with "val = v1 * v2" or "v1 < 3" or "case: foo --> bar"
306 // So we allow ':<>*"
307
308 std::string badchars = "\"/\\|?";
309 // Badchar Class Regex Count of failing test names
310 // All ":<>\"/\\|?*" 611
311 // Allow ':' "<>\"/\\|?*" 128
312 // Allow ':<>' "\"/\\|?*" 12
313 // Allow ':<>*' "\"/\\|?" 0
314
315 std::string::size_type badch = testCase->m_name.find_first_of(badchars);
316 if (badch != std::string::npos)
317 {
318 /*
319 To count the bad test names, use NS_LOG_UNCOND instead
320 of NS_FATAL_ERROR, and the command
321 $ ./ns3 run "test-runner --list" 2>&1 | grep "^Invalid" | wc
322 */
323 NS_LOG_UNCOND("Invalid test name: cannot contain any of '" << badchars
324 << "': " << testCase->m_name);
325 }
326
327 testCase->m_duration = duration;
328 testCase->m_parent = this;
329 m_children.push_back(testCase);
330}
331
332bool
334{
335 NS_LOG_FUNCTION(this);
336 return m_result->childrenFailed || !m_result->failure.empty();
337}
338
339void
341{
342 NS_LOG_FUNCTION(this << runner);
343 m_result = new Result();
344 m_runner = runner;
346 DoSetup();
348 for (auto i = m_children.begin(); i != m_children.end(); ++i)
349 {
351 TestCase* test = *i;
352 test->Run(runner);
353 if (IsFailed())
354 {
355 goto out;
356 }
357 }
358 DoRun();
359out:
360 m_result->clock.End();
361 DoTeardown();
363 m_runner = nullptr;
364}
365
366std::string
368{
369 NS_LOG_FUNCTION(this);
370 return m_name;
371}
372
375{
376 return m_parent;
377}
378
379void
381 std::string actual,
382 std::string limit,
383 std::string message,
384 std::string file,
385 int32_t line)
386{
387 NS_LOG_FUNCTION(this << cond << actual << limit << message << file << line);
388 m_result->failure.emplace_back(cond, actual, limit, message, file, line);
389 // set childrenFailed flag on parents.
390 TestCase* current = m_parent;
391 while (current != nullptr)
392 {
393 current->m_result->childrenFailed = true;
394 current = current->m_parent;
395 }
396}
397
398bool
404
405bool
411
412std::string
414{
415 NS_LOG_FUNCTION(this << filename);
416 const TestCase* current = this;
417 while (current != nullptr && current->m_dataDir.empty())
418 {
419 current = current->m_parent;
420 }
421 if (current == nullptr)
422 {
423 NS_FATAL_ERROR("No one called SetDataDir prior to calling this function");
424 }
425
426 std::string a = SystemPath::Append(m_runner->GetTopLevelSourceDir(), current->m_dataDir);
427 std::string b = SystemPath::Append(a, filename);
428 return b;
429}
430
431std::string
433{
434 NS_LOG_FUNCTION(this << filename);
436 {
437 return CreateDataDirFilename(filename);
438 }
439 else
440 {
441 std::list<std::string> names;
442 const TestCase* current = this;
443 while (current != nullptr)
444 {
445 names.push_front(current->m_name);
446 current = current->m_parent;
447 }
448 std::string tempDir = SystemPath::Append(m_runner->GetTempDir(),
449 SystemPath::Join(names.begin(), names.end()));
450 tempDir = SystemPath::CreateValidSystemPath(tempDir);
451
453 return SystemPath::Append(tempDir, filename);
454 }
455}
456
457bool
459{
460 NS_LOG_FUNCTION(this);
461 return !IsStatusSuccess();
462}
463
464bool
466{
467 NS_LOG_FUNCTION(this);
468 return m_result->failure.empty();
469}
470
471void
472TestCase::SetDataDir(std::string directory)
473{
474 NS_LOG_FUNCTION(this << directory);
475 m_dataDir = directory;
476}
477
478void
480{
481 NS_LOG_FUNCTION(this);
482}
483
484void
489
491 : TestCase(name),
492 m_type(type)
493{
494 NS_LOG_FUNCTION(this << name << type);
496}
497
500{
501 NS_LOG_FUNCTION(this);
502 return m_type;
503}
504
505void
507{
508 NS_LOG_FUNCTION(this);
509}
510
512 : m_tempDir(""),
513 m_assertOnFailure(false),
514 m_continueOnFailure(true),
515 m_updateData(false)
516{
517 NS_LOG_FUNCTION(this);
518}
519
520void
522{
523 NS_LOG_FUNCTION(this << testSuite);
524 m_suites.push_back(testSuite);
525}
526
527bool
533
534bool
540
541bool
543{
544 NS_LOG_FUNCTION(this);
545 return m_updateData;
546}
547
548std::string
550{
551 NS_LOG_FUNCTION(this);
552 return m_tempDir;
553}
554
555bool
557{
558 NS_LOG_FUNCTION(this << path);
559 bool haveVersion = false;
560 bool haveLicense = false;
561
562 //
563 // If there's a file named VERSION and a file named LICENSE in this
564 // directory, we assume it's our top level source directory.
565 //
566
567 std::list<std::string> files = SystemPath::ReadFiles(path);
568 for (auto i = files.begin(); i != files.end(); ++i)
569 {
570 if (*i == "VERSION")
571 {
572 haveVersion = true;
573 }
574 else if (*i == "LICENSE")
575 {
576 haveLicense = true;
577 }
578 }
579
580 return haveVersion && haveLicense;
581}
582
583std::string
585{
586 NS_LOG_FUNCTION(this);
587 std::string self = SystemPath::FindSelfDirectory();
588 std::list<std::string> elements = SystemPath::Split(self);
589 while (!elements.empty())
590 {
591 std::string path = SystemPath::Join(elements.begin(), elements.end());
592 if (IsTopLevelSourceDir(path))
593 {
594 return path;
595 }
596 elements.pop_back();
597 }
598 NS_FATAL_ERROR("Could not find source directory from self=" << self);
599 return self;
600}
601
602//
603// XML files have restrictions on certain characters that may be present in
604// data. We need to replace these characters with their alternate
605// representation on the way into the XML file.
606//
607std::string
609{
610 NS_LOG_FUNCTION(this << xml);
611 typedef std::map<char, std::string> specials_map;
612 specials_map specials;
613 specials['<'] = "&lt;";
614 specials['>'] = "&gt;";
615 specials['&'] = "&amp;";
616 specials['"'] = "&#39;";
617 specials['\''] = "&quot;";
618
619 std::string result;
620 std::size_t length = xml.length();
621
622 for (size_t i = 0; i < length; ++i)
623 {
624 char character = xml[i];
625
626 auto it = specials.find(character);
627
628 if (it == specials.end())
629 {
630 result.push_back(character);
631 }
632 else
633 {
634 result += it->second;
635 }
636 }
637 return result;
638}
639
640/** Helper to indent output a specified number of steps. */
641struct Indent
642{
643 /**
644 * Constructor.
645 * \param [in] level The number of steps. A step is " ".
646 */
647 Indent(int level);
648 /** The number of steps. */
649 int level;
650};
651
652Indent::Indent(int _level)
653 : level(_level)
654{
655 NS_LOG_FUNCTION(this << _level);
656}
657
658/**
659 * Output streamer for Indent.
660 * \param [in,out] os The output stream.
661 * \param [in] val The Indent object.
662 * \returns The stream.
663 */
664std::ostream&
665operator<<(std::ostream& os, const Indent& val)
666{
667 for (int i = 0; i < val.level; i++)
668 {
669 os << " ";
670 }
671 return os;
672}
673
674void
675TestRunnerImpl::PrintReport(TestCase* test, std::ostream* os, bool xml, int level)
676{
677 NS_LOG_FUNCTION(this << test << os << xml << level);
678 if (test->m_result == nullptr)
679 {
680 // Do not print reports for tests that were not run.
681 return;
682 }
683 // Report times in seconds, from ms timer
684 const double MS_PER_SEC = 1000.;
685 double real = test->m_result->clock.GetElapsedReal() / MS_PER_SEC;
686 double user = test->m_result->clock.GetElapsedUser() / MS_PER_SEC;
687 double system = test->m_result->clock.GetElapsedSystem() / MS_PER_SEC;
688
689 std::streamsize oldPrecision = (*os).precision(3);
690 *os << std::fixed;
691
692 std::string statusString = test->IsFailed() ? "FAIL" : "PASS";
693 if (xml)
694 {
695 *os << Indent(level) << "<Test>" << std::endl;
696 *os << Indent(level + 1) << "<Name>" << ReplaceXmlSpecialCharacters(test->m_name)
697 << "</Name>" << std::endl;
698 *os << Indent(level + 1) << "<Result>" << statusString << "</Result>" << std::endl;
699 *os << Indent(level + 1) << "<Time real=\"" << real << "\" user=\"" << user
700 << "\" system=\"" << system << "\"/>" << std::endl;
701 for (uint32_t i = 0; i < test->m_result->failure.size(); i++)
702 {
703 TestCaseFailure failure = test->m_result->failure[i];
704 *os << Indent(level + 2) << "<FailureDetails>" << std::endl
705 << Indent(level + 3) << "<Condition>" << ReplaceXmlSpecialCharacters(failure.cond)
706 << "</Condition>" << std::endl
707 << Indent(level + 3) << "<Actual>" << ReplaceXmlSpecialCharacters(failure.actual)
708 << "</Actual>" << std::endl
709 << Indent(level + 3) << "<Limit>" << ReplaceXmlSpecialCharacters(failure.limit)
710 << "</Limit>" << std::endl
711 << Indent(level + 3) << "<Message>" << ReplaceXmlSpecialCharacters(failure.message)
712 << "</Message>" << std::endl
713 << Indent(level + 3) << "<File>" << ReplaceXmlSpecialCharacters(failure.file)
714 << "</File>" << std::endl
715 << Indent(level + 3) << "<Line>" << failure.line << "</Line>" << std::endl
716 << Indent(level + 2) << "</FailureDetails>" << std::endl;
717 }
718 for (uint32_t i = 0; i < test->m_children.size(); i++)
719 {
720 TestCase* child = test->m_children[i];
721 PrintReport(child, os, xml, level + 1);
722 }
723 *os << Indent(level) << "</Test>" << std::endl;
724 }
725 else
726 {
727 *os << Indent(level) << statusString << " " << test->GetName() << " " << real << " s"
728 << std::endl;
729 if (m_verbose)
730 {
731 for (uint32_t i = 0; i < test->m_result->failure.size(); i++)
732 {
733 *os << Indent(level) << test->m_result->failure[i] << std::endl;
734 }
735 for (uint32_t i = 0; i < test->m_children.size(); i++)
736 {
737 TestCase* child = test->m_children[i];
738 PrintReport(child, os, xml, level + 1);
739 }
740 }
741 }
742
743 (*os).unsetf(std::ios_base::floatfield);
744 (*os).precision(oldPrecision);
745}
746
747void
748TestRunnerImpl::PrintHelp(const char* program_name) const
749{
750 NS_LOG_FUNCTION(this << program_name);
751 std::cout
752 << "Usage: " << program_name << " [OPTIONS]" << std::endl
753 << std::endl
754 << "Options: " << std::endl
755 << " --help : print these options" << std::endl
756 << " --print-test-name-list : print the list of names of tests available" << std::endl
757 << " --list : an alias for --print-test-name-list" << std::endl
758 << " --print-test-types : print the type of tests along with their names" << std::endl
759 << " --print-test-type-list : print the list of types of tests available" << std::endl
760 << " --print-temp-dir : print name of temporary directory before running "
761 << std::endl
762 << " the tests" << std::endl
763 << " --test-type=TYPE : process only tests of type TYPE" << std::endl
764 << " --test-name=NAME : process only test whose name matches NAME" << std::endl
765 << " --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
766 << " for --test-name=NAME" << std::endl
767 << " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
768 << " when running under a debugger" << std::endl
769 << " --stop-on-failure : when a test fails, stop immediately" << std::endl
770 << " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
771 << " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
772 << " includes QUICK and TAKES_FOREVER includes " << std::endl
773 << " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
774 << " run by default)" << std::endl
775 << " --verbose : print details of test execution" << std::endl
776 << " --xml : format test run output as xml" << std::endl
777 << " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
778 << " --datadir=DIR : set data dir for tests to read reference files" << std::endl
779 << " --out=FILE : send test result to FILE instead of standard "
780 << "output" << std::endl
781 << " --append=FILE : append test result to FILE instead of standard "
782 << "output" << std::endl;
783}
784
785void
786TestRunnerImpl::PrintTestNameList(std::list<TestCase*>::const_iterator begin,
787 std::list<TestCase*>::const_iterator end,
788 bool printTestType) const
789{
790 NS_LOG_FUNCTION(this << &begin << &end << printTestType);
791 std::map<TestSuite::Type, std::string> label;
792
793 label[TestSuite::Type::ALL] = "all ";
794 label[TestSuite::Type::UNIT] = "unit ";
795 label[TestSuite::Type::SYSTEM] = "system ";
796 label[TestSuite::Type::EXAMPLE] = "example ";
797 label[TestSuite::Type::PERFORMANCE] = "performance ";
798
799 for (auto i = begin; i != end; ++i)
800 {
801 auto test = dynamic_cast<TestSuite*>(*i);
802 NS_ASSERT(test != nullptr);
803 if (printTestType)
804 {
805 std::cout << label[test->GetTestType()];
806 }
807 std::cout << test->GetName() << std::endl;
808 }
809}
810
811void
813{
814 NS_LOG_FUNCTION(this);
815 std::cout << " core: Run all TestSuite-based tests (exclude examples)" << std::endl;
816 std::cout << " example: Examples (to see if example programs run successfully)"
817 << std::endl;
818 std::cout
819 << " performance: Performance Tests (check to see if the system is as fast as expected)"
820 << std::endl;
821 std::cout << " system: System Tests (spans modules to check integration of modules)"
822 << std::endl;
823 std::cout << " unit: Unit Tests (within modules to check basic functionality)"
824 << std::endl;
825}
826
827std::list<TestCase*>
828TestRunnerImpl::FilterTests(std::string testName,
829 TestSuite::Type testType,
830 TestCase::Duration maximumTestDuration)
831{
832 NS_LOG_FUNCTION(this << testName << testType);
833 std::list<TestCase*> tests;
834 for (uint32_t i = 0; i < m_suites.size(); ++i)
835 {
836 TestSuite* test = m_suites[i];
837 if (testType != TestSuite::Type::ALL && test->GetTestType() != testType)
838 {
839 // skip test
840 continue;
841 }
842 if (!testName.empty() && test->GetName() != testName)
843 {
844 // skip test
845 continue;
846 }
847
848 // Remove any test cases that should be skipped.
849 for (auto j = test->m_children.begin(); j != test->m_children.end();)
850 {
851 TestCase* testCase = *j;
852
853 // If this test case takes longer than the maximum test
854 // duration that should be run, then don't run it.
855 if (testCase->m_duration > maximumTestDuration)
856 {
857 // Free this test case's memory.
858 delete *j;
859
860 // Remove this test case from the test suite.
861 j = test->m_children.erase(j);
862 }
863 else
864 {
865 // Only advance through the vector elements if this test
866 // case wasn't deleted.
867 ++j;
868 }
869 }
870
871 // Add this test suite.
872 tests.push_back(test);
873 }
874 return tests;
875}
876
877int
878TestRunnerImpl::Run(int argc, char* argv[])
879{
880 NS_LOG_FUNCTION(this << argc << argv);
881 std::string testName = "";
882 std::string testTypeString = "";
883 std::string out = "";
884 std::string fullness = "";
885 bool xml = false;
886 bool append = false;
887 bool printTempDir = false;
888 bool printTestTypeList = false;
889 bool printTestNameList = false;
890 bool printTestTypeAndName = false;
891 TestCase::Duration maximumTestDuration = TestCase::Duration::QUICK;
892 char* progname = argv[0];
893
894 char** argi = argv;
895 ++argi;
896
897 while (*argi != nullptr)
898 {
899 std::string arg = *argi;
900
901 if (arg == "--assert-on-failure")
902 {
903 m_assertOnFailure = true;
904 }
905 else if (arg == "--stop-on-failure")
906 {
907 m_continueOnFailure = false;
908 }
909 else if (arg == "--verbose")
910 {
911 m_verbose = true;
912 }
913 else if (arg == "--print-temp-dir")
914 {
915 printTempDir = true;
916 }
917 else if (arg == "--update-data")
918 {
919 m_updateData = true;
920 }
921 else if (arg == "--print-test-name-list" || arg == "--list")
922 {
923 printTestNameList = true;
924 }
925 else if (arg == "--print-test-types")
926 {
927 printTestTypeAndName = true;
928 }
929 else if (arg == "--print-test-type-list")
930 {
931 printTestTypeList = true;
932 }
933 else if (arg == "--append")
934 {
935 append = true;
936 }
937 else if (arg == "--xml")
938 {
939 xml = true;
940 }
941 else if (arg.find("--test-type=") != std::string::npos)
942 {
943 testTypeString = arg.substr(arg.find_first_of('=') + 1);
944 }
945 else if (arg.find("--test-name=") != std::string::npos ||
946 arg.find("--suite=") != std::string::npos)
947 {
948 testName = arg.substr(arg.find_first_of('=') + 1);
949 }
950 else if (arg.find("--tempdir=") != std::string::npos)
951 {
952 m_tempDir = arg.substr(arg.find_first_of('=') + 1);
953 }
954 else if (arg.find("--out=") != std::string::npos)
955 {
956 out = arg.substr(arg.find_first_of('=') + 1);
957 }
958 else if (arg.find("--fullness=") != std::string::npos)
959 {
960 fullness = arg.substr(arg.find_first_of('=') + 1);
961
962 // Set the maximum test length allowed.
963 if (fullness == "QUICK")
964 {
965 maximumTestDuration = TestCase::Duration::QUICK;
966 }
967 else if (fullness == "EXTENSIVE")
968 {
969 maximumTestDuration = TestCase::Duration::EXTENSIVE;
970 }
971 else if (fullness == "TAKES_FOREVER")
972 {
973 maximumTestDuration = TestCase::Duration::TAKES_FOREVER;
974 }
975 else
976 {
977 // Wrong fullness option
978 PrintHelp(progname);
979 return 3;
980 }
981 }
982 else
983 {
984 // Print the help if arg == "--help" or arg is an un-recognized command-line argument
985 PrintHelp(progname);
986 return 0;
987 }
988 argi++;
989 }
990 TestSuite::Type testType;
991 if (testTypeString.empty() || testTypeString == "core")
992 {
993 testType = TestSuite::Type::ALL;
994 }
995 else if (testTypeString == "example")
996 {
997 testType = TestSuite::Type::EXAMPLE;
998 }
999 else if (testTypeString == "unit")
1000 {
1001 testType = TestSuite::Type::UNIT;
1002 }
1003 else if (testTypeString == "system")
1004 {
1005 testType = TestSuite::Type::SYSTEM;
1006 }
1007 else if (testTypeString == "performance")
1008 {
1010 }
1011 else
1012 {
1013 std::cout << "Invalid test type specified: " << testTypeString << std::endl;
1015 return 1;
1016 }
1017
1018 std::list<TestCase*> tests = FilterTests(testName, testType, maximumTestDuration);
1019
1020 if (m_tempDir.empty())
1021 {
1023 }
1024 if (printTempDir)
1025 {
1026 std::cout << m_tempDir << std::endl;
1027 }
1028 if (printTestNameList)
1029 {
1030 PrintTestNameList(tests.begin(), tests.end(), printTestTypeAndName);
1031 return 0;
1032 }
1033 if (printTestTypeList)
1034 {
1036 return 0;
1037 }
1038
1039 std::ostream* os;
1040 if (!out.empty())
1041 {
1042 std::ofstream* ofs;
1043 ofs = new std::ofstream();
1044 std::ios_base::openmode mode = std::ios_base::out;
1045 if (append)
1046 {
1047 mode |= std::ios_base::app;
1048 }
1049 else
1050 {
1051 mode |= std::ios_base::trunc;
1052 }
1053 ofs->open(out, mode);
1054 os = ofs;
1055 }
1056 else
1057 {
1058 os = &std::cout;
1059 }
1060
1061 // let's run our tests now.
1062 bool failed = false;
1063 if (tests.empty())
1064 {
1065 std::cerr << "Error: no tests match the requested string" << std::endl;
1066 return 1;
1067 }
1068 else if (tests.size() > 1)
1069 {
1070 std::cerr << "Error: tests should be launched separately (one at a time)" << std::endl;
1071 return 1;
1072 }
1073
1074 for (auto i = tests.begin(); i != tests.end(); ++i)
1075 {
1076 TestCase* test = *i;
1077
1078#ifdef ENABLE_DES_METRICS
1079 {
1080 /*
1081 Reorganize argv
1082 Since DES Metrics uses argv[0] for the trace file name,
1083 grab the test name and put it in argv[0],
1084 with test-runner as argv[1]
1085 then the rest of the original arguments.
1086 */
1087 std::string testname = test->GetName();
1088 std::string runner = "[" + SystemPath::Split(argv[0]).back() + "]";
1089
1090 std::vector<std::string> desargs;
1091 desargs.push_back(testname);
1092 desargs.push_back(runner);
1093 for (int i = 1; i < argc; ++i)
1094 {
1095 desargs.push_back(argv[i]);
1096 }
1097
1099 }
1100#endif
1101
1102 test->Run(this);
1103 PrintReport(test, os, xml, 0);
1104 if (test->IsFailed())
1105 {
1106 failed = true;
1108 {
1109 return 1;
1110 }
1111 }
1112 }
1113
1114 if (!out.empty())
1115 {
1116 delete os;
1117 }
1118
1119 return failed ? 1 : 0;
1120}
1121
1122int
1123TestRunner::Run(int argc, char* argv[])
1124{
1125 NS_LOG_FUNCTION(argc << argv);
1126 return TestRunnerImpl::Get()->Run(argc, argv);
1127}
1128
1129std::ostream&
1130operator<<(std::ostream& os, TestSuite::Type type)
1131{
1132 switch (type)
1133 {
1135 return os << "ALL";
1137 return os << "UNIT";
1139 return os << "SYSTEM";
1141 return os << "EXAMPLE";
1143 return os << "PERFORMANCE";
1144 };
1145 return os << "UNKNOWN(" << static_cast<uint32_t>(type) << ")";
1146}
1147
1148std::ostream&
1149operator<<(std::ostream& os, TestCase::Duration duration)
1150{
1151 switch (duration)
1152 {
1154 return os << "QUICK";
1156 return os << "EXTENSIVE";
1158 return os << "TAKES_FOREVER";
1159 };
1160 return os << "UNKNOWN(" << static_cast<uint32_t>(duration) << ")";
1161}
1162
1163} // namespace ns3
NS_ABORT_x macro definitions.
NS_ASSERT() and NS_ASSERT_MSG() macro definitions.
void Initialize(std::vector< std::string > args, std::string outDir="")
Open the DesMetrics trace file and print the header.
static void ResetNextStreamIndex()
Resets the global stream index counter.
A template singleton.
Definition singleton.h:57
static TestRunnerImpl * Get()
Definition singleton.h:96
Measure elapsed wall clock time in milliseconds.
int64_t End()
Stop measuring the time since Start() was called.
void Start()
Start a measure.
encapsulates test code
Definition test.h:1050
std::string m_name
TestCase name.
Definition test.h:1255
bool MustContinueOnFailure() const
Check if this run should continue on failure.
Definition test.cc:406
bool IsStatusFailure() const
Check if any tests failed.
Definition test.cc:458
std::string m_dataDir
My data directory.
Definition test.h:1252
static constexpr auto QUICK
Deprecated test duration simple enums.
Definition test.h:1067
std::string CreateDataDirFilename(std::string filename)
Construct the full path to a file in the data directory.
Definition test.cc:413
TestCase * m_parent
Pointer to my parent TestCase.
Definition test.h:1250
void AddTestCase(TestCase *testCase, Duration duration=Duration::QUICK)
Add an individual child TestCase to this test suite.
Definition test.cc:292
Result * m_result
Results data.
Definition test.h:1254
bool IsStatusSuccess() const
Check if all tests passed.
Definition test.cc:465
virtual void DoSetup()
Implementation to do any local setup required for this TestCase.
Definition test.cc:479
virtual ~TestCase()
Destructor.
Definition test.cc:278
Duration
How long the test takes to execute.
Definition test.h:1054
@ EXTENSIVE
Medium length test.
@ TAKES_FOREVER
Very long running test.
std::string CreateTempDirFilename(std::string filename)
Construct the full path to a file in a temporary directory.
Definition test.cc:432
TestRunnerImpl * m_runner
Pointer to the TestRunner.
Definition test.h:1253
TestCase * GetParent() const
Get the parent of this TestCase.
Definition test.cc:374
bool MustAssertOnFailure() const
Check if this run should assert on failure.
Definition test.cc:399
void SetDataDir(std::string directory)
Set the data directory where reference trace files can be found.
Definition test.cc:472
virtual void DoTeardown()
Implementation to do any local setup required for this TestCase.
Definition test.cc:485
void Run(TestRunnerImpl *runner)
Executes DoSetup(), DoRun(), and DoTeardown() for the TestCase.
Definition test.cc:340
TestCase(const TestCase &)=delete
virtual void DoRun()=0
Implementation to actually run this TestCase.
std::string GetName() const
Definition test.cc:367
Duration m_duration
TestCase duration.
Definition test.h:1256
void ReportTestFailure(std::string cond, std::string actual, std::string limit, std::string message, std::string file, int32_t line)
Log the failure of this TestCase.
Definition test.cc:380
bool IsFailed() const
Check if any tests failed.
Definition test.cc:333
std::vector< TestCase * > m_children
Vector of my children.
Definition test.h:1251
static int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition test.cc:1123
Container for all tests.
Definition test.cc:130
void PrintHelp(const char *programName) const
Print the help text.
Definition test.cc:748
bool m_assertOnFailure
true if we should assert on failure.
Definition test.cc:240
std::string ReplaceXmlSpecialCharacters(std::string xml) const
Clean up characters not allowed in XML.
Definition test.cc:608
std::list< TestCase * > FilterTests(std::string testName, TestSuite::Type testType, TestCase::Duration maximumTestDuration)
Generate the list of tests matching the constraints.
Definition test.cc:828
bool MustUpdateData() const
Check if this run should update the reference data.
Definition test.cc:542
bool IsTopLevelSourceDir(std::string path) const
Check if this is the root of the source tree.
Definition test.cc:556
bool m_continueOnFailure
true if we should continue on failure.
Definition test.cc:241
bool m_updateData
true if we should update reference data.
Definition test.cc:242
std::string m_tempDir
The temporary directory.
Definition test.cc:238
std::string GetTempDir() const
Get the path to temporary directory.
Definition test.cc:549
void PrintReport(TestCase *test, std::ostream *os, bool xml, int level)
Print the test report.
Definition test.cc:675
std::vector< TestSuite * > TestSuiteVector
Container type for the test.
Definition test.cc:235
bool MustContinueOnFailure() const
Check if this run should continue on failure.
Definition test.cc:535
int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition test.cc:878
bool MustAssertOnFailure() const
Check if this run should assert on failure.
Definition test.cc:528
TestRunnerImpl()
Constructor.
Definition test.cc:511
void AddTestSuite(TestSuite *testSuite)
Add a new top-level TestSuite.
Definition test.cc:521
bool m_verbose
Produce verbose output.
Definition test.cc:239
TestSuiteVector m_suites
The list of tests.
Definition test.cc:237
std::string GetTopLevelSourceDir() const
Get the path to the root of the source tree.
Definition test.cc:584
void PrintTestTypeList() const
Print the list of test types.
Definition test.cc:812
void PrintTestNameList(std::list< TestCase * >::const_iterator begin, std::list< TestCase * >::const_iterator end, bool printTestType) const
Print the list of all requested test suites.
Definition test.cc:786
A suite of tests to run.
Definition test.h:1267
Type
Type of test.
Definition test.h:1274
@ PERFORMANCE
This test suite implements a Performance Test.
@ ALL
Token to represent all tests.
@ EXAMPLE
This test suite implements an Example Test.
@ UNIT
This test suite implements a Unit Test.
@ SYSTEM
This test suite implements a System Test.
TestSuite(std::string name, Type type=Type::UNIT)
Construct a new test suite.
Definition test.cc:490
TestSuite::Type m_type
Type of this TestSuite.
Definition test.h:1319
void DoRun() override
Implementation to actually run this TestCase.
Definition test.cc:506
TestSuite::Type GetTestType()
get the kind of test this test suite implements
Definition test.cc:499
Declaration of the various ns3::Config functions and classes.
ns3::DesMetrics declaration.
#define NS_ASSERT(condition)
At runtime, in debugging builds, if this condition is not true, the program prints the source file,...
Definition assert.h:55
void Reset()
Reset the initial value of every attribute as well as the value of every global to what they were bef...
Definition config.cc:848
#define NS_FATAL_ERROR(msg)
Report a fatal error with a message and terminate.
#define NS_LOG_UNCOND(msg)
Output the requested message unconditionally.
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition log.h:191
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by ",...
std::list< std::string > ReadFiles(std::string path)
Get the list of files located in a file system directory.
std::list< std::string > Split(std::string path)
Split a file system path into directories according to the local path separator.
void MakeDirectories(std::string path)
Create all the directories leading to path.
std::string MakeTemporaryDirectoryName()
Get the name of a temporary directory.
std::string Append(std::string left, std::string right)
Join two file system path elements.
std::string Join(std::list< std::string >::const_iterator begin, std::list< std::string >::const_iterator end)
Join a list of file system path directories into a single file system path.
std::string CreateValidSystemPath(const std::string path)
Replace incompatible characters in a path, to get a path compatible with different file systems.
std::string FindSelfDirectory()
Get the file system path to the current executable.
bool TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
Compare two double precision floating point numbers and declare them equal if they are within some ep...
Definition test.cc:36
Debug message logging.
Every class exported by the ns3 library is enclosed in the ns3 namespace.
std::ostream & operator<<(std::ostream &os, const Angles &a)
Definition angles.cc:148
-ns3 Test suite for the ns3 wrapper script
ns3::RngSeedManager declaration.
ns3::Singleton declaration and template implementation.
Helper to indent output a specified number of steps.
Definition test.cc:642
Indent(int level)
Constructor.
Definition test.cc:652
int level
The number of steps.
Definition test.cc:649
Container for results from a TestCase.
Definition test.cc:112
Result()
Constructor.
Definition test.cc:261
std::vector< TestCaseFailure > failure
TestCaseFailure records for each child.
Definition test.cc:119
bool childrenFailed
true if any child TestCases failed.
Definition test.cc:121
SystemWallClockMs clock
Test running time.
Definition test.cc:117
Container for details of a test failure.
Definition test.cc:65
std::string actual
The actual value returned by the test.
Definition test.cc:83
std::string file
The source file.
Definition test.cc:86
std::string message
The associated message.
Definition test.cc:85
int32_t line
The source line.
Definition test.cc:87
TestCaseFailure(std::string _cond, std::string _actual, std::string _limit, std::string _message, std::string _file, int32_t _line)
Constructor.
Definition test.cc:245
std::string cond
The name of the condition being tested.
Definition test.cc:82
std::string limit
The expected value.
Definition test.cc:84
ns3::SystemPath declarations.
ns3::TestCase, ns3::TestSuite, ns3::TestRunner declarations, and NS_TEST_ASSERT macro definitions.