2 * Run a set of tests, reporting results.
8 * Expects a list of executables located in the given file, one line per
9 * executable. For each one, runs it as part of a test suite, reporting
10 * results. Test output should start with a line containing the number of
11 * tests (numbered from 1 to this number), and then each line should be in the
18 * where <number> is the number of the test. ok indicates success, not ok
19 * indicates failure, and "# skip" indicates the test was skipped for some
20 * reason (maybe because it doesn't apply to this platform). This is a subset
21 * of TAP as documented in Test::Harness::TAP, which comes with Perl.
23 * Any bug reports, bug fixes, and improvements are very much welcome and
24 * should be sent to the e-mail address below.
26 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009
27 * Russ Allbery <rra@stanford.edu>
29 * Permission is hereby granted, free of charge, to any person obtaining a
30 * copy of this software and associated documentation files (the "Software"),
31 * to deal in the Software without restriction, including without limitation
32 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
33 * and/or sell copies of the Software, and to permit persons to whom the
34 * Software is furnished to do so, subject to the following conditions:
36 * The above copyright notice and this permission notice shall be included in
37 * all copies or substantial portions of the Software.
39 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
40 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
41 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
42 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
43 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
44 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
45 * DEALINGS IN THE SOFTWARE.
57 #include <sys/types.h>
62 /* sys/time.h must be included before sys/resource.h on some platforms. */
63 #include <sys/resource.h>
65 /* AIX doesn't have WCOREDUMP. */
67 # define WCOREDUMP(status) ((unsigned)(status) & 0x80)
71 * The source and build versions of the tests directory. This is used to set
72 * the SOURCE and BUILD environment variables and find test programs, if set.
73 * Normally, this should be set as part of the build process to the test
74 * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
83 /* Test status codes. */
91 /* Error exit statuses for test processes. */
92 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
93 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
94 #define CHILDERR_STDERR 102 /* Couldn't open stderr file. */
96 /* Structure to hold data for a set of tests. */
98 char *file; /* The file name of the test. */
99 char *path; /* The path to the test program. */
100 int count; /* Expected count of tests. */
101 int current; /* The last seen test number. */
102 int length; /* The length of the last status message. */
103 int passed; /* Count of passing tests. */
104 int failed; /* Count of failing lists. */
105 int skipped; /* Count of skipped tests (passed). */
106 enum test_status *results; /* Table of results by test number. */
107 int aborted; /* Whether the set as aborted. */
108 int reported; /* Whether the results were reported. */
109 int status; /* The exit status of the test. */
110 int all_skipped; /* Whether all tests were skipped. */
111 char *reason; /* Why all tests were skipped. */
114 /* Structure to hold a linked list of test sets. */
117 struct testlist *next;
121 * Header used for test output. %s is replaced by the file name of the list
124 static const char banner[] = "\n\
125 Running all tests listed in %s. If any tests fail, run the failing\n\
126 test program with runtests -o to see more details.\n\n";
128 /* Header for reports of failed tests. */
129 static const char header[] = "\n\
130 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
131 -------------------------- -------------- ---- ---- ------------------------";
133 /* Include the file name and line number in malloc failures. */
134 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
135 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
139 * Report a fatal error, including the results of strerror, and exit.
142 sysdie(const char *format, ...)
149 fprintf(stderr, "runtests: ");
150 va_start(args, format);
151 vfprintf(stderr, format, args);
153 fprintf(stderr, ": %s\n", strerror(oerrno));
159 * Allocate memory, reporting a fatal error and exiting on failure.
162 x_malloc(size_t size, const char *file, int line)
168 sysdie("failed to malloc %lu bytes at %s line %d",
169 (unsigned long) size, file, line);
175 * Copy a string, reporting a fatal error and exiting on failure.
178 x_strdup(const char *s, const char *file, int line)
186 sysdie("failed to strdup %lu bytes at %s line %d",
187 (unsigned long) len, file, line);
194 * Given a struct timeval, return the number of seconds it represents as a
195 * double. Use difftime() to convert a time_t to a double.
198 tv_seconds(const struct timeval *tv)
200 return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
205 * Given two struct timevals, return the difference in seconds.
208 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
210 return tv_seconds(tv1) - tv_seconds(tv0);
215 * Given two struct timevals, return the sum in seconds as a double.
218 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
220 return tv_seconds(tv1) + tv_seconds(tv2);
225 * Given a pointer to a string, skip any leading whitespace and return a
226 * pointer to the first non-whitespace character.
229 skip_whitespace(const char *p)
231 while (isspace((unsigned char)(*p)))
238 * Read the first line of test output, which should contain the range of
239 * test numbers, and initialize the testset structure. Assume it was zeroed
240 * before being passed in. Return true if initialization succeeds, false
244 test_init(const char *line, struct testset *ts)
249 * Prefer a simple number of tests, but if the count is given as a range
250 * such as 1..10, accept that too for compatibility with Perl's
253 line = skip_whitespace(line);
254 if (strncmp(line, "1..", 3) == 0)
258 * Get the count, check it for validity, and initialize the struct. If we
259 * have something of the form "1..0 # skip foo", the whole file was
260 * skipped; record that.
262 i = strtol(line, (char **) &line, 10);
264 line = skip_whitespace(line);
266 line = skip_whitespace(line + 1);
267 if (strncasecmp(line, "skip", 4) == 0) {
268 line = skip_whitespace(line + 4);
270 ts->reason = xstrdup(line);
271 ts->reason[strlen(ts->reason) - 1] = '\0';
280 puts("ABORTED (invalid test count)");
286 ts->results = xmalloc(ts->count * sizeof(enum test_status));
287 for (i = 0; i < ts->count; i++)
288 ts->results[i] = TEST_INVALID;
294 * Start a program, connecting its stdout to a pipe on our end and its stderr
295 * to /dev/null, and storing the file descriptor to read from in the two
296 * argument. Returns the PID of the new process. Errors are fatal.
299 test_start(const char *path, int *fd)
304 if (pipe(fds) == -1) {
307 sysdie("can't create pipe");
310 if (child == (pid_t) -1) {
313 sysdie("can't fork");
314 } else if (child == 0) {
315 /* In child. Set up our stdout and stderr. */
316 errfd = open("/dev/null", O_WRONLY);
318 _exit(CHILDERR_STDERR);
319 if (dup2(errfd, 2) == -1)
322 if (dup2(fds[1], 1) == -1)
325 /* Now, exec our process. */
326 if (execl(path, path, (char *) 0) == -1)
327 _exit(CHILDERR_EXEC);
329 /* In parent. Close the extra file descriptor. */
338 * Back up over the output saying what test we were executing.
341 test_backspace(struct testset *ts)
345 if (!isatty(STDOUT_FILENO))
347 for (i = 0; i < ts->length; i++)
349 for (i = 0; i < ts->length; i++)
351 for (i = 0; i < ts->length; i++)
358 * Given a single line of output from a test, parse it and return the success
359 * status of that test. Anything printed to stdout not matching the form
360 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
364 test_checkline(const char *line, struct testset *ts)
366 enum test_status status = TEST_PASS;
371 /* Before anything, check for a test abort. */
372 bail = strstr(line, "Bail out!");
374 bail = skip_whitespace(bail + strlen("Bail out!"));
378 length = strlen(bail);
379 if (bail[length - 1] == '\n')
382 printf("ABORTED (%.*s)\n", length, bail);
390 * If the given line isn't newline-terminated, it was too big for an
391 * fgets(), which means ignore it.
393 if (line[strlen(line) - 1] != '\n')
396 /* Parse the line, ignoring something we can't parse. */
397 if (strncmp(line, "not ", 4) == 0) {
401 if (strncmp(line, "ok", 2) != 0)
403 line = skip_whitespace(line + 2);
405 current = strtol(line, &end, 10);
406 if (errno != 0 || end == line)
407 current = ts->current + 1;
408 if (current <= 0 || current > ts->count) {
410 printf("ABORTED (invalid test number %d)\n", current);
417 * Handle directives. We should probably do something more interesting
418 * with unexpected passes of todo tests.
420 while (isdigit((unsigned char)(*line)))
422 line = skip_whitespace(line);
424 line = skip_whitespace(line + 1);
425 if (strncasecmp(line, "skip", 4) == 0)
427 if (strncasecmp(line, "todo", 4) == 0)
428 status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
431 /* Make sure that the test number is in range and not a duplicate. */
432 if (ts->results[current - 1] != TEST_INVALID) {
434 printf("ABORTED (duplicate test number %d)\n", current);
440 /* Good results. Increment our various counters. */
442 case TEST_PASS: ts->passed++; break;
443 case TEST_FAIL: ts->failed++; break;
444 case TEST_SKIP: ts->skipped++; break;
447 ts->current = current;
448 ts->results[current - 1] = status;
450 if (isatty(STDOUT_FILENO)) {
451 ts->length = printf("%d/%d", current, ts->count);
458 * Print out a range of test numbers, returning the number of characters it
459 * took up. Add a comma and a space before the range if chars indicates that
460 * something has already been printed on the line, and print ... instead if
461 * chars plus the space needed would go over the limit (use a limit of 0 to
465 test_print_range(int first, int last, int chars, int limit)
473 if (!limit || chars <= limit) out += printf(", ");
475 for (n = first; n > 0; n /= 10)
478 for (n = last; n > 0; n /= 10)
482 if (limit && chars + needed > limit) {
484 out += printf("...");
487 out += printf("%d-", first);
488 out += printf("%d", last);
495 * Summarize a single test set. The second argument is 0 if the set exited
496 * cleanly, a positive integer representing the exit status if it exited
497 * with a non-zero status, and a negative integer representing the signal
498 * that terminated it if it was killed by a signal.
501 test_summarize(struct testset *ts, int status)
510 fputs("ABORTED", stdout);
512 printf(" (passed %d/%d)", ts->passed, ts->count - ts->skipped);
514 for (i = 0; i < ts->count; i++) {
515 if (ts->results[i] == TEST_INVALID) {
517 fputs("MISSED ", stdout);
518 if (first && i == last)
522 test_print_range(first, last, missing - 1, 0);
530 test_print_range(first, last, missing - 1, 0);
533 for (i = 0; i < ts->count; i++) {
534 if (ts->results[i] == TEST_FAIL) {
535 if (missing && !failed)
538 fputs("FAILED ", stdout);
539 if (first && i == last)
543 test_print_range(first, last, failed - 1, 0);
551 test_print_range(first, last, failed - 1, 0);
552 if (!missing && !failed) {
553 fputs(!status ? "ok" : "dubious", stdout);
554 if (ts->skipped > 0) {
555 if (ts->skipped == 1)
556 printf(" (skipped %d test)", ts->skipped);
558 printf(" (skipped %d tests)", ts->skipped);
563 printf(" (exit status %d)", status);
565 printf(" (killed by signal %d%s)", -status,
566 WCOREDUMP(ts->status) ? ", core dumped" : "");
572 * Given a test set, analyze the results, classify the exit status, handle a
573 * few special error messages, and then pass it along to test_summarize()
574 * for the regular output.
577 test_analyze(struct testset *ts)
581 if (ts->all_skipped) {
582 if (ts->reason == NULL)
585 printf("skipped (%s)\n", ts->reason);
587 } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
588 switch (WEXITSTATUS(ts->status)) {
591 puts("ABORTED (can't dup file descriptors)");
595 puts("ABORTED (execution failed -- not found?)");
597 case CHILDERR_STDERR:
599 puts("ABORTED (can't open /dev/null)");
602 test_summarize(ts, WEXITSTATUS(ts->status));
606 } else if (WIFSIGNALED(ts->status)) {
607 test_summarize(ts, -WTERMSIG(ts->status));
610 test_summarize(ts, 0);
611 return (ts->failed == 0);
617 * Runs a single test set, accumulating and then reporting the results.
618 * Returns true if the test set was successfully run and all tests passed,
622 test_run(struct testset *ts)
624 pid_t testpid, child;
625 int outfd, i, status;
630 * Initialize the test and our data structures, flagging this set in error
631 * if the initialization fails.
633 testpid = test_start(ts->path, &outfd);
634 output = fdopen(outfd, "r");
638 sysdie("fdopen failed");
640 if (!fgets(buffer, sizeof(buffer), output))
642 if (!ts->aborted && !test_init(buffer, ts))
645 /* Pass each line of output to test_checkline(). */
646 while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
647 test_checkline(buffer, ts);
653 * Consume the rest of the test output, close the output descriptor,
654 * retrieve the exit status, and pass that information to test_analyze()
655 * for eventual output.
657 while (fgets(buffer, sizeof(buffer), output))
660 child = waitpid(testpid, &ts->status, 0);
661 if (child == (pid_t) -1) {
666 sysdie("waitpid for %u failed", (unsigned int) testpid);
670 status = test_analyze(ts);
672 /* Convert missing tests to failed tests. */
673 for (i = 0; i < ts->count; i++) {
674 if (ts->results[i] == TEST_INVALID) {
676 ts->results[i] = TEST_FAIL;
684 /* Summarize a list of test failures. */
686 test_fail_summary(const struct testlist *fails)
689 int i, chars, total, first, last;
693 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
694 -------------------------- -------------- ---- ---- -------------- */
695 for (; fails; fails = fails->next) {
697 total = ts->count - ts->skipped;
698 printf("%-26.26s %4d/%-4d %3.0f%% %4d ", ts->file, ts->failed,
699 total, total ? (ts->failed * 100.0) / total : 0,
701 if (WIFEXITED(ts->status))
702 printf("%4d ", WEXITSTATUS(ts->status));
712 for (i = 0; i < ts->count; i++) {
713 if (ts->results[i] == TEST_FAIL) {
714 if (first && i == last)
718 chars += test_print_range(first, last, chars, 20);
725 test_print_range(first, last, chars, 20);
732 * Given the name of a test, a pointer to the testset struct, and the source
733 * and build directories, find the test. We try first relative to the current
734 * directory, then in the build directory (if not NULL), then in the source
735 * directory. In each of those directories, we first try a "-t" extension and
736 * then a ".t" extension. When we find an executable program, we fill in the
737 * path member of the testset struct. If none of those paths are executable,
738 * just fill in the name of the test with "-t" appended.
740 * The caller is responsible for freeing the path member of the testset
744 find_test(const char *name, struct testset *ts, const char *source,
748 const char *bases[] = { ".", build, source, NULL };
751 for (i = 0; bases[i] != NULL; i++) {
752 path = xmalloc(strlen(bases[i]) + strlen(name) + 4);
753 sprintf(path, "%s/%s-t", bases[i], name);
754 if (access(path, X_OK) != 0)
755 path[strlen(path) - 2] = '.';
756 if (access(path, X_OK) == 0)
762 path = xmalloc(strlen(name) + 3);
763 sprintf(path, "%s-t", name);
770 * Run a batch of tests from a given file listing each test on a line by
771 * itself. Takes two additional parameters: the root of the source directory
772 * and the root of the build directory. Test programs will be first searched
773 * for in the current directory, then the build directory, then the source
774 * directory. The file must be rewindable. Returns true iff all tests
778 test_batch(const char *testlist, const char *source, const char *build)
785 struct testset ts, *tmp;
786 struct timeval start, end;
788 struct testlist *failhead = 0;
789 struct testlist *failtail = 0;
797 * Open our file of tests to run and scan it, checking for lines that
798 * are too long and searching for the longest line.
800 tests = fopen(testlist, "r");
802 sysdie("can't open %s", testlist);
804 while (fgets(buffer, sizeof(buffer), tests)) {
806 length = strlen(buffer) - 1;
807 if (buffer[length] != '\n') {
808 fprintf(stderr, "%s:%d: line too long\n", testlist, line);
811 if (length > longest)
814 if (fseek(tests, 0, SEEK_SET) == -1)
815 sysdie("can't rewind %s", testlist);
818 * Add two to longest and round up to the nearest tab stop. This is how
819 * wide the column for printing the current test name will be.
823 longest += 8 - (longest % 8);
825 /* Start the wall clock timer. */
826 gettimeofday(&start, NULL);
829 * Now, plow through our tests again, running each one. Check line
830 * length again out of paranoia.
833 while (fgets(buffer, sizeof(buffer), tests)) {
835 length = strlen(buffer) - 1;
836 if (buffer[length] != '\n') {
837 fprintf(stderr, "%s:%d: line too long\n", testlist, line);
840 buffer[length] = '\0';
841 fputs(buffer, stdout);
842 for (i = length; i < longest; i++)
844 if (isatty(STDOUT_FILENO))
846 memset(&ts, 0, sizeof(ts));
847 ts.file = xstrdup(buffer);
848 find_test(buffer, &ts, source, build);
853 if (ts.reason != NULL)
856 tmp = xmalloc(sizeof(struct testset));
857 memcpy(tmp, &ts, sizeof(struct testset));
859 failhead = xmalloc(sizeof(struct testset));
864 failtail->next = xmalloc(sizeof(struct testset));
865 failtail = failtail->next;
870 aborted += ts.aborted;
871 total += ts.count + ts.all_skipped;
873 skipped += ts.skipped + ts.all_skipped;
878 /* Stop the timer and get our child resource statistics. */
879 gettimeofday(&end, NULL);
880 getrusage(RUSAGE_CHILDREN, &stats);
882 /* Print out our final results. */
884 test_fail_summary(failhead);
888 printf("Aborted %d test set", aborted);
890 printf("Aborted %d test sets", aborted);
891 printf(", passed %d/%d tests", passed, total);
893 else if (failed == 0)
894 fputs("All tests successful", stdout);
896 printf("Failed %d/%d tests, %.2f%% okay", failed, total,
897 (total - failed) * 100.0 / total);
900 printf(", %d test skipped", skipped);
902 printf(", %d tests skipped", skipped);
905 printf("Files=%d, Tests=%d", line, total);
906 printf(", %.2f seconds", tv_diff(&end, &start));
907 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
908 tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
909 tv_sum(&stats.ru_utime, &stats.ru_stime));
910 return (failed == 0 && aborted == 0);
915 * Run a single test case. This involves just running the test program after
916 * having done the environment setup and finding the test program.
919 test_single(const char *program, const char *source, const char *build)
923 memset(&ts, 0, sizeof(ts));
924 find_test(program, &ts, source, build);
925 if (execl(ts.path, ts.path, (char *) 0) == -1)
926 sysdie("cannot exec %s", ts.path);
931 * Main routine. Set the SOURCE and BUILD environment variables and then,
932 * given a file listing tests, run each test listed.
935 main(int argc, char *argv[])
941 const char *source = SOURCE;
942 const char *build = BUILD;
944 while ((option = getopt(argc, argv, "b:os:")) != EOF) {
962 fprintf(stderr, "Usage: runtests <test-list>\n");
966 if (source != NULL) {
967 setting = xmalloc(strlen("SOURCE=") + strlen(source) + 1);
968 sprintf(setting, "SOURCE=%s", source);
969 if (putenv(setting) != 0)
970 sysdie("cannot set SOURCE in the environment");
973 setting = xmalloc(strlen("BUILD=") + strlen(build) + 1);
974 sprintf(setting, "BUILD=%s", build);
975 if (putenv(setting) != 0)
976 sysdie("cannot set BUILD in the environment");
980 test_single(argv[0], source, build);
983 list = strrchr(argv[0], '/');
988 printf(banner, list);
989 exit(test_batch(argv[0], source, build) ? 0 : 1);