2 * Run a set of tests, reporting results.
6 * runtests [-b <build-dir>] [-s <source-dir>] <test-list>
7 * runtests -o [-b <build-dir>] [-s <source-dir>] <test>
9 * In the first case, expects a list of executables located in the given file,
10 * one line per executable. For each one, runs it as part of a test suite,
11 * reporting results. Test output should start with a line containing the
12 * number of tests (numbered from 1 to this number), optionally preceded by
13 * "1..", although that line may be given anywhere in the output. Each
14 * additional line should be in the following format:
19 * not ok <number> # todo
21 * where <number> is the number of the test. An optional comment is permitted
22 * after the number if preceded by whitespace. ok indicates success, not ok
23 * indicates failure. "# skip" and "# todo" are a special cases of a comment,
24 * and must start with exactly that formatting. They indicate the test was
25 * skipped for some reason (maybe because it doesn't apply to this platform)
26 * or is testing something known to currently fail. The text following either
27 * "# skip" or "# todo" and whitespace is the reason.
29 * As a special case, the first line of the output may be in the form:
31 * 1..0 # skip some reason
33 * which indicates that this entire test case should be skipped and gives a
36 * Any other lines are ignored, although for compliance with the TAP protocol
37 * all lines other than the ones in the above format should be sent to
38 * standard error rather than standard output and start with #.
40 * This is a subset of TAP as documented in Test::Harness::TAP or
41 * TAP::Parser::Grammar, which comes with Perl.
43 * If the -o option is given, instead run a single test and display all of its
44 * output. This is intended for use with failing tests so that the person
45 * running the test suite can get more details about what failed.
47 * If built with the C preprocessor symbols SOURCE and BUILD defined, C TAP
48 * Harness will export those values in the environment so that tests can find
49 * the source and build directory and will look for tests under both
50 * directories. These paths can also be set with the -b and -s command-line
51 * options, which will override anything set at build time.
53 * Any bug reports, bug fixes, and improvements are very much welcome and
54 * should be sent to the e-mail address below. This program is part of C TAP
55 * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
57 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,
58 * 2014 Russ Allbery <eagle@eyrie.org>
60 * Permission is hereby granted, free of charge, to any person obtaining a
61 * copy of this software and associated documentation files (the "Software"),
62 * to deal in the Software without restriction, including without limitation
63 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
64 * and/or sell copies of the Software, and to permit persons to whom the
65 * Software is furnished to do so, subject to the following conditions:
67 * The above copyright notice and this permission notice shall be included in
68 * all copies or substantial portions of the Software.
70 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
71 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
72 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
73 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
74 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
75 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
76 * DEALINGS IN THE SOFTWARE.
79 /* Required for fdopen(), getopt(), and putenv(). */
80 #if defined(__STRICT_ANSI__) || defined(PEDANTIC)
81 # ifndef _XOPEN_SOURCE
82 # define _XOPEN_SOURCE 500
97 #include <sys/types.h>
102 /* sys/time.h must be included before sys/resource.h on some platforms. */
103 #include <sys/resource.h>
105 /* AIX 6.1 (and possibly later) doesn't have WCOREDUMP. */
107 # define WCOREDUMP(status) ((unsigned)(status) & 0x80)
111 * POSIX requires that these be defined in <unistd.h>, but they're not always
112 * available. If one of them has been defined, all the rest almost certainly
116 # define STDIN_FILENO 0
117 # define STDOUT_FILENO 1
118 # define STDERR_FILENO 2
122 * Used for iterating through arrays. Returns the number of elements in the
123 * array (useful for a < upper bound in a for loop).
125 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
128 * The source and build versions of the tests directory. This is used to set
129 * the SOURCE and BUILD environment variables and find test programs, if set.
130 * Normally, this should be set as part of the build process to the test
131 * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
140 /* Test status codes. */
148 /* Indicates the state of our plan. */
150 PLAN_INIT, /* Nothing seen yet. */
151 PLAN_FIRST, /* Plan seen before any tests. */
152 PLAN_PENDING, /* Test seen and no plan yet. */
153 PLAN_FINAL /* Plan seen after some tests. */
156 /* Error exit statuses for test processes. */
157 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
158 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
159 #define CHILDERR_STDIN 102 /* Couldn't open stdin file. */
160 #define CHILDERR_STDERR 103 /* Couldn't open stderr file. */
162 /* Structure to hold data for a set of tests. */
164 char *file; /* The file name of the test. */
165 char *path; /* The path to the test program. */
166 enum plan_status plan; /* The status of our plan. */
167 unsigned long count; /* Expected count of tests. */
168 unsigned long current; /* The last seen test number. */
169 unsigned int length; /* The length of the last status message. */
170 unsigned long passed; /* Count of passing tests. */
171 unsigned long failed; /* Count of failing lists. */
172 unsigned long skipped; /* Count of skipped tests (passed). */
173 unsigned long allocated; /* The size of the results table. */
174 enum test_status *results; /* Table of results by test number. */
175 unsigned int aborted; /* Whether the set was aborted. */
176 int reported; /* Whether the results were reported. */
177 int status; /* The exit status of the test. */
178 unsigned int all_skipped; /* Whether all tests were skipped. */
179 char *reason; /* Why all tests were skipped. */
182 /* Structure to hold a linked list of test sets. */
185 struct testlist *next;
189 * Usage message. Should be used as a printf format with four arguments: the
190 * path to runtests, given three times, and the usage_description. This is
191 * split into variables to satisfy the pedantic ISO C90 limit on strings.
193 static const char usage_message[] = "\
194 Usage: %s [-b <build-dir>] [-s <source-dir>] <test> ...\n\
195 %s [-b <build-dir>] [-s <source-dir>] -l <test-list>\n\
196 %s -o [-b <build-dir>] [-s <source-dir>] <test>\n\
198 static const char usage_extra[] = "\
200 -b <build-dir> Set the build directory to <build-dir>\n\
201 -l <list> Take the list of tests to run from <test-list>\n\
202 -o Run a single test rather than a list of tests\n\
203 -s <source-dir> Set the source directory to <source-dir>\n\
205 runtests normally runs each test listed on the command line. With the -l\n\
206 option, it instead runs every test listed in a file. With the -o option,\n\
207 it instead runs a single test and shows its complete output.\n";
210 * Header used for test output. %s is replaced by the file name of the list
213 static const char banner[] = "\n\
214 Running all tests listed in %s. If any tests fail, run the failing\n\
215 test program with runtests -o to see more details.\n\n";
217 /* Header for reports of failed tests. */
218 static const char header[] = "\n\
219 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
220 -------------------------- -------------- ---- ---- ------------------------";
222 /* Include the file name and line number in malloc failures. */
223 #define xcalloc(n, size) x_calloc((n), (size), __FILE__, __LINE__)
224 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
225 #define xrealloc(p, size) x_realloc((p), (size), __FILE__, __LINE__)
226 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
229 * __attribute__ is available in gcc 2.5 and later, but only with gcc 2.7
230 * could you use the __format__ form of the attributes, which is what we use
231 * (to avoid confusion with other macros).
233 #ifndef __attribute__
234 # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7)
235 # define __attribute__(spec) /* empty */
240 * We use __alloc_size__, but it was only available in fairly recent versions
241 * of GCC. Suppress warnings about the unknown attribute if GCC is too old.
242 * We know that we're GCC at this point, so we can use the GCC variadic macro
243 * extension, which will still work with versions of GCC too old to have C99
244 * variadic macro support.
246 #if !defined(__attribute__) && !defined(__alloc_size__)
247 # if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 3)
248 # define __alloc_size__(spec, args...) /* empty */
253 * LLVM and Clang pretend to be GCC but don't support all of the __attribute__
254 * settings that GCC does. For them, suppress warnings about unknown
255 * attributes on declarations. This unfortunately will affect the entire
256 * compilation context, but there's no push and pop available.
258 #if !defined(__attribute__) && (defined(__llvm__) || defined(__clang__))
259 # pragma GCC diagnostic ignored "-Wattributes"
262 /* Declare internal functions that benefit from compiler attributes. */
263 static void sysdie(const char *, ...)
264 __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
265 static void *x_calloc(size_t, size_t, const char *, int)
266 __attribute__((__alloc_size__(1, 2), __malloc__, __nonnull__));
267 static void *x_malloc(size_t, const char *, int)
268 __attribute__((__alloc_size__(1), __malloc__, __nonnull__));
269 static void *x_realloc(void *, size_t, const char *, int)
270 __attribute__((__alloc_size__(2), __malloc__, __nonnull__(3)));
271 static char *x_strdup(const char *, const char *, int)
272 __attribute__((__malloc__, __nonnull__));
276 * Report a fatal error, including the results of strerror, and exit.
279 sysdie(const char *format, ...)
286 fprintf(stderr, "runtests: ");
287 va_start(args, format);
288 vfprintf(stderr, format, args);
290 fprintf(stderr, ": %s\n", strerror(oerrno));
296 * Allocate zeroed memory, reporting a fatal error and exiting on failure.
299 x_calloc(size_t n, size_t size, const char *file, int line)
304 size = (size > 0) ? size : 1;
307 sysdie("failed to calloc %lu bytes at %s line %d",
308 (unsigned long) size, file, line);
314 * Allocate memory, reporting a fatal error and exiting on failure.
317 x_malloc(size_t size, const char *file, int line)
323 sysdie("failed to malloc %lu bytes at %s line %d",
324 (unsigned long) size, file, line);
330 * Reallocate memory, reporting a fatal error and exiting on failure.
333 x_realloc(void *p, size_t size, const char *file, int line)
335 p = realloc(p, size);
337 sysdie("failed to realloc %lu bytes at %s line %d",
338 (unsigned long) size, file, line);
344 * Copy a string, reporting a fatal error and exiting on failure.
347 x_strdup(const char *s, const char *file, int line)
355 sysdie("failed to strdup %lu bytes at %s line %d",
356 (unsigned long) len, file, line);
363 * Given a struct timeval, return the number of seconds it represents as a
364 * double. Use difftime() to convert a time_t to a double.
367 tv_seconds(const struct timeval *tv)
369 return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
374 * Given two struct timevals, return the difference in seconds.
377 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
379 return tv_seconds(tv1) - tv_seconds(tv0);
384 * Given two struct timevals, return the sum in seconds as a double.
387 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
389 return tv_seconds(tv1) + tv_seconds(tv2);
394 * Given a pointer to a string, skip any leading whitespace and return a
395 * pointer to the first non-whitespace character.
398 skip_whitespace(const char *p)
400 while (isspace((unsigned char)(*p)))
407 * Start a program, connecting its stdout to a pipe on our end and its stderr
408 * to /dev/null, and storing the file descriptor to read from in the two
409 * argument. Returns the PID of the new process. Errors are fatal.
412 test_start(const char *path, int *fd)
414 int fds[2], infd, errfd;
417 /* Create a pipe used to capture the output from the test program. */
418 if (pipe(fds) == -1) {
421 sysdie("can't create pipe");
424 /* Fork a child process, massage the file descriptors, and exec. */
430 sysdie("can't fork");
432 /* In the child. Set up our standard output. */
435 close(STDOUT_FILENO);
436 if (dup2(fds[1], STDOUT_FILENO) < 0)
440 /* Point standard input at /dev/null. */
442 infd = open("/dev/null", O_RDONLY);
444 _exit(CHILDERR_STDIN);
445 if (infd != STDIN_FILENO) {
446 if (dup2(infd, STDIN_FILENO) < 0)
451 /* Point standard error at /dev/null. */
452 close(STDERR_FILENO);
453 errfd = open("/dev/null", O_WRONLY);
455 _exit(CHILDERR_STDERR);
456 if (errfd != STDERR_FILENO) {
457 if (dup2(errfd, STDERR_FILENO) < 0)
462 /* Now, exec our process. */
463 if (execl(path, path, (char *) 0) == -1)
464 _exit(CHILDERR_EXEC);
466 /* In parent. Close the extra file descriptor. */
477 * Back up over the output saying what test we were executing.
480 test_backspace(struct testset *ts)
484 if (!isatty(STDOUT_FILENO))
486 for (i = 0; i < ts->length; i++)
488 for (i = 0; i < ts->length; i++)
490 for (i = 0; i < ts->length; i++)
497 * Read the plan line of test output, which should contain the range of test
498 * numbers. We may initialize the testset structure here if we haven't yet
499 * seen a test. Return true if initialization succeeded and the test should
500 * continue, false otherwise.
503 test_plan(const char *line, struct testset *ts)
509 * Accept a plan without the leading 1.. for compatibility with older
510 * versions of runtests. This will only be allowed if we've not yet seen
513 line = skip_whitespace(line);
514 if (strncmp(line, "1..", 3) == 0)
518 * Get the count, check it for validity, and initialize the struct. If we
519 * have something of the form "1..0 # skip foo", the whole file was
520 * skipped; record that. If we do skip the whole file, zero out all of
521 * our statistics, since they're no longer relevant. strtol is called
522 * with a second argument to advance the line pointer past the count to
523 * make it simpler to detect the # skip case.
525 n = strtol(line, (char **) &line, 10);
527 line = skip_whitespace(line);
529 line = skip_whitespace(line + 1);
530 if (strncasecmp(line, "skip", 4) == 0) {
531 line = skip_whitespace(line + 4);
533 ts->reason = xstrdup(line);
534 ts->reason[strlen(ts->reason) - 1] = '\0';
547 puts("ABORTED (invalid test count)");
552 if (ts->plan == PLAN_INIT && ts->allocated == 0) {
555 ts->plan = PLAN_FIRST;
556 ts->results = xmalloc(ts->count * sizeof(enum test_status));
557 for (i = 0; i < ts->count; i++)
558 ts->results[i] = TEST_INVALID;
559 } else if (ts->plan == PLAN_PENDING) {
560 if ((unsigned long) n < ts->count) {
562 printf("ABORTED (invalid test number %lu)\n", ts->count);
568 if ((unsigned long) n > ts->allocated) {
569 ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
570 for (i = ts->allocated; i < ts->count; i++)
571 ts->results[i] = TEST_INVALID;
574 ts->plan = PLAN_FINAL;
581 * Given a single line of output from a test, parse it and return the success
582 * status of that test. Anything printed to stdout not matching the form
583 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
587 test_checkline(const char *line, struct testset *ts)
589 enum test_status status = TEST_PASS;
593 unsigned long i, current;
596 /* Before anything, check for a test abort. */
597 bail = strstr(line, "Bail out!");
599 bail = skip_whitespace(bail + strlen("Bail out!"));
603 length = strlen(bail);
604 if (bail[length - 1] == '\n')
607 printf("ABORTED (%.*s)\n", (int) length, bail);
615 * If the given line isn't newline-terminated, it was too big for an
616 * fgets(), which means ignore it.
618 if (line[strlen(line) - 1] != '\n')
621 /* If the line begins with a hash mark, ignore it. */
625 /* If we haven't yet seen a plan, look for one. */
626 if (ts->plan == PLAN_INIT && isdigit((unsigned char)(*line))) {
627 if (!test_plan(line, ts))
629 } else if (strncmp(line, "1..", 3) == 0) {
630 if (ts->plan == PLAN_PENDING) {
631 if (!test_plan(line, ts))
635 puts("ABORTED (multiple plans)");
642 /* Parse the line, ignoring something we can't parse. */
643 if (strncmp(line, "not ", 4) == 0) {
647 if (strncmp(line, "ok", 2) != 0)
649 line = skip_whitespace(line + 2);
651 number = strtol(line, &end, 10);
652 if (errno != 0 || end == line)
653 number = ts->current + 1;
655 if (number <= 0 || (current > ts->count && ts->plan == PLAN_FIRST)) {
657 printf("ABORTED (invalid test number %lu)\n", current);
663 /* We have a valid test result. Tweak the results array if needed. */
664 if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) {
665 ts->plan = PLAN_PENDING;
666 if (current > ts->count)
668 if (current > ts->allocated) {
671 n = (ts->allocated == 0) ? 32 : ts->allocated * 2;
674 ts->results = xrealloc(ts->results, n * sizeof(enum test_status));
675 for (i = ts->allocated; i < n; i++)
676 ts->results[i] = TEST_INVALID;
682 * Handle directives. We should probably do something more interesting
683 * with unexpected passes of todo tests.
685 while (isdigit((unsigned char)(*line)))
687 line = skip_whitespace(line);
689 line = skip_whitespace(line + 1);
690 if (strncasecmp(line, "skip", 4) == 0)
692 if (strncasecmp(line, "todo", 4) == 0)
693 status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
696 /* Make sure that the test number is in range and not a duplicate. */
697 if (ts->results[current - 1] != TEST_INVALID) {
699 printf("ABORTED (duplicate test number %lu)\n", current);
705 /* Good results. Increment our various counters. */
707 case TEST_PASS: ts->passed++; break;
708 case TEST_FAIL: ts->failed++; break;
709 case TEST_SKIP: ts->skipped++; break;
710 case TEST_INVALID: break;
712 ts->current = current;
713 ts->results[current - 1] = status;
714 if (isatty(STDOUT_FILENO)) {
716 if (ts->plan == PLAN_PENDING)
717 outlen = printf("%lu/?", current);
719 outlen = printf("%lu/%lu", current, ts->count);
720 ts->length = (outlen >= 0) ? outlen : 0;
727 * Print out a range of test numbers, returning the number of characters it
728 * took up. Takes the first number, the last number, the number of characters
729 * already printed on the line, and the limit of number of characters the line
730 * can hold. Add a comma and a space before the range if chars indicates that
731 * something has already been printed on the line, and print ... instead if
732 * chars plus the space needed would go over the limit (use a limit of 0 to
736 test_print_range(unsigned long first, unsigned long last, unsigned int chars,
739 unsigned int needed = 0;
742 for (n = first; n > 0; n /= 10)
745 for (n = last; n > 0; n /= 10)
751 if (limit > 0 && chars + needed > limit) {
753 if (chars <= limit) {
765 printf("%lu-", first);
773 * Summarize a single test set. The second argument is 0 if the set exited
774 * cleanly, a positive integer representing the exit status if it exited
775 * with a non-zero status, and a negative integer representing the signal
776 * that terminated it if it was killed by a signal.
779 test_summarize(struct testset *ts, int status)
782 unsigned long missing = 0;
783 unsigned long failed = 0;
784 unsigned long first = 0;
785 unsigned long last = 0;
788 fputs("ABORTED", stdout);
790 printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped);
792 for (i = 0; i < ts->count; i++) {
793 if (ts->results[i] == TEST_INVALID) {
795 fputs("MISSED ", stdout);
796 if (first && i == last)
800 test_print_range(first, last, missing - 1, 0);
808 test_print_range(first, last, missing - 1, 0);
811 for (i = 0; i < ts->count; i++) {
812 if (ts->results[i] == TEST_FAIL) {
813 if (missing && !failed)
816 fputs("FAILED ", stdout);
817 if (first && i == last)
821 test_print_range(first, last, failed - 1, 0);
829 test_print_range(first, last, failed - 1, 0);
830 if (!missing && !failed) {
831 fputs(!status ? "ok" : "dubious", stdout);
832 if (ts->skipped > 0) {
833 if (ts->skipped == 1)
834 printf(" (skipped %lu test)", ts->skipped);
836 printf(" (skipped %lu tests)", ts->skipped);
841 printf(" (exit status %d)", status);
843 printf(" (killed by signal %d%s)", -status,
844 WCOREDUMP(ts->status) ? ", core dumped" : "");
850 * Given a test set, analyze the results, classify the exit status, handle a
851 * few special error messages, and then pass it along to test_summarize() for
852 * the regular output. Returns true if the test set ran successfully and all
853 * tests passed or were skipped, false otherwise.
856 test_analyze(struct testset *ts)
860 if (ts->all_skipped) {
861 if (ts->reason == NULL)
864 printf("skipped (%s)\n", ts->reason);
866 } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
867 switch (WEXITSTATUS(ts->status)) {
870 puts("ABORTED (can't dup file descriptors)");
874 puts("ABORTED (execution failed -- not found?)");
877 case CHILDERR_STDERR:
879 puts("ABORTED (can't open /dev/null)");
882 test_summarize(ts, WEXITSTATUS(ts->status));
886 } else if (WIFSIGNALED(ts->status)) {
887 test_summarize(ts, -WTERMSIG(ts->status));
889 } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) {
890 puts("ABORTED (no valid test plan)");
894 test_summarize(ts, 0);
895 return (ts->failed == 0);
901 * Runs a single test set, accumulating and then reporting the results.
902 * Returns true if the test set was successfully run and all tests passed,
906 test_run(struct testset *ts)
908 pid_t testpid, child;
914 /* Run the test program. */
915 testpid = test_start(ts->path, &outfd);
916 output = fdopen(outfd, "r");
920 sysdie("fdopen failed");
923 /* Pass each line of output to test_checkline(). */
924 while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
925 test_checkline(buffer, ts);
926 if (ferror(output) || ts->plan == PLAN_INIT)
931 * Consume the rest of the test output, close the output descriptor,
932 * retrieve the exit status, and pass that information to test_analyze()
933 * for eventual output.
935 while (fgets(buffer, sizeof(buffer), output))
938 child = waitpid(testpid, &ts->status, 0);
939 if (child == (pid_t) -1) {
944 sysdie("waitpid for %u failed", (unsigned int) testpid);
948 status = test_analyze(ts);
950 /* Convert missing tests to failed tests. */
951 for (i = 0; i < ts->count; i++) {
952 if (ts->results[i] == TEST_INVALID) {
954 ts->results[i] = TEST_FAIL;
962 /* Summarize a list of test failures. */
964 test_fail_summary(const struct testlist *fails)
968 unsigned long i, first, last, total;
972 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
973 -------------------------- -------------- ---- ---- -------------- */
974 for (; fails; fails = fails->next) {
976 total = ts->count - ts->skipped;
977 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
978 total, total ? (ts->failed * 100.0) / total : 0,
980 if (WIFEXITED(ts->status))
981 printf("%4d ", WEXITSTATUS(ts->status));
991 for (i = 0; i < ts->count; i++) {
992 if (ts->results[i] == TEST_FAIL) {
993 if (first != 0 && i == last)
997 chars += test_print_range(first, last, chars, 19);
1004 test_print_range(first, last, chars, 19);
1011 * Check whether a given file path is a valid test. Currently, this checks
1012 * whether it is executable and is a regular file. Returns true or false.
1015 is_valid_test(const char *path)
1019 if (access(path, X_OK) < 0)
1021 if (stat(path, &st) < 0)
1023 if (!S_ISREG(st.st_mode))
1030 * Given the name of a test, a pointer to the testset struct, and the source
1031 * and build directories, find the test. We try first relative to the current
1032 * directory, then in the build directory (if not NULL), then in the source
1033 * directory. In each of those directories, we first try a "-t" extension and
1034 * then a ".t" extension. When we find an executable program, we return the
1035 * path to that program. If none of those paths are executable, just fill in
1036 * the name of the test as is.
1038 * The caller is responsible for freeing the path member of the testset
1042 find_test(const char *name, const char *source, const char *build)
1045 const char *bases[3], *suffix, *base;
1047 const char *suffixes[3] = { "-t", ".t", "" };
1049 /* Possible base directories. */
1054 /* Try each suffix with each base. */
1055 for (i = 0; i < ARRAY_SIZE(suffixes); i++) {
1056 suffix = suffixes[i];
1057 for (j = 0; j < ARRAY_SIZE(bases); j++) {
1061 path = xmalloc(strlen(base) + strlen(name) + strlen(suffix) + 2);
1062 sprintf(path, "%s/%s%s", base, name, suffix);
1063 if (is_valid_test(path))
1070 path = xstrdup(name);
1076 * Read a list of tests from a file, returning the list of tests as a struct
1077 * testlist. Reports an error to standard error and exits if the list of
1078 * tests cannot be read.
1080 static struct testlist *
1081 read_test_list(const char *filename)
1086 char buffer[BUFSIZ];
1087 struct testlist *listhead, *current;
1089 /* Create the initial container list that will hold our results. */
1090 listhead = xmalloc(sizeof(struct testlist));
1091 listhead->ts = NULL;
1092 listhead->next = NULL;
1096 * Open our file of tests to run and read it line by line, creating a new
1097 * struct testlist and struct testset for each line.
1099 file = fopen(filename, "r");
1101 sysdie("can't open %s", filename);
1103 while (fgets(buffer, sizeof(buffer), file)) {
1105 length = strlen(buffer) - 1;
1106 if (buffer[length] != '\n') {
1107 fprintf(stderr, "%s:%u: line too long\n", filename, line);
1110 buffer[length] = '\0';
1111 if (current == NULL)
1114 current->next = xmalloc(sizeof(struct testlist));
1115 current = current->next;
1116 current->next = NULL;
1118 current->ts = xcalloc(1, sizeof(struct testset));
1119 current->ts->plan = PLAN_INIT;
1120 current->ts->file = xstrdup(buffer);
1121 current->ts->reason = NULL;
1125 /* Return the results. */
1131 * Build a list of tests from command line arguments. Takes the argv and argc
1132 * representing the command line arguments and returns a newly allocated test
1133 * list. The caller is responsible for freeing.
1135 static struct testlist *
1136 build_test_list(char *argv[], int argc)
1139 struct testlist *listhead, *current;
1141 /* Create the initial container list that will hold our results. */
1142 listhead = xmalloc(sizeof(struct testlist));
1143 listhead->ts = NULL;
1144 listhead->next = NULL;
1147 /* Walk the list of arguments and create test sets for them. */
1148 for (i = 0; i < argc; i++) {
1149 if (current == NULL)
1152 current->next = xmalloc(sizeof(struct testlist));
1153 current = current->next;
1154 current->next = NULL;
1156 current->ts = xcalloc(1, sizeof(struct testset));
1157 current->ts->plan = PLAN_INIT;
1158 current->ts->file = xstrdup(argv[i]);
1159 current->ts->reason = NULL;
1162 /* Return the results. */
1167 /* Free a struct testset. */
1169 free_testset(struct testset *ts)
1180 * Run a batch of tests. Takes two additional parameters: the root of the
1181 * source directory and the root of the build directory. Test programs will
1182 * be first searched for in the current directory, then the build directory,
1183 * then the source directory. Returns true iff all tests passed, and always
1184 * frees the test list that's passed in.
1187 test_batch(struct testlist *tests, const char *source, const char *build)
1191 unsigned int longest = 0;
1192 unsigned int count = 0;
1194 struct timeval start, end;
1195 struct rusage stats;
1196 struct testlist *failhead = NULL;
1197 struct testlist *failtail = NULL;
1198 struct testlist *current, *next;
1200 unsigned long total = 0;
1201 unsigned long passed = 0;
1202 unsigned long skipped = 0;
1203 unsigned long failed = 0;
1204 unsigned long aborted = 0;
1206 /* Walk the list of tests to find the longest name. */
1207 for (current = tests; current != NULL; current = current->next) {
1208 length = strlen(current->ts->file);
1209 if (length > longest)
1214 * Add two to longest and round up to the nearest tab stop. This is how
1215 * wide the column for printing the current test name will be.
1219 longest += 8 - (longest % 8);
1221 /* Start the wall clock timer. */
1222 gettimeofday(&start, NULL);
1224 /* Now, plow through our tests again, running each one. */
1225 for (current = tests; current != NULL; current = current->next) {
1228 /* Print out the name of the test file. */
1229 fputs(ts->file, stdout);
1230 for (i = strlen(ts->file); i < longest; i++)
1232 if (isatty(STDOUT_FILENO))
1236 ts->path = find_test(ts->file, source, build);
1237 succeeded = test_run(ts);
1240 /* Record cumulative statistics. */
1241 aborted += ts->aborted;
1242 total += ts->count + ts->all_skipped;
1243 passed += ts->passed;
1244 skipped += ts->skipped + ts->all_skipped;
1245 failed += ts->failed;
1248 /* If the test fails, we shuffle it over to the fail list. */
1250 if (failhead == NULL) {
1251 failhead = xmalloc(sizeof(struct testset));
1252 failtail = failhead;
1254 failtail->next = xmalloc(sizeof(struct testset));
1255 failtail = failtail->next;
1258 failtail->next = NULL;
1263 /* Stop the timer and get our child resource statistics. */
1264 gettimeofday(&end, NULL);
1265 getrusage(RUSAGE_CHILDREN, &stats);
1267 /* Summarize the failures and free the failure list. */
1268 if (failhead != NULL) {
1269 test_fail_summary(failhead);
1270 while (failhead != NULL) {
1271 next = failhead->next;
1277 /* Free the memory used by the test lists. */
1278 while (tests != NULL) {
1280 free_testset(tests->ts);
1285 /* Print out the final test summary. */
1289 printf("Aborted %lu test set", aborted);
1291 printf("Aborted %lu test sets", aborted);
1292 printf(", passed %lu/%lu tests", passed, total);
1294 else if (failed == 0)
1295 fputs("All tests successful", stdout);
1297 printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
1298 (total - failed) * 100.0 / total);
1301 printf(", %lu test skipped", skipped);
1303 printf(", %lu tests skipped", skipped);
1306 printf("Files=%u, Tests=%lu", count, total);
1307 printf(", %.2f seconds", tv_diff(&end, &start));
1308 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
1309 tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
1310 tv_sum(&stats.ru_utime, &stats.ru_stime));
1311 return (failed == 0 && aborted == 0);
1316 * Run a single test case. This involves just running the test program after
1317 * having done the environment setup and finding the test program.
1320 test_single(const char *program, const char *source, const char *build)
1324 path = find_test(program, source, build);
1325 if (execl(path, path, (char *) 0) == -1)
1326 sysdie("cannot exec %s", path);
1331 * Main routine. Set the SOURCE and BUILD environment variables and then,
1332 * given a file listing tests, run each test listed.
1335 main(int argc, char *argv[])
1340 char *source_env = NULL;
1341 char *build_env = NULL;
1342 const char *shortlist;
1343 const char *list = NULL;
1344 const char *source = SOURCE;
1345 const char *build = BUILD;
1346 struct testlist *tests;
1348 while ((option = getopt(argc, argv, "b:hl:os:")) != EOF) {
1354 printf(usage_message, argv[0], argv[0], argv[0], usage_extra);
1372 if ((list == NULL && argc < 1) || (list != NULL && argc > 0)) {
1373 fprintf(stderr, usage_message, argv[0], argv[0], argv[0], usage_extra);
1377 /* Set SOURCE and BUILD environment variables. */
1378 if (source != NULL) {
1379 source_env = xmalloc(strlen("SOURCE=") + strlen(source) + 1);
1380 sprintf(source_env, "SOURCE=%s", source);
1381 if (putenv(source_env) != 0)
1382 sysdie("cannot set SOURCE in the environment");
1384 if (build != NULL) {
1385 build_env = xmalloc(strlen("BUILD=") + strlen(build) + 1);
1386 sprintf(build_env, "BUILD=%s", build);
1387 if (putenv(build_env) != 0)
1388 sysdie("cannot set BUILD in the environment");
1391 /* Run the tests as instructed. */
1393 test_single(argv[0], source, build);
1394 else if (list != NULL) {
1395 shortlist = strrchr(list, '/');
1396 if (shortlist == NULL)
1400 printf(banner, shortlist);
1401 tests = read_test_list(list);
1402 status = test_batch(tests, source, build) ? 0 : 1;
1404 tests = build_test_list(argv, argc);
1405 status = test_batch(tests, source, build) ? 0 : 1;
1408 /* For valgrind cleanliness, free all our memory. */
1409 if (source_env != NULL) {
1410 putenv((char *) "SOURCE=");
1413 if (build_env != NULL) {
1414 putenv((char *) "BUILD=");