2 * Run a set of tests, reporting results.
4 * Test suite driver that runs a set of tests implementing a subset of the
5 * Test Anything Protocol (TAP) and reports the results.
7 * Any bug reports, bug fixes, and improvements are very much welcome and
8 * should be sent to the e-mail address below. This program is part of C TAP
9 * Harness <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
11 * Copyright 2000-2001, 2004, 2006-2019 Russ Allbery <eagle@eyrie.org>
13 * Permission is hereby granted, free of charge, to any person obtaining a
14 * copy of this software and associated documentation files (the "Software"),
15 * to deal in the Software without restriction, including without limitation
16 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
17 * and/or sell copies of the Software, and to permit persons to whom the
18 * Software is furnished to do so, subject to the following conditions:
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
26 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
29 * DEALINGS IN THE SOFTWARE.
31 * SPDX-License-Identifier: MIT
37 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>
38 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] <test> [<test> ...]
39 * runtests -o [-h] [-b <build-dir>] [-s <source-dir>] <test>
41 * In the first case, expects a list of executables located in the given file,
42 * one line per executable, possibly followed by a space-separated list of
43 * options. For each one, runs it as part of a test suite, reporting results.
44 * In the second case, use the same infrastructure, but run only the tests
45 * listed on the command line.
47 * Test output should start with a line containing the number of tests
48 * (numbered from 1 to this number), optionally preceded by "1..", although
49 * that line may be given anywhere in the output. Each additional line should
50 * be in the following format:
55 * not ok <number> # todo
57 * where <number> is the number of the test. An optional comment is permitted
58 * after the number if preceded by whitespace. ok indicates success, not ok
59 * indicates failure. "# skip" and "# todo" are a special cases of a comment,
60 * and must start with exactly that formatting. They indicate the test was
61 * skipped for some reason (maybe because it doesn't apply to this platform)
62 * or is testing something known to currently fail. The text following either
63 * "# skip" or "# todo" and whitespace is the reason.
65 * As a special case, the first line of the output may be in the form:
67 * 1..0 # skip some reason
69 * which indicates that this entire test case should be skipped and gives a
72 * Any other lines are ignored, although for compliance with the TAP protocol
73 * all lines other than the ones in the above format should be sent to
74 * standard error rather than standard output and start with #.
76 * This is a subset of TAP as documented in Test::Harness::TAP or
77 * TAP::Parser::Grammar, which comes with Perl.
79 * If the -o option is given, instead run a single test and display all of its
80 * output. This is intended for use with failing tests so that the person
81 * running the test suite can get more details about what failed.
83 * If built with the C preprocessor symbols C_TAP_SOURCE and C_TAP_BUILD
84 * defined, C TAP Harness will export those values in the environment so that
85 * tests can find the source and build directory and will look for tests under
86 * both directories. These paths can also be set with the -b and -s
87 * command-line options, which will override anything set at build time.
89 * If the -v option is given, or the C_TAP_VERBOSE environment variable is set,
90 * display the full output of each test as it runs rather than showing a
91 * summary of the results of each test.
94 /* Required for fdopen(), getopt(), and putenv(). */
95 #if defined(__STRICT_ANSI__) || defined(PEDANTIC)
96 # ifndef _XOPEN_SOURCE
97 # define _XOPEN_SOURCE 500
111 #include <sys/stat.h>
112 #include <sys/time.h>
113 #include <sys/types.h>
114 #include <sys/wait.h>
118 /* sys/time.h must be included before sys/resource.h on some platforms. */
119 #include <sys/resource.h>
121 /* AIX 6.1 (and possibly later) doesn't have WCOREDUMP. */
123 # define WCOREDUMP(status) ((unsigned) (status) &0x80)
127 * POSIX requires that these be defined in <unistd.h>, but they're not always
128 * available. If one of them has been defined, all the rest almost certainly
132 # define STDIN_FILENO 0
133 # define STDOUT_FILENO 1
134 # define STDERR_FILENO 2
138 * Used for iterating through arrays. Returns the number of elements in the
139 * array (useful for a < upper bound in a for loop).
141 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
144 * The source and build versions of the tests directory. This is used to set
145 * the C_TAP_SOURCE and C_TAP_BUILD environment variables (and the SOURCE and
146 * BUILD environment variables set for backward compatibility) and find test
147 * programs, if set. Normally, this should be set as part of the build
148 * process to the test subdirectories of $(abs_top_srcdir) and
149 * $(abs_top_builddir) respectively.
152 # define C_TAP_SOURCE NULL
155 # define C_TAP_BUILD NULL
158 /* Test status codes. */
159 enum test_status { TEST_FAIL, TEST_PASS, TEST_SKIP, TEST_INVALID };
161 /* Really, just a boolean, but this is more self-documenting. */
162 enum test_verbose { CONCISE = 0, VERBOSE = 1 };
164 /* Indicates the state of our plan. */
166 PLAN_INIT, /* Nothing seen yet. */
167 PLAN_FIRST, /* Plan seen before any tests. */
168 PLAN_PENDING, /* Test seen and no plan yet. */
169 PLAN_FINAL /* Plan seen after some tests. */
172 /* Error exit statuses for test processes. */
173 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
174 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
175 #define CHILDERR_STDIN 102 /* Couldn't open stdin file. */
176 #define CHILDERR_STDERR 103 /* Couldn't open stderr file. */
178 /* Structure to hold data for a set of tests. */
180 char *file; /* The file name of the test. */
181 char **command; /* The argv vector to run the command. */
182 enum plan_status plan; /* The status of our plan. */
183 unsigned long count; /* Expected count of tests. */
184 unsigned long current; /* The last seen test number. */
185 unsigned int length; /* The length of the last status message. */
186 unsigned long passed; /* Count of passing tests. */
187 unsigned long failed; /* Count of failing lists. */
188 unsigned long skipped; /* Count of skipped tests (passed). */
189 unsigned long allocated; /* The size of the results table. */
190 enum test_status *results; /* Table of results by test number. */
191 unsigned int aborted; /* Whether the set was aborted. */
192 unsigned int reported; /* Whether the results were reported. */
193 int status; /* The exit status of the test. */
194 unsigned int all_skipped; /* Whether all tests were skipped. */
195 char *reason; /* Why all tests were skipped. */
198 /* Structure to hold a linked list of test sets. */
201 struct testlist *next;
205 * Usage message. Should be used as a printf format with four arguments: the
206 * path to runtests, given three times, and the usage_description. This is
207 * split into variables to satisfy the pedantic ISO C90 limit on strings.
209 static const char usage_message[] = "\
210 Usage: %s [-hv] [-b <build-dir>] [-s <source-dir>] <test> ...\n\
211 %s [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>\n\
212 %s -o [-h] [-b <build-dir>] [-s <source-dir>] <test>\n\
215 -b <build-dir> Set the build directory to <build-dir>\n\
217 static const char usage_extra[] = "\
218 -l <list> Take the list of tests to run from <test-list>\n\
219 -o Run a single test rather than a list of tests\n\
220 -s <source-dir> Set the source directory to <source-dir>\n\
221 -v Show the full output of each test\n\
223 runtests normally runs each test listed on the command line. With the -l\n\
224 option, it instead runs every test listed in a file. With the -o option,\n\
225 it instead runs a single test and shows its complete output.\n";
228 * Header used for test output. %s is replaced by the file name of the list
231 static const char banner[] = "\n\
232 Running all tests listed in %s. If any tests fail, run the failing\n\
233 test program with runtests -o to see more details.\n\n";
235 /* Header for reports of failed tests. */
236 static const char header[] = "\n\
237 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
238 -------------------------- -------------- ---- ---- ------------------------";
240 /* Include the file name and line number in malloc failures. */
241 #define xcalloc(n, type) \
242 ((type *) x_calloc((n), sizeof(type), __FILE__, __LINE__))
243 #define xmalloc(size) ((char *) x_malloc((size), __FILE__, __LINE__))
244 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
245 #define xstrndup(p, size) x_strndup((p), (size), __FILE__, __LINE__)
246 #define xreallocarray(p, n, type) \
247 ((type *) x_reallocarray((p), (n), sizeof(type), __FILE__, __LINE__))
250 * __attribute__ is available in gcc 2.5 and later, but only with gcc 2.7
251 * could you use the __format__ form of the attributes, which is what we use
252 * (to avoid confusion with other macros).
254 #ifndef __attribute__
255 # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7)
256 # define __attribute__(spec) /* empty */
261 * We use __alloc_size__, but it was only available in fairly recent versions
262 * of GCC. Suppress warnings about the unknown attribute if GCC is too old.
263 * We know that we're GCC at this point, so we can use the GCC variadic macro
264 * extension, which will still work with versions of GCC too old to have C99
265 * variadic macro support.
267 #if !defined(__attribute__) && !defined(__alloc_size__)
268 # if defined(__GNUC__) && !defined(__clang__)
269 # if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 3)
270 # define __alloc_size__(spec, args...) /* empty */
276 * LLVM and Clang pretend to be GCC but don't support all of the __attribute__
277 * settings that GCC does. For them, suppress warnings about unknown
278 * attributes on declarations. This unfortunately will affect the entire
279 * compilation context, but there's no push and pop available.
281 #if !defined(__attribute__) && (defined(__llvm__) || defined(__clang__))
282 # pragma GCC diagnostic ignored "-Wattributes"
285 /* Declare internal functions that benefit from compiler attributes. */
286 static void die(const char *, ...)
287 __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
288 static void sysdie(const char *, ...)
289 __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
290 static void *x_calloc(size_t, size_t, const char *, int)
291 __attribute__((__alloc_size__(1, 2), __malloc__, __nonnull__));
292 static void *x_malloc(size_t, const char *, int)
293 __attribute__((__alloc_size__(1), __malloc__, __nonnull__));
294 static void *x_reallocarray(void *, size_t, size_t, const char *, int)
295 __attribute__((__alloc_size__(2, 3), __malloc__, __nonnull__(4)));
296 static char *x_strdup(const char *, const char *, int)
297 __attribute__((__malloc__, __nonnull__));
298 static char *x_strndup(const char *, size_t, const char *, int)
299 __attribute__((__malloc__, __nonnull__));
303 * Report a fatal error and exit.
306 die(const char *format, ...)
311 fprintf(stderr, "runtests: ");
312 va_start(args, format);
313 vfprintf(stderr, format, args);
315 fprintf(stderr, "\n");
321 * Report a fatal error, including the results of strerror, and exit.
324 sysdie(const char *format, ...)
331 fprintf(stderr, "runtests: ");
332 va_start(args, format);
333 vfprintf(stderr, format, args);
335 fprintf(stderr, ": %s\n", strerror(oerrno));
341 * Allocate zeroed memory, reporting a fatal error and exiting on failure.
344 x_calloc(size_t n, size_t size, const char *file, int line)
349 size = (size > 0) ? size : 1;
352 sysdie("failed to calloc %lu bytes at %s line %d",
353 (unsigned long) size, file, line);
359 * Allocate memory, reporting a fatal error and exiting on failure.
362 x_malloc(size_t size, const char *file, int line)
368 sysdie("failed to malloc %lu bytes at %s line %d",
369 (unsigned long) size, file, line);
375 * Reallocate memory, reporting a fatal error and exiting on failure.
377 * We should technically use SIZE_MAX here for the overflow check, but
378 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
379 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
380 * can assume that UINT_MAX <= SIZE_MAX. And we should not be allocating
381 * anything anywhere near that large.
383 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
384 * I disbelieve in the existence of such systems and they will have to cope
385 * without overflow checks.)
388 x_reallocarray(void *p, size_t n, size_t size, const char *file, int line)
391 size = (size > 0) ? size : 1;
393 if (n > 0 && UINT_MAX / n <= size)
394 sysdie("realloc too large at %s line %d", file, line);
395 p = realloc(p, n * size);
397 sysdie("failed to realloc %lu bytes at %s line %d",
398 (unsigned long) (n * size), file, line);
404 * Copy a string, reporting a fatal error and exiting on failure.
407 x_strdup(const char *s, const char *file, int line)
413 p = (char *) malloc(len);
415 sysdie("failed to strdup %lu bytes at %s line %d", (unsigned long) len,
423 * Copy the first n characters of a string, reporting a fatal error and
424 * existing on failure.
426 * Avoid using the system strndup function since it may not exist (on Mac OS
427 * X, for example), and there's no need to introduce another portability
431 x_strndup(const char *s, size_t size, const char *file, int line)
437 /* Don't assume that the source string is nul-terminated. */
438 for (p = s; (size_t)(p - s) < size && *p != '\0'; p++)
440 len = (size_t)(p - s);
441 copy = (char *) malloc(len + 1);
443 sysdie("failed to strndup %lu bytes at %s line %d",
444 (unsigned long) len, file, line);
445 memcpy(copy, s, len);
452 * Form a new string by concatenating multiple strings. The arguments must be
453 * terminated by (const char *) 0.
455 * This function only exists because we can't assume asprintf. We can't
456 * simulate asprintf with snprintf because we're only assuming SUSv3, which
457 * does not require that snprintf with a NULL buffer return the required
458 * length. When those constraints are relaxed, this should be ripped out and
459 * replaced with asprintf or a more trivial replacement with snprintf.
462 concat(const char *first, ...)
471 * Find the total memory required. Ensure we don't overflow length. We
472 * aren't guaranteed to have SIZE_MAX, so use UINT_MAX as an acceptable
473 * substitute (see the x_nrealloc comments).
475 va_start(args, first);
476 for (string = first; string != NULL; string = va_arg(args, const char *)) {
477 if (length >= UINT_MAX - strlen(string)) {
479 sysdie("strings too long in concat");
481 length += strlen(string);
486 /* Create the string. */
487 result = xmalloc(length);
488 va_start(args, first);
490 for (string = first; string != NULL; string = va_arg(args, const char *)) {
491 memcpy(result + offset, string, strlen(string));
492 offset += strlen(string);
495 result[offset] = '\0';
501 * Given a struct timeval, return the number of seconds it represents as a
502 * double. Use difftime() to convert a time_t to a double.
505 tv_seconds(const struct timeval *tv)
507 return difftime(tv->tv_sec, 0) + (double) tv->tv_usec * 1e-6;
512 * Given two struct timevals, return the difference in seconds.
515 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
517 return tv_seconds(tv1) - tv_seconds(tv0);
522 * Given two struct timevals, return the sum in seconds as a double.
525 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
527 return tv_seconds(tv1) + tv_seconds(tv2);
532 * Given a pointer to a string, skip any leading whitespace and return a
533 * pointer to the first non-whitespace character.
536 skip_whitespace(const char *p)
538 while (isspace((unsigned char) (*p)))
545 * Given a pointer to a string, skip any non-whitespace characters and return
546 * a pointer to the first whitespace character, or to the end of the string.
549 skip_non_whitespace(const char *p)
551 while (*p != '\0' && !isspace((unsigned char) (*p)))
558 * Start a program, connecting its stdout to a pipe on our end and its stderr
559 * to /dev/null, and storing the file descriptor to read from in the two
560 * argument. Returns the PID of the new process. Errors are fatal.
563 test_start(char *const *command, int *fd)
565 int fds[2], infd, errfd;
568 /* Create a pipe used to capture the output from the test program. */
569 if (pipe(fds) == -1) {
572 sysdie("can't create pipe");
575 /* Fork a child process, massage the file descriptors, and exec. */
581 sysdie("can't fork");
583 /* In the child. Set up our standard output. */
586 close(STDOUT_FILENO);
587 if (dup2(fds[1], STDOUT_FILENO) < 0)
591 /* Point standard input at /dev/null. */
593 infd = open("/dev/null", O_RDONLY);
595 _exit(CHILDERR_STDIN);
596 if (infd != STDIN_FILENO) {
597 if (dup2(infd, STDIN_FILENO) < 0)
602 /* Point standard error at /dev/null. */
603 close(STDERR_FILENO);
604 errfd = open("/dev/null", O_WRONLY);
606 _exit(CHILDERR_STDERR);
607 if (errfd != STDERR_FILENO) {
608 if (dup2(errfd, STDERR_FILENO) < 0)
613 /* Now, exec our process. */
614 if (execv(command[0], command) == -1)
615 _exit(CHILDERR_EXEC);
618 /* In parent. Close the extra file descriptor. */
629 * Back up over the output saying what test we were executing.
632 test_backspace(struct testset *ts)
636 if (!isatty(STDOUT_FILENO))
638 for (i = 0; i < ts->length; i++)
640 for (i = 0; i < ts->length; i++)
642 for (i = 0; i < ts->length; i++)
649 * Allocate or resize the array of test results to be large enough to contain
650 * the test number in.
653 resize_results(struct testset *ts, unsigned long n)
658 /* If there's already enough space, return quickly. */
659 if (n <= ts->allocated)
663 * If no space has been allocated, do the initial allocation. Otherwise,
664 * resize. Start with 32 test cases and then add 1024 with each resize to
665 * try to reduce the number of reallocations.
667 if (ts->allocated == 0) {
668 s = (n > 32) ? n : 32;
669 ts->results = xcalloc(s, enum test_status);
671 s = (n > ts->allocated + 1024) ? n : ts->allocated + 1024;
672 ts->results = xreallocarray(ts->results, s, enum test_status);
675 /* Set the results for the newly-allocated test array. */
676 for (i = ts->allocated; i < s; i++)
677 ts->results[i] = TEST_INVALID;
683 * Report an invalid test number and set the appropriate flags. Pulled into a
684 * separate function since we do this in several places.
687 invalid_test_number(struct testset *ts, long n, enum test_verbose verbose)
691 printf("ABORTED (invalid test number %ld)\n", n);
698 * Read the plan line of test output, which should contain the range of test
699 * numbers. We may initialize the testset structure here if we haven't yet
700 * seen a test. Return true if initialization succeeded and the test should
701 * continue, false otherwise.
704 test_plan(const char *line, struct testset *ts, enum test_verbose verbose)
709 * Accept a plan without the leading 1.. for compatibility with older
710 * versions of runtests. This will only be allowed if we've not yet seen
713 line = skip_whitespace(line);
714 if (strncmp(line, "1..", 3) == 0)
718 * Get the count and check it for validity.
720 * If we have something of the form "1..0 # skip foo", the whole file was
721 * skipped; record that. If we do skip the whole file, zero out all of
722 * our statistics, since they're no longer relevant.
724 * strtol is called with a second argument to advance the line pointer
725 * past the count to make it simpler to detect the # skip case.
727 n = strtol(line, (char **) &line, 10);
729 line = skip_whitespace(line);
731 line = skip_whitespace(line + 1);
732 if (strncasecmp(line, "skip", 4) == 0) {
733 line = skip_whitespace(line + 4);
735 ts->reason = xstrdup(line);
736 ts->reason[strlen(ts->reason) - 1] = '\0';
749 puts("ABORTED (invalid test count)");
756 * If we are doing lazy planning, check the plan against the largest test
757 * number that we saw and fail now if we saw a check outside the plan
760 if (ts->plan == PLAN_PENDING && (unsigned long) n < ts->count) {
761 invalid_test_number(ts, (long) ts->count, verbose);
766 * Otherwise, allocated or resize the results if needed and update count,
767 * and then record that we've seen a plan.
769 resize_results(ts, (unsigned long) n);
770 ts->count = (unsigned long) n;
771 if (ts->plan == PLAN_INIT)
772 ts->plan = PLAN_FIRST;
773 else if (ts->plan == PLAN_PENDING)
774 ts->plan = PLAN_FINAL;
780 * Given a single line of output from a test, parse it and return the success
781 * status of that test. Anything printed to stdout not matching the form
782 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
786 test_checkline(const char *line, struct testset *ts, enum test_verbose verbose)
788 enum test_status status = TEST_PASS;
792 unsigned long current;
795 /* Before anything, check for a test abort. */
796 bail = strstr(line, "Bail out!");
798 bail = skip_whitespace(bail + strlen("Bail out!"));
802 length = strlen(bail);
803 if (bail[length - 1] == '\n')
807 printf("ABORTED (%.*s)\n", (int) length, bail);
815 * If the given line isn't newline-terminated, it was too big for an
816 * fgets(), which means ignore it.
818 if (line[strlen(line) - 1] != '\n')
821 /* If the line begins with a hash mark, ignore it. */
825 /* If we haven't yet seen a plan, look for one. */
826 if (ts->plan == PLAN_INIT && isdigit((unsigned char) (*line))) {
827 if (!test_plan(line, ts, verbose))
829 } else if (strncmp(line, "1..", 3) == 0) {
830 if (ts->plan == PLAN_PENDING) {
831 if (!test_plan(line, ts, verbose))
836 puts("ABORTED (multiple plans)");
843 /* Parse the line, ignoring something we can't parse. */
844 if (strncmp(line, "not ", 4) == 0) {
848 if (strncmp(line, "ok", 2) != 0)
850 line = skip_whitespace(line + 2);
852 number = strtol(line, &end, 10);
853 if (errno != 0 || end == line)
854 current = ts->current + 1;
855 else if (number <= 0) {
856 invalid_test_number(ts, number, verbose);
859 current = (unsigned long) number;
860 if (current > ts->count && ts->plan == PLAN_FIRST) {
861 invalid_test_number(ts, (long) current, verbose);
865 /* We have a valid test result. Tweak the results array if needed. */
866 if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) {
867 ts->plan = PLAN_PENDING;
868 resize_results(ts, current);
869 if (current > ts->count)
874 * Handle directives. We should probably do something more interesting
875 * with unexpected passes of todo tests.
877 while (isdigit((unsigned char) (*line)))
879 line = skip_whitespace(line);
881 line = skip_whitespace(line + 1);
882 if (strncasecmp(line, "skip", 4) == 0)
884 if (strncasecmp(line, "todo", 4) == 0)
885 status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
888 /* Make sure that the test number is in range and not a duplicate. */
889 if (ts->results[current - 1] != TEST_INVALID) {
892 printf("ABORTED (duplicate test number %lu)\n", current);
898 /* Good results. Increment our various counters. */
912 ts->current = current;
913 ts->results[current - 1] = status;
914 if (!verbose && isatty(STDOUT_FILENO)) {
916 if (ts->plan == PLAN_PENDING)
917 outlen = printf("%lu/?", current);
919 outlen = printf("%lu/%lu", current, ts->count);
920 ts->length = (outlen >= 0) ? (unsigned int) outlen : 0;
927 * Print out a range of test numbers, returning the number of characters it
928 * took up. Takes the first number, the last number, the number of characters
929 * already printed on the line, and the limit of number of characters the line
930 * can hold. Add a comma and a space before the range if chars indicates that
931 * something has already been printed on the line, and print ... instead if
932 * chars plus the space needed would go over the limit (use a limit of 0 to
936 test_print_range(unsigned long first, unsigned long last, unsigned long chars,
939 unsigned int needed = 0;
942 for (n = first; n > 0; n /= 10)
945 for (n = last; n > 0; n /= 10)
951 if (limit > 0 && chars + needed > limit) {
953 if (chars <= limit) {
965 printf("%lu-", first);
973 * Summarize a single test set. The second argument is 0 if the set exited
974 * cleanly, a positive integer representing the exit status if it exited
975 * with a non-zero status, and a negative integer representing the signal
976 * that terminated it if it was killed by a signal.
979 test_summarize(struct testset *ts, int status)
982 unsigned long missing = 0;
983 unsigned long failed = 0;
984 unsigned long first = 0;
985 unsigned long last = 0;
988 fputs("ABORTED", stdout);
990 printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped);
992 for (i = 0; i < ts->count; i++) {
993 if (ts->results[i] == TEST_INVALID) {
995 fputs("MISSED ", stdout);
996 if (first && i == last)
1000 test_print_range(first, last, missing - 1, 0);
1008 test_print_range(first, last, missing - 1, 0);
1011 for (i = 0; i < ts->count; i++) {
1012 if (ts->results[i] == TEST_FAIL) {
1013 if (missing && !failed)
1014 fputs("; ", stdout);
1016 fputs("FAILED ", stdout);
1017 if (first && i == last)
1021 test_print_range(first, last, failed - 1, 0);
1029 test_print_range(first, last, failed - 1, 0);
1030 if (!missing && !failed) {
1031 fputs(!status ? "ok" : "dubious", stdout);
1032 if (ts->skipped > 0) {
1033 if (ts->skipped == 1)
1034 printf(" (skipped %lu test)", ts->skipped);
1036 printf(" (skipped %lu tests)", ts->skipped);
1041 printf(" (exit status %d)", status);
1042 else if (status < 0)
1043 printf(" (killed by signal %d%s)", -status,
1044 WCOREDUMP(ts->status) ? ", core dumped" : "");
1050 * Given a test set, analyze the results, classify the exit status, handle a
1051 * few special error messages, and then pass it along to test_summarize() for
1052 * the regular output. Returns true if the test set ran successfully and all
1053 * tests passed or were skipped, false otherwise.
1056 test_analyze(struct testset *ts)
1060 if (ts->all_skipped) {
1061 if (ts->reason == NULL)
1064 printf("skipped (%s)\n", ts->reason);
1066 } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
1067 switch (WEXITSTATUS(ts->status)) {
1070 puts("ABORTED (can't dup file descriptors)");
1074 puts("ABORTED (execution failed -- not found?)");
1076 case CHILDERR_STDIN:
1077 case CHILDERR_STDERR:
1079 puts("ABORTED (can't open /dev/null)");
1082 test_summarize(ts, WEXITSTATUS(ts->status));
1086 } else if (WIFSIGNALED(ts->status)) {
1087 test_summarize(ts, -WTERMSIG(ts->status));
1089 } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) {
1090 puts("ABORTED (no valid test plan)");
1094 test_summarize(ts, 0);
1095 return (ts->failed == 0);
1101 * Runs a single test set, accumulating and then reporting the results.
1102 * Returns true if the test set was successfully run and all tests passed,
1106 test_run(struct testset *ts, enum test_verbose verbose)
1108 pid_t testpid, child;
1112 char buffer[BUFSIZ];
1114 /* Run the test program. */
1115 testpid = test_start(ts->command, &outfd);
1116 output = fdopen(outfd, "r");
1120 sysdie("fdopen failed");
1124 * Pass each line of output to test_checkline(), and print the line if
1125 * verbosity is requested.
1127 while (!ts->aborted && fgets(buffer, sizeof(buffer), output)) {
1129 printf("%s", buffer);
1130 test_checkline(buffer, ts, verbose);
1132 if (ferror(output) || ts->plan == PLAN_INIT)
1138 * Consume the rest of the test output, close the output descriptor,
1139 * retrieve the exit status, and pass that information to test_analyze()
1140 * for eventual output.
1142 while (fgets(buffer, sizeof(buffer), output))
1144 printf("%s", buffer);
1146 child = waitpid(testpid, &ts->status, 0);
1147 if (child == (pid_t) -1) {
1148 if (!ts->reported) {
1152 sysdie("waitpid for %u failed", (unsigned int) testpid);
1154 if (ts->all_skipped)
1156 status = test_analyze(ts);
1158 /* Convert missing tests to failed tests. */
1159 for (i = 0; i < ts->count; i++) {
1160 if (ts->results[i] == TEST_INVALID) {
1162 ts->results[i] = TEST_FAIL;
1170 /* Summarize a list of test failures. */
1172 test_fail_summary(const struct testlist *fails)
1176 unsigned long i, first, last, total;
1181 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
1182 -------------------------- -------------- ---- ---- -------------- */
1183 for (; fails; fails = fails->next) {
1185 total = ts->count - ts->skipped;
1186 failed = (double) ts->failed;
1187 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
1188 total, total ? (failed * 100.0) / (double) total : 0,
1190 if (WIFEXITED(ts->status))
1191 printf("%4d ", WEXITSTATUS(ts->status));
1201 for (i = 0; i < ts->count; i++) {
1202 if (ts->results[i] == TEST_FAIL) {
1203 if (first != 0 && i == last)
1207 chars += test_print_range(first, last, chars, 19);
1214 test_print_range(first, last, chars, 19);
1221 * Check whether a given file path is a valid test. Currently, this checks
1222 * whether it is executable and is a regular file. Returns true or false.
1225 is_valid_test(const char *path)
1229 if (access(path, X_OK) < 0)
1231 if (stat(path, &st) < 0)
1233 if (!S_ISREG(st.st_mode))
1240 * Given the name of a test, a pointer to the testset struct, and the source
1241 * and build directories, find the test. We try first relative to the current
1242 * directory, then in the build directory (if not NULL), then in the source
1243 * directory. In each of those directories, we first try a "-t" extension and
1244 * then a ".t" extension. When we find an executable program, we return the
1245 * path to that program. If none of those paths are executable, just fill in
1246 * the name of the test as is.
1248 * The caller is responsible for freeing the path member of the testset
1252 find_test(const char *name, const char *source, const char *build)
1255 const char *bases[3], *suffix, *base;
1257 const char *suffixes[3] = {"-t", ".t", ""};
1259 /* Possible base directories. */
1264 /* Try each suffix with each base. */
1265 for (i = 0; i < ARRAY_SIZE(suffixes); i++) {
1266 suffix = suffixes[i];
1267 for (j = 0; j < ARRAY_SIZE(bases); j++) {
1271 path = concat(base, "/", name, suffix, (const char *) 0);
1272 if (is_valid_test(path))
1279 path = xstrdup(name);
1285 * Parse a single line of a test list and store the test name and command to
1286 * execute it in the given testset struct.
1288 * Normally, each line is just the name of the test, which is located in the
1289 * test directory and turned into a command to run. However, each line may
1290 * have whitespace-separated options, which change the command that's run.
1291 * Current supported options are:
1294 * Run the test under valgrind if C_TAP_VALGRIND is set. The contents
1295 * of that environment variable are taken as the valgrind command (with
1296 * options) to run. The command is parsed with a simple split on
1297 * whitespace and no quoting is supported.
1300 * If running under valgrind, use libtool to invoke valgrind. This avoids
1301 * running valgrind on the wrapper shell script generated by libtool. If
1302 * set, C_TAP_LIBTOOL must be set to the full path to the libtool program
1303 * to use to run valgrind and thus the test. Ignored if the test isn't
1304 * being run under valgrind.
1307 parse_test_list_line(const char *line, struct testset *ts, const char *source,
1310 const char *p, *end, *option, *libtool;
1311 const char *valgrind = NULL;
1312 unsigned int use_libtool = 0;
1313 unsigned int use_valgrind = 0;
1316 /* Determine the name of the test. */
1317 p = skip_non_whitespace(line);
1318 ts->file = xstrndup(line, p - line);
1320 /* Check if any test options are set. */
1321 p = skip_whitespace(p);
1322 while (*p != '\0') {
1323 end = skip_non_whitespace(p);
1324 if (strncmp(p, "libtool", end - p) == 0) {
1326 } else if (strncmp(p, "valgrind", end - p) == 0) {
1327 valgrind = getenv("C_TAP_VALGRIND");
1328 use_valgrind = (valgrind != NULL);
1330 option = xstrndup(p, end - p);
1331 die("unknown test list option %s", option);
1333 p = skip_whitespace(end);
1336 /* Construct the argv to run the test. First, find the length. */
1338 if (use_valgrind && valgrind != NULL) {
1339 p = skip_whitespace(valgrind);
1340 while (*p != '\0') {
1342 p = skip_whitespace(skip_non_whitespace(p));
1348 /* Now, build the command. */
1349 ts->command = xcalloc(len + 1, char *);
1351 if (use_valgrind && valgrind != NULL) {
1353 libtool = getenv("C_TAP_LIBTOOL");
1354 if (libtool == NULL)
1355 die("valgrind with libtool requested, but C_TAP_LIBTOOL is not"
1357 ts->command[i++] = xstrdup(libtool);
1358 ts->command[i++] = xstrdup("--mode=execute");
1360 p = skip_whitespace(valgrind);
1361 while (*p != '\0') {
1362 end = skip_non_whitespace(p);
1363 ts->command[i++] = xstrndup(p, end - p);
1364 p = skip_whitespace(end);
1368 die("internal error while constructing command line");
1369 ts->command[i++] = find_test(ts->file, source, build);
1370 ts->command[i] = NULL;
1375 * Read a list of tests from a file, returning the list of tests as a struct
1376 * testlist, or NULL if there were no tests (such as a file containing only
1377 * comments). Reports an error to standard error and exits if the list of
1378 * tests cannot be read.
1380 static struct testlist *
1381 read_test_list(const char *filename, const char *source, const char *build)
1386 char buffer[BUFSIZ];
1388 struct testlist *listhead, *current;
1390 /* Create the initial container list that will hold our results. */
1391 listhead = xcalloc(1, struct testlist);
1395 * Open our file of tests to run and read it line by line, creating a new
1396 * struct testlist and struct testset for each line.
1398 file = fopen(filename, "r");
1400 sysdie("can't open %s", filename);
1402 while (fgets(buffer, sizeof(buffer), file)) {
1404 length = strlen(buffer) - 1;
1405 if (buffer[length] != '\n') {
1406 fprintf(stderr, "%s:%u: line too long\n", filename, line);
1409 buffer[length] = '\0';
1411 /* Skip comments, leading spaces, and blank lines. */
1412 start = skip_whitespace(buffer);
1413 if (strlen(start) == 0)
1415 if (start[0] == '#')
1418 /* Allocate the new testset structure. */
1419 if (current == NULL)
1422 current->next = xcalloc(1, struct testlist);
1423 current = current->next;
1425 current->ts = xcalloc(1, struct testset);
1426 current->ts->plan = PLAN_INIT;
1428 /* Parse the line and store the results in the testset struct. */
1429 parse_test_list_line(start, current->ts, source, build);
1433 /* If there were no tests, current is still NULL. */
1434 if (current == NULL) {
1439 /* Return the results. */
1445 * Build a list of tests from command line arguments. Takes the argv and argc
1446 * representing the command line arguments and returns a newly allocated test
1447 * list, or NULL if there were no tests. The caller is responsible for
1450 static struct testlist *
1451 build_test_list(char *argv[], int argc, const char *source, const char *build)
1454 struct testlist *listhead, *current;
1456 /* Create the initial container list that will hold our results. */
1457 listhead = xcalloc(1, struct testlist);
1460 /* Walk the list of arguments and create test sets for them. */
1461 for (i = 0; i < argc; i++) {
1462 if (current == NULL)
1465 current->next = xcalloc(1, struct testlist);
1466 current = current->next;
1468 current->ts = xcalloc(1, struct testset);
1469 current->ts->plan = PLAN_INIT;
1470 current->ts->file = xstrdup(argv[i]);
1471 current->ts->command = xcalloc(2, char *);
1472 current->ts->command[0] = find_test(current->ts->file, source, build);
1473 current->ts->command[1] = NULL;
1476 /* If there were no tests, current is still NULL. */
1477 if (current == NULL) {
1482 /* Return the results. */
1487 /* Free a struct testset. */
1489 free_testset(struct testset *ts)
1494 for (i = 0; ts->command[i] != NULL; i++)
1495 free(ts->command[i]);
1504 * Run a batch of tests. Takes two additional parameters: the root of the
1505 * source directory and the root of the build directory. Test programs will
1506 * be first searched for in the current directory, then the build directory,
1507 * then the source directory. Returns true iff all tests passed, and always
1508 * frees the test list that's passed in.
1511 test_batch(struct testlist *tests, enum test_verbose verbose)
1515 unsigned int count = 0;
1517 struct timeval start, end;
1518 struct rusage stats;
1519 struct testlist *failhead = NULL;
1520 struct testlist *failtail = NULL;
1521 struct testlist *current, *next;
1523 unsigned long total = 0;
1524 unsigned long passed = 0;
1525 unsigned long skipped = 0;
1526 unsigned long failed = 0;
1527 unsigned long aborted = 0;
1529 /* Walk the list of tests to find the longest name. */
1530 for (current = tests; current != NULL; current = current->next) {
1531 length = strlen(current->ts->file);
1532 if (length > longest)
1537 * Add two to longest and round up to the nearest tab stop. This is how
1538 * wide the column for printing the current test name will be.
1542 longest += 8 - (longest % 8);
1544 /* Start the wall clock timer. */
1545 gettimeofday(&start, NULL);
1547 /* Now, plow through our tests again, running each one. */
1548 for (current = tests; current != NULL; current = current->next) {
1551 /* Print out the name of the test file. */
1552 fputs(ts->file, stdout);
1554 fputs("\n\n", stdout);
1556 for (i = strlen(ts->file); i < longest; i++)
1558 if (isatty(STDOUT_FILENO))
1562 succeeded = test_run(ts, verbose);
1567 /* Record cumulative statistics. */
1568 aborted += ts->aborted;
1569 total += ts->count + ts->all_skipped;
1570 passed += ts->passed;
1571 skipped += ts->skipped + ts->all_skipped;
1572 failed += ts->failed;
1575 /* If the test fails, we shuffle it over to the fail list. */
1577 if (failhead == NULL) {
1578 failhead = xcalloc(1, struct testlist);
1579 failtail = failhead;
1581 failtail->next = xcalloc(1, struct testlist);
1582 failtail = failtail->next;
1585 failtail->next = NULL;
1590 /* Stop the timer and get our child resource statistics. */
1591 gettimeofday(&end, NULL);
1592 getrusage(RUSAGE_CHILDREN, &stats);
1594 /* Summarize the failures and free the failure list. */
1595 if (failhead != NULL) {
1596 test_fail_summary(failhead);
1597 while (failhead != NULL) {
1598 next = failhead->next;
1604 /* Free the memory used by the test lists. */
1605 while (tests != NULL) {
1607 free_testset(tests->ts);
1612 /* Print out the final test summary. */
1616 printf("Aborted %lu test set", aborted);
1618 printf("Aborted %lu test sets", aborted);
1619 printf(", passed %lu/%lu tests", passed, total);
1620 } else if (failed == 0)
1621 fputs("All tests successful", stdout);
1623 printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
1624 (double) (total - failed) * 100.0 / (double) total);
1627 printf(", %lu test skipped", skipped);
1629 printf(", %lu tests skipped", skipped);
1632 printf("Files=%u, Tests=%lu", count, total);
1633 printf(", %.2f seconds", tv_diff(&end, &start));
1634 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n", tv_seconds(&stats.ru_utime),
1635 tv_seconds(&stats.ru_stime),
1636 tv_sum(&stats.ru_utime, &stats.ru_stime));
1637 return (failed == 0 && aborted == 0);
1642 * Run a single test case. This involves just running the test program after
1643 * having done the environment setup and finding the test program.
1646 test_single(const char *program, const char *source, const char *build)
1650 path = find_test(program, source, build);
1651 if (execl(path, path, (char *) 0) == -1)
1652 sysdie("cannot exec %s", path);
1657 * Main routine. Set the C_TAP_SOURCE, C_TAP_BUILD, SOURCE, and BUILD
1658 * environment variables and then, given a file listing tests, run each test
1662 main(int argc, char *argv[])
1667 enum test_verbose verbose = CONCISE;
1668 char *c_tap_source_env = NULL;
1669 char *c_tap_build_env = NULL;
1670 char *source_env = NULL;
1671 char *build_env = NULL;
1672 const char *program;
1673 const char *shortlist;
1674 const char *list = NULL;
1675 const char *source = C_TAP_SOURCE;
1676 const char *build = C_TAP_BUILD;
1677 struct testlist *tests;
1680 while ((option = getopt(argc, argv, "b:hl:os:v")) != EOF) {
1686 printf(usage_message, program, program, program, usage_extra);
1706 if ((list == NULL && argc < 1) || (list != NULL && argc > 0)) {
1707 fprintf(stderr, usage_message, program, program, program, usage_extra);
1712 * If C_TAP_VERBOSE is set in the environment, that also turns on verbose
1715 if (getenv("C_TAP_VERBOSE") != NULL)
1719 * Set C_TAP_SOURCE and C_TAP_BUILD environment variables. Also set
1720 * SOURCE and BUILD for backward compatibility, although we're trying to
1721 * migrate to the ones with a C_TAP_* prefix.
1723 if (source != NULL) {
1724 c_tap_source_env = concat("C_TAP_SOURCE=", source, (const char *) 0);
1725 if (putenv(c_tap_source_env) != 0)
1726 sysdie("cannot set C_TAP_SOURCE in the environment");
1727 source_env = concat("SOURCE=", source, (const char *) 0);
1728 if (putenv(source_env) != 0)
1729 sysdie("cannot set SOURCE in the environment");
1731 if (build != NULL) {
1732 c_tap_build_env = concat("C_TAP_BUILD=", build, (const char *) 0);
1733 if (putenv(c_tap_build_env) != 0)
1734 sysdie("cannot set C_TAP_BUILD in the environment");
1735 build_env = concat("BUILD=", build, (const char *) 0);
1736 if (putenv(build_env) != 0)
1737 sysdie("cannot set BUILD in the environment");
1740 /* Run the tests as instructed. */
1742 test_single(argv[0], source, build);
1743 else if (list != NULL) {
1744 shortlist = strrchr(list, '/');
1745 if (shortlist == NULL)
1749 printf(banner, shortlist);
1750 tests = read_test_list(list, source, build);
1751 status = test_batch(tests, verbose) ? 0 : 1;
1753 tests = build_test_list(argv, argc, source, build);
1754 status = test_batch(tests, verbose) ? 0 : 1;
1757 /* For valgrind cleanliness, free all our memory. */
1758 if (source_env != NULL) {
1759 putenv((char *) "C_TAP_SOURCE=");
1760 putenv((char *) "SOURCE=");
1761 free(c_tap_source_env);
1764 if (build_env != NULL) {
1765 putenv((char *) "C_TAP_BUILD=");
1766 putenv((char *) "BUILD=");
1767 free(c_tap_build_env);