2 * Some utility routines for writing tests.
4 * Here are a variety of utility routines for writing tests compatible with
5 * the TAP protocol. All routines of the form ok() or is*() take a test
6 * number and some number of appropriate arguments, check to be sure the
7 * results match the expected output using the arguments, and print out
8 * something appropriate for that test number. Other utility routines help in
9 * constructing more complex tests, skipping tests, reporting errors, setting
10 * up the TAP output format, or finding things in the test environment.
12 * This file is part of C TAP Harness. The current version plus supporting
13 * documentation is at <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
15 * Copyright 2009, 2010, 2011, 2012, 2013 Russ Allbery <eagle@eyrie.org>
16 * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014
17 * The Board of Trustees of the Leland Stanford Junior University
19 * Permission is hereby granted, free of charge, to any person obtaining a
20 * copy of this software and associated documentation files (the "Software"),
21 * to deal in the Software without restriction, including without limitation
22 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
23 * and/or sell copies of the Software, and to permit persons to whom the
24 * Software is furnished to do so, subject to the following conditions:
26 * The above copyright notice and this permission notice shall be included in
27 * all copies or substantial portions of the Software.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
32 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
34 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
35 * DEALINGS IN THE SOFTWARE.
46 # include <sys/stat.h>
48 #include <sys/types.h>
51 #include <tests/tap/basic.h>
53 /* Windows provides mkdir and rmdir under different names. */
55 # define mkdir(p, m) _mkdir(p)
56 # define rmdir(p) _rmdir(p)
60 * The test count. Always contains the number that will be used for the next
61 * test status. This is exported to callers of the library.
63 unsigned long testnum = 1;
66 * Status information stored so that we can give a test summary at the end of
67 * the test case. We store the planned final test and the count of failures.
68 * We can get the highest test count from testnum.
70 static unsigned long _planned = 0;
71 static unsigned long _failed = 0;
74 * Store the PID of the process that called plan() and only summarize
75 * results when that process exits, so as to not misreport results in forked
78 static pid_t _process = 0;
81 * If true, we're doing lazy planning and will print out the plan based on the
82 * last test number at the end of testing.
87 * If true, the test was aborted by calling bail(). Currently, this is only
88 * used to ensure that we pass a false value to any cleanup functions even if
89 * all tests to that point have passed.
91 static int _aborted = 0;
94 * Registered cleanup functions. These are stored as a linked list and run in
95 * registered order by finish when the test program exits. Each function is
96 * passed a boolean value indicating whether all tests were successful.
99 test_cleanup_func func;
100 struct cleanup_func *next;
102 static struct cleanup_func *cleanup_funcs = NULL;
105 * Registered diag files. Any output found in these files will be printed out
106 * as if it were passed to diag() before any other output we do. This allows
107 * background processes to log to a file and have that output interleved with
115 struct diag_file *next;
117 static struct diag_file *diag_files = NULL;
120 * Print a specified prefix and then the test description. Handles turning
121 * the argument list into a va_args structure suitable for passing to
122 * print_desc, which has to be done in a macro. Assumes that format is the
123 * argument immediately before the variadic arguments.
125 #define PRINT_DESC(prefix, format) \
127 if (format != NULL) { \
129 if (prefix != NULL) \
130 printf("%s", prefix); \
131 va_start(args, format); \
132 vprintf(format, args); \
139 * Check all registered diag_files for any output. We only print out the
140 * output if we see a complete line; otherwise, we wait for the next newline.
143 check_diag_files(void)
145 struct diag_file *file;
151 * Walk through each file and read each line of output available. The
152 * general scheme here used is as follows: try to read a line of output at
153 * a time. If we get NULL, check for EOF; on EOF, advance to the next
156 * If we get some data, see if it ends in a newline. If it doesn't end in
157 * a newline, we have one of two cases: our buffer isn't large enough, in
158 * which case we resize it and try again, or we have incomplete data in
159 * the file, in which case we rewind the file and will try again next
162 for (file = diag_files; file != NULL; file = file->next) {
163 clearerr(file->file);
165 /* Store the current position in case we have to rewind. */
166 if (fgetpos(file->file, &where) < 0)
167 sysbail("cannot get position in %s", file->name);
169 /* Continue until we get EOF or an incomplete line of data. */
171 while (!feof(file->file) && !incomplete) {
172 if (fgets(file->buffer, file->bufsize, file->file) == NULL) {
173 if (ferror(file->file))
174 sysbail("cannot read from %s", file->name);
179 * See if the line ends in a newline. If not, see which error
182 length = strlen(file->buffer);
183 if (file->buffer[length - 1] != '\n') {
184 if (length < file->bufsize - 1)
187 file->bufsize += BUFSIZ;
188 file->buffer = brealloc(file->buffer, file->bufsize);
192 * On either incomplete lines or too small of a buffer, rewind
193 * and read the file again (on the next pass, if incomplete).
194 * It's simpler than trying to double-buffer the file.
196 if (fsetpos(file->file, &where) < 0)
197 sysbail("cannot set position in %s", file->name);
201 /* We saw a complete line. Print it out. */
202 printf("# %s", file->buffer);
209 * Our exit handler. Called on completion of the test to report a summary of
210 * results provided we're still in the original process. This also handles
211 * printing out the plan if we used plan_lazy(), although that's suppressed if
212 * we never ran a test (due to an early bail, for example), and running any
213 * registered cleanup functions.
218 int success, primary;
219 struct cleanup_func *current;
220 unsigned long highest = testnum - 1;
221 struct diag_file *file, *tmp;
223 /* Check for pending diag_file output. */
226 /* Free the diag_files. */
228 while (file != NULL) {
239 * Determine whether all tests were successful, which is needed before
240 * calling cleanup functions since we pass that fact to the functions.
242 if (_planned == 0 && _lazy)
244 success = (!_aborted && _planned == highest && _failed == 0);
247 * If there are any registered cleanup functions, we run those first. We
248 * always run them, even if we didn't run a test. Don't do anything
249 * except free the diag_files and call cleanup functions if we aren't the
250 * primary process (the process in which plan or plan_lazy was called),
251 * and tell the cleanup functions that fact.
253 primary = (_process == 0 || getpid() == _process);
254 while (cleanup_funcs != NULL) {
255 cleanup_funcs->func(success, primary);
256 current = cleanup_funcs;
257 cleanup_funcs = cleanup_funcs->next;
263 /* Don't do anything further if we never planned a test. */
267 /* If we're aborting due to bail, don't print summaries. */
271 /* Print out the lazy plan if needed. */
273 if (_lazy && _planned > 0)
274 printf("1..%lu\n", _planned);
276 /* Print out a summary of the results. */
277 if (_planned > highest)
278 diag("Looks like you planned %lu test%s but only ran %lu", _planned,
279 (_planned > 1 ? "s" : ""), highest);
280 else if (_planned < highest)
281 diag("Looks like you planned %lu test%s but ran %lu extra", _planned,
282 (_planned > 1 ? "s" : ""), highest - _planned);
283 else if (_failed > 0)
284 diag("Looks like you failed %lu test%s of %lu", _failed,
285 (_failed > 1 ? "s" : ""), _planned);
286 else if (_planned != 1)
287 diag("All %lu tests successful or skipped", _planned);
289 diag("%lu test successful or skipped", _planned);
294 * Initialize things. Turns on line buffering on stdout and then prints out
295 * the number of tests in the test suite. We intentionally don't check for
296 * pending diag_file output here, since it should really come after the plan.
299 plan(unsigned long count)
301 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
302 sysdiag("cannot set stdout to line buffered");
304 printf("1..%lu\n", count);
308 if (atexit(finish) != 0) {
309 sysdiag("cannot register exit handler");
310 diag("cleanups will not be run");
316 * Initialize things for lazy planning, where we'll automatically print out a
317 * plan at the end of the program. Turns on line buffering on stdout as well.
322 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
323 sysdiag("cannot set stdout to line buffered");
327 if (atexit(finish) != 0)
328 sysbail("cannot register exit handler to display plan");
333 * Skip the entire test suite and exits. Should be called instead of plan(),
334 * not after it, since it prints out a special plan line. Ignore diag_file
335 * output here, since it's not clear if it's allowed before the plan.
338 skip_all(const char *format, ...)
341 printf("1..0 # skip");
342 PRINT_DESC(" ", format);
349 * Takes a boolean success value and assumes the test passes if that value
350 * is true and fails if that value is false.
353 ok(int success, const char *format, ...)
357 printf("%sok %lu", success ? "" : "not ", testnum++);
360 PRINT_DESC(" - ", format);
366 * Same as ok(), but takes the format arguments as a va_list.
369 okv(int success, const char *format, va_list args)
373 printf("%sok %lu", success ? "" : "not ", testnum++);
376 if (format != NULL) {
378 vprintf(format, args);
388 skip(const char *reason, ...)
392 printf("ok %lu # skip", testnum++);
393 PRINT_DESC(" ", reason);
399 * Report the same status on the next count tests.
402 ok_block(unsigned long count, int status, const char *format, ...)
408 for (i = 0; i < count; i++) {
409 printf("%sok %lu", status ? "" : "not ", testnum++);
412 PRINT_DESC(" - ", format);
419 * Skip the next count tests.
422 skip_block(unsigned long count, const char *reason, ...)
428 for (i = 0; i < count; i++) {
429 printf("ok %lu # skip", testnum++);
430 PRINT_DESC(" ", reason);
437 * Takes an expected integer and a seen integer and assumes the test passes
438 * if those two numbers match.
441 is_int(long wanted, long seen, const char *format, ...)
446 printf("ok %lu", testnum++);
448 diag("wanted: %ld", wanted);
449 diag(" seen: %ld", seen);
450 printf("not ok %lu", testnum++);
453 PRINT_DESC(" - ", format);
459 * Takes a string and what the string should be, and assumes the test passes
460 * if those strings match (using strcmp).
463 is_string(const char *wanted, const char *seen, const char *format, ...)
471 if (strcmp(wanted, seen) == 0)
472 printf("ok %lu", testnum++);
474 diag("wanted: %s", wanted);
475 diag(" seen: %s", seen);
476 printf("not ok %lu", testnum++);
479 PRINT_DESC(" - ", format);
485 * Takes an expected unsigned long and a seen unsigned long and assumes the
486 * test passes if the two numbers match. Otherwise, reports them in hex.
489 is_hex(unsigned long wanted, unsigned long seen, const char *format, ...)
494 printf("ok %lu", testnum++);
496 diag("wanted: %lx", (unsigned long) wanted);
497 diag(" seen: %lx", (unsigned long) seen);
498 printf("not ok %lu", testnum++);
501 PRINT_DESC(" - ", format);
507 * Bail out with an error.
510 bail(const char *format, ...)
518 printf("Bail out! ");
519 va_start(args, format);
520 vprintf(format, args);
528 * Bail out with an error, appending strerror(errno).
531 sysbail(const char *format, ...)
540 printf("Bail out! ");
541 va_start(args, format);
542 vprintf(format, args);
544 printf(": %s\n", strerror(oerrno));
550 * Report a diagnostic to stderr.
553 diag(const char *format, ...)
561 va_start(args, format);
562 vprintf(format, args);
569 * Report a diagnostic to stderr, appending strerror(errno).
572 sysdiag(const char *format, ...)
581 va_start(args, format);
582 vprintf(format, args);
584 printf(": %s\n", strerror(oerrno));
589 * Register a new file for diag_file processing.
592 diag_file_add(const char *name)
594 struct diag_file *file, *prev;
596 file = bcalloc(1, sizeof(struct diag_file));
597 file->name = bstrdup(name);
598 file->file = fopen(file->name, "r");
599 if (file->file == NULL)
600 sysbail("cannot open %s", name);
601 file->buffer = bmalloc(BUFSIZ);
602 file->bufsize = BUFSIZ;
603 if (diag_files == NULL)
606 for (prev = diag_files; prev->next != NULL; prev = prev->next)
614 * Remove a file from diag_file processing. If the file is not found, do
615 * nothing, since there are some situations where it can be removed twice
616 * (such as if it's removed from a cleanup function, since cleanup functions
617 * are called after freeing all the diag_files).
620 diag_file_remove(const char *name)
622 struct diag_file *file;
623 struct diag_file **prev = &diag_files;
625 for (file = diag_files; file != NULL; file = file->next) {
626 if (strcmp(file->name, name) == 0) {
640 * Allocate cleared memory, reporting a fatal error with bail on failure.
643 bcalloc(size_t n, size_t size)
649 sysbail("failed to calloc %lu", (unsigned long)(n * size));
655 * Allocate memory, reporting a fatal error with bail on failure.
664 sysbail("failed to malloc %lu", (unsigned long) size);
670 * Reallocate memory, reporting a fatal error with bail on failure.
673 brealloc(void *p, size_t size)
675 p = realloc(p, size);
677 sysbail("failed to realloc %lu bytes", (unsigned long) size);
683 * Copy a string, reporting a fatal error with bail on failure.
686 bstrdup(const char *s)
694 sysbail("failed to strdup %lu bytes", (unsigned long) len);
701 * Copy up to n characters of a string, reporting a fatal error with bail on
702 * failure. Don't use the system strndup function, since it may not exist and
703 * the TAP library doesn't assume any portability support.
706 bstrndup(const char *s, size_t n)
712 /* Don't assume that the source string is nul-terminated. */
713 for (p = s; (size_t) (p - s) < n && *p != '\0'; p++)
716 copy = malloc(length + 1);
718 sysbail("failed to strndup %lu bytes", (unsigned long) length);
719 memcpy(copy, s, length);
726 * Locate a test file. Given the partial path to a file, look under BUILD and
727 * then SOURCE for the file and return the full path to the file. Returns
728 * NULL if the file doesn't exist. A non-NULL return should be freed with
729 * test_file_path_free().
731 * This function uses sprintf because it attempts to be independent of all
732 * other portability layers. The use immediately after a memory allocation
733 * should be safe without using snprintf or strlcpy/strlcat.
736 test_file_path(const char *file)
741 const char *envs[] = { "BUILD", "SOURCE", NULL };
744 for (i = 0; envs[i] != NULL; i++) {
745 base = getenv(envs[i]);
748 length = strlen(base) + 1 + strlen(file) + 1;
749 path = bmalloc(length);
750 sprintf(path, "%s/%s", base, file);
751 if (access(path, R_OK) == 0)
761 * Free a path returned from test_file_path(). This function exists primarily
762 * for Windows, where memory must be freed from the same library domain that
763 * it was allocated from.
766 test_file_path_free(char *path)
773 * Create a temporary directory, tmp, under BUILD if set and the current
774 * directory if it does not. Returns the path to the temporary directory in
775 * newly allocated memory, and calls bail on any failure. The return value
776 * should be freed with test_tmpdir_free.
778 * This function uses sprintf because it attempts to be independent of all
779 * other portability layers. The use immediately after a memory allocation
780 * should be safe without using snprintf or strlcpy/strlcat.
789 build = getenv("BUILD");
792 length = strlen(build) + strlen("/tmp") + 1;
793 path = bmalloc(length);
794 sprintf(path, "%s/tmp", build);
795 if (access(path, X_OK) < 0)
796 if (mkdir(path, 0777) < 0)
797 sysbail("error creating temporary directory %s", path);
803 * Free a path returned from test_tmpdir() and attempt to remove the
804 * directory. If we can't delete the directory, don't worry; something else
805 * that hasn't yet cleaned up may still be using it.
808 test_tmpdir_free(char *path)
817 * Register a cleanup function that is called when testing ends. All such
818 * registered functions will be run by finish.
821 test_cleanup_register(test_cleanup_func func)
823 struct cleanup_func *cleanup, **last;
825 cleanup = bmalloc(sizeof(struct cleanup_func));
826 cleanup->func = func;
827 cleanup->next = NULL;
828 last = &cleanup_funcs;
829 while (*last != NULL)
830 last = &(*last)->next;