2 * Some utility routines for writing tests.
4 * Here are a variety of utility routines for writing tests compatible with
5 * the TAP protocol. All routines of the form ok() or is*() take a test
6 * number and some number of appropriate arguments, check to be sure the
7 * results match the expected output using the arguments, and print out
8 * something appropriate for that test number. Other utility routines help in
9 * constructing more complex tests, skipping tests, reporting errors, setting
10 * up the TAP output format, or finding things in the test environment.
12 * This file is part of C TAP Harness. The current version plus supporting
13 * documentation is at <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
15 * Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016
16 * Russ Allbery <eagle@eyrie.org>
17 * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014
18 * The Board of Trustees of the Leland Stanford Junior University
20 * Permission is hereby granted, free of charge, to any person obtaining a
21 * copy of this software and associated documentation files (the "Software"),
22 * to deal in the Software without restriction, including without limitation
23 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 * and/or sell copies of the Software, and to permit persons to whom the
25 * Software is furnished to do so, subject to the following conditions:
27 * The above copyright notice and this permission notice shall be included in
28 * all copies or substantial portions of the Software.
30 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 * DEALINGS IN THE SOFTWARE.
48 # include <sys/stat.h>
50 #include <sys/types.h>
53 #include <tests/tap/basic.h>
55 /* Windows provides mkdir and rmdir under different names. */
57 # define mkdir(p, m) _mkdir(p)
58 # define rmdir(p) _rmdir(p)
62 * The test count. Always contains the number that will be used for the next
63 * test status. This is exported to callers of the library.
65 unsigned long testnum = 1;
68 * Status information stored so that we can give a test summary at the end of
69 * the test case. We store the planned final test and the count of failures.
70 * We can get the highest test count from testnum.
72 static unsigned long _planned = 0;
73 static unsigned long _failed = 0;
76 * Store the PID of the process that called plan() and only summarize
77 * results when that process exits, so as to not misreport results in forked
80 static pid_t _process = 0;
83 * If true, we're doing lazy planning and will print out the plan based on the
84 * last test number at the end of testing.
89 * If true, the test was aborted by calling bail(). Currently, this is only
90 * used to ensure that we pass a false value to any cleanup functions even if
91 * all tests to that point have passed.
93 static int _aborted = 0;
96 * Registered cleanup functions. These are stored as a linked list and run in
97 * registered order by finish when the test program exits. Each function is
98 * passed a boolean value indicating whether all tests were successful.
100 struct cleanup_func {
101 test_cleanup_func func;
102 struct cleanup_func *next;
104 static struct cleanup_func *cleanup_funcs = NULL;
107 * Registered diag files. Any output found in these files will be printed out
108 * as if it were passed to diag() before any other output we do. This allows
109 * background processes to log to a file and have that output interleaved with
117 struct diag_file *next;
119 static struct diag_file *diag_files = NULL;
122 * Print a specified prefix and then the test description. Handles turning
123 * the argument list into a va_args structure suitable for passing to
124 * print_desc, which has to be done in a macro. Assumes that format is the
125 * argument immediately before the variadic arguments.
127 #define PRINT_DESC(prefix, format) \
129 if (format != NULL) { \
131 if (prefix != NULL) \
132 printf("%s", prefix); \
133 va_start(args, format); \
134 vprintf(format, args); \
141 * Form a new string by concatenating multiple strings. The arguments must be
142 * terminated by (const char *) 0.
144 * This function only exists because we can't assume asprintf. We can't
145 * simulate asprintf with snprintf because we're only assuming SUSv3, which
146 * does not require that snprintf with a NULL buffer return the required
147 * length. When those constraints are relaxed, this should be ripped out and
148 * replaced with asprintf or a more trivial replacement with snprintf.
151 concat(const char *first, ...)
160 * Find the total memory required. Ensure we don't overflow length. See
161 * the comment for breallocarray for why we're using UINT_MAX here.
163 va_start(args, first);
164 for (string = first; string != NULL; string = va_arg(args, const char *)) {
165 if (length >= UINT_MAX - strlen(string))
166 bail("strings too long in concat");
167 length += strlen(string);
172 /* Create the string. */
173 result = bmalloc(length);
174 va_start(args, first);
176 for (string = first; string != NULL; string = va_arg(args, const char *)) {
177 memcpy(result + offset, string, strlen(string));
178 offset += strlen(string);
181 result[offset] = '\0';
187 * Check all registered diag_files for any output. We only print out the
188 * output if we see a complete line; otherwise, we wait for the next newline.
191 check_diag_files(void)
193 struct diag_file *file;
196 int size, incomplete;
199 * Walk through each file and read each line of output available. The
200 * general scheme here used is as follows: try to read a line of output at
201 * a time. If we get NULL, check for EOF; on EOF, advance to the next
204 * If we get some data, see if it ends in a newline. If it doesn't end in
205 * a newline, we have one of two cases: our buffer isn't large enough, in
206 * which case we resize it and try again, or we have incomplete data in
207 * the file, in which case we rewind the file and will try again next
210 for (file = diag_files; file != NULL; file = file->next) {
211 clearerr(file->file);
213 /* Store the current position in case we have to rewind. */
214 if (fgetpos(file->file, &where) < 0)
215 sysbail("cannot get position in %s", file->name);
217 /* Continue until we get EOF or an incomplete line of data. */
219 while (!feof(file->file) && !incomplete) {
220 size = file->bufsize > INT_MAX ? INT_MAX : (int) file->bufsize;
221 if (fgets(file->buffer, size, file->file) == NULL) {
222 if (ferror(file->file))
223 sysbail("cannot read from %s", file->name);
228 * See if the line ends in a newline. If not, see which error
229 * case we have. Use UINT_MAX as a substitute for SIZE_MAX (see
230 * the comment for breallocarray).
232 length = strlen(file->buffer);
233 if (file->buffer[length - 1] != '\n') {
234 if (length < file->bufsize - 1)
237 if (file->bufsize >= UINT_MAX - BUFSIZ)
238 sysbail("line too long in %s", file->name);
239 file->bufsize += BUFSIZ;
240 file->buffer = brealloc(file->buffer, file->bufsize);
244 * On either incomplete lines or too small of a buffer, rewind
245 * and read the file again (on the next pass, if incomplete).
246 * It's simpler than trying to double-buffer the file.
248 if (fsetpos(file->file, &where) < 0)
249 sysbail("cannot set position in %s", file->name);
253 /* We saw a complete line. Print it out. */
254 printf("# %s", file->buffer);
261 * Our exit handler. Called on completion of the test to report a summary of
262 * results provided we're still in the original process. This also handles
263 * printing out the plan if we used plan_lazy(), although that's suppressed if
264 * we never ran a test (due to an early bail, for example), and running any
265 * registered cleanup functions.
270 int success, primary;
271 struct cleanup_func *current;
272 unsigned long highest = testnum - 1;
273 struct diag_file *file, *tmp;
275 /* Check for pending diag_file output. */
278 /* Free the diag_files. */
280 while (file != NULL) {
291 * Determine whether all tests were successful, which is needed before
292 * calling cleanup functions since we pass that fact to the functions.
294 if (_planned == 0 && _lazy)
296 success = (!_aborted && _planned == highest && _failed == 0);
299 * If there are any registered cleanup functions, we run those first. We
300 * always run them, even if we didn't run a test. Don't do anything
301 * except free the diag_files and call cleanup functions if we aren't the
302 * primary process (the process in which plan or plan_lazy was called),
303 * and tell the cleanup functions that fact.
305 primary = (_process == 0 || getpid() == _process);
306 while (cleanup_funcs != NULL) {
307 cleanup_funcs->func(success, primary);
308 current = cleanup_funcs;
309 cleanup_funcs = cleanup_funcs->next;
315 /* Don't do anything further if we never planned a test. */
319 /* If we're aborting due to bail, don't print summaries. */
323 /* Print out the lazy plan if needed. */
325 if (_lazy && _planned > 0)
326 printf("1..%lu\n", _planned);
328 /* Print out a summary of the results. */
329 if (_planned > highest)
330 diag("Looks like you planned %lu test%s but only ran %lu", _planned,
331 (_planned > 1 ? "s" : ""), highest);
332 else if (_planned < highest)
333 diag("Looks like you planned %lu test%s but ran %lu extra", _planned,
334 (_planned > 1 ? "s" : ""), highest - _planned);
335 else if (_failed > 0)
336 diag("Looks like you failed %lu test%s of %lu", _failed,
337 (_failed > 1 ? "s" : ""), _planned);
338 else if (_planned != 1)
339 diag("All %lu tests successful or skipped", _planned);
341 diag("%lu test successful or skipped", _planned);
346 * Initialize things. Turns on line buffering on stdout and then prints out
347 * the number of tests in the test suite. We intentionally don't check for
348 * pending diag_file output here, since it should really come after the plan.
351 plan(unsigned long count)
353 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
354 sysdiag("cannot set stdout to line buffered");
356 printf("1..%lu\n", count);
360 if (atexit(finish) != 0) {
361 sysdiag("cannot register exit handler");
362 diag("cleanups will not be run");
368 * Initialize things for lazy planning, where we'll automatically print out a
369 * plan at the end of the program. Turns on line buffering on stdout as well.
374 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
375 sysdiag("cannot set stdout to line buffered");
379 if (atexit(finish) != 0)
380 sysbail("cannot register exit handler to display plan");
385 * Skip the entire test suite and exits. Should be called instead of plan(),
386 * not after it, since it prints out a special plan line. Ignore diag_file
387 * output here, since it's not clear if it's allowed before the plan.
390 skip_all(const char *format, ...)
393 printf("1..0 # skip");
394 PRINT_DESC(" ", format);
401 * Takes a boolean success value and assumes the test passes if that value
402 * is true and fails if that value is false.
405 ok(int success, const char *format, ...)
409 printf("%sok %lu", success ? "" : "not ", testnum++);
412 PRINT_DESC(" - ", format);
419 * Same as ok(), but takes the format arguments as a va_list.
422 okv(int success, const char *format, va_list args)
426 printf("%sok %lu", success ? "" : "not ", testnum++);
429 if (format != NULL) {
431 vprintf(format, args);
442 skip(const char *reason, ...)
446 printf("ok %lu # skip", testnum++);
447 PRINT_DESC(" ", reason);
453 * Report the same status on the next count tests.
456 ok_block(unsigned long count, int success, const char *format, ...)
462 for (i = 0; i < count; i++) {
463 printf("%sok %lu", success ? "" : "not ", testnum++);
466 PRINT_DESC(" - ", format);
474 * Skip the next count tests.
477 skip_block(unsigned long count, const char *reason, ...)
483 for (i = 0; i < count; i++) {
484 printf("ok %lu # skip", testnum++);
485 PRINT_DESC(" ", reason);
492 * Takes an expected integer and a seen integer and assumes the test passes
493 * if those two numbers match.
496 is_int(long wanted, long seen, const char *format, ...)
502 success = (wanted == seen);
504 printf("ok %lu", testnum++);
506 diag("wanted: %ld", wanted);
507 diag(" seen: %ld", seen);
508 printf("not ok %lu", testnum++);
511 PRINT_DESC(" - ", format);
518 * Takes a string and what the string should be, and assumes the test passes
519 * if those strings match (using strcmp).
522 is_string(const char *wanted, const char *seen, const char *format, ...)
532 success = (strcmp(wanted, seen) == 0);
534 printf("ok %lu", testnum++);
536 diag("wanted: %s", wanted);
537 diag(" seen: %s", seen);
538 printf("not ok %lu", testnum++);
541 PRINT_DESC(" - ", format);
548 * Takes an expected unsigned long and a seen unsigned long and assumes the
549 * test passes if the two numbers match. Otherwise, reports them in hex.
552 is_hex(unsigned long wanted, unsigned long seen, const char *format, ...)
558 success = (wanted == seen);
560 printf("ok %lu", testnum++);
562 diag("wanted: %lx", (unsigned long) wanted);
563 diag(" seen: %lx", (unsigned long) seen);
564 printf("not ok %lu", testnum++);
567 PRINT_DESC(" - ", format);
574 * Bail out with an error.
577 bail(const char *format, ...)
585 printf("Bail out! ");
586 va_start(args, format);
587 vprintf(format, args);
595 * Bail out with an error, appending strerror(errno).
598 sysbail(const char *format, ...)
607 printf("Bail out! ");
608 va_start(args, format);
609 vprintf(format, args);
611 printf(": %s\n", strerror(oerrno));
617 * Report a diagnostic to stderr. Always returns 1 to allow embedding in
618 * compound statements.
621 diag(const char *format, ...)
629 va_start(args, format);
630 vprintf(format, args);
638 * Report a diagnostic to stderr, appending strerror(errno). Always returns 1
639 * to allow embedding in compound statements.
642 sysdiag(const char *format, ...)
651 va_start(args, format);
652 vprintf(format, args);
654 printf(": %s\n", strerror(oerrno));
660 * Register a new file for diag_file processing.
663 diag_file_add(const char *name)
665 struct diag_file *file, *prev;
667 file = bcalloc(1, sizeof(struct diag_file));
668 file->name = bstrdup(name);
669 file->file = fopen(file->name, "r");
670 if (file->file == NULL)
671 sysbail("cannot open %s", name);
672 file->buffer = bmalloc(BUFSIZ);
673 file->bufsize = BUFSIZ;
674 if (diag_files == NULL)
677 for (prev = diag_files; prev->next != NULL; prev = prev->next)
685 * Remove a file from diag_file processing. If the file is not found, do
686 * nothing, since there are some situations where it can be removed twice
687 * (such as if it's removed from a cleanup function, since cleanup functions
688 * are called after freeing all the diag_files).
691 diag_file_remove(const char *name)
693 struct diag_file *file;
694 struct diag_file **prev = &diag_files;
696 for (file = diag_files; file != NULL; file = file->next) {
697 if (strcmp(file->name, name) == 0) {
711 * Allocate cleared memory, reporting a fatal error with bail on failure.
714 bcalloc(size_t n, size_t size)
720 sysbail("failed to calloc %lu", (unsigned long)(n * size));
726 * Allocate memory, reporting a fatal error with bail on failure.
735 sysbail("failed to malloc %lu", (unsigned long) size);
741 * Reallocate memory, reporting a fatal error with bail on failure.
744 brealloc(void *p, size_t size)
746 p = realloc(p, size);
748 sysbail("failed to realloc %lu bytes", (unsigned long) size);
754 * The same as brealloc, but determine the size by multiplying an element
755 * count by a size, similar to calloc. The multiplication is checked for
758 * We should technically use SIZE_MAX here for the overflow check, but
759 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
760 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
761 * can assume that UINT_MAX <= SIZE_MAX.
763 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
764 * I disbelieve in the existence of such systems and they will have to cope
765 * without overflow checks.)
768 breallocarray(void *p, size_t n, size_t size)
770 if (n > 0 && UINT_MAX / n <= size)
771 bail("reallocarray too large");
772 p = realloc(p, n * size);
774 sysbail("failed to realloc %lu bytes", (unsigned long) (n * size));
780 * Copy a string, reporting a fatal error with bail on failure.
783 bstrdup(const char *s)
791 sysbail("failed to strdup %lu bytes", (unsigned long) len);
798 * Copy up to n characters of a string, reporting a fatal error with bail on
799 * failure. Don't use the system strndup function, since it may not exist and
800 * the TAP library doesn't assume any portability support.
803 bstrndup(const char *s, size_t n)
809 /* Don't assume that the source string is nul-terminated. */
810 for (p = s; (size_t) (p - s) < n && *p != '\0'; p++)
812 length = (size_t) (p - s);
813 copy = malloc(length + 1);
815 sysbail("failed to strndup %lu bytes", (unsigned long) length);
816 memcpy(copy, s, length);
823 * Locate a test file. Given the partial path to a file, look under
824 * C_TAP_BUILD and then C_TAP_SOURCE for the file and return the full path to
825 * the file. Returns NULL if the file doesn't exist. A non-NULL return
826 * should be freed with test_file_path_free().
829 test_file_path(const char *file)
833 const char *envs[] = { "C_TAP_BUILD", "C_TAP_SOURCE", NULL };
836 for (i = 0; envs[i] != NULL; i++) {
837 base = getenv(envs[i]);
840 path = concat(base, "/", file, (const char *) 0);
841 if (access(path, R_OK) == 0)
851 * Free a path returned from test_file_path(). This function exists primarily
852 * for Windows, where memory must be freed from the same library domain that
853 * it was allocated from.
856 test_file_path_free(char *path)
863 * Create a temporary directory, tmp, under C_TAP_BUILD if set and the current
864 * directory if it does not. Returns the path to the temporary directory in
865 * newly allocated memory, and calls bail on any failure. The return value
866 * should be freed with test_tmpdir_free.
868 * This function uses sprintf because it attempts to be independent of all
869 * other portability layers. The use immediately after a memory allocation
870 * should be safe without using snprintf or strlcpy/strlcat.
878 build = getenv("C_TAP_BUILD");
881 path = concat(build, "/tmp", (const char *) 0);
882 if (access(path, X_OK) < 0)
883 if (mkdir(path, 0777) < 0)
884 sysbail("error creating temporary directory %s", path);
890 * Free a path returned from test_tmpdir() and attempt to remove the
891 * directory. If we can't delete the directory, don't worry; something else
892 * that hasn't yet cleaned up may still be using it.
895 test_tmpdir_free(char *path)
904 * Register a cleanup function that is called when testing ends. All such
905 * registered functions will be run by finish.
908 test_cleanup_register(test_cleanup_func func)
910 struct cleanup_func *cleanup, **last;
912 cleanup = bmalloc(sizeof(struct cleanup_func));
913 cleanup->func = func;
914 cleanup->next = NULL;
915 last = &cleanup_funcs;
916 while (*last != NULL)
917 last = &(*last)->next;