2 * Some utility routines for writing tests.
4 * Here are a variety of utility routines for writing tests compatible with
5 * the TAP protocol. All routines of the form ok() or is*() take a test
6 * number and some number of appropriate arguments, check to be sure the
7 * results match the expected output using the arguments, and print out
8 * something appropriate for that test number. Other utility routines help in
9 * constructing more complex tests, skipping tests, reporting errors, setting
10 * up the TAP output format, or finding things in the test environment.
12 * This file is part of C TAP Harness. The current version plus supporting
13 * documentation is at <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
15 * Written by Russ Allbery <eagle@eyrie.org>
16 * Copyright 2009-2019 Russ Allbery <eagle@eyrie.org>
17 * Copyright 2001-2002, 2004-2008, 2011-2014
18 * The Board of Trustees of the Leland Stanford Junior University
20 * Permission is hereby granted, free of charge, to any person obtaining a
21 * copy of this software and associated documentation files (the "Software"),
22 * to deal in the Software without restriction, including without limitation
23 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 * and/or sell copies of the Software, and to permit persons to whom the
25 * Software is furnished to do so, subject to the following conditions:
27 * The above copyright notice and this permission notice shall be included in
28 * all copies or substantial portions of the Software.
30 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 * DEALINGS IN THE SOFTWARE.
38 * SPDX-License-Identifier: MIT
50 # include <sys/stat.h>
52 #include <sys/types.h>
55 #include <tests/tap/basic.h>
57 /* Windows provides mkdir and rmdir under different names. */
59 # define mkdir(p, m) _mkdir(p)
60 # define rmdir(p) _rmdir(p)
64 * The test count. Always contains the number that will be used for the next
65 * test status. This is exported to callers of the library.
67 unsigned long testnum = 1;
70 * Status information stored so that we can give a test summary at the end of
71 * the test case. We store the planned final test and the count of failures.
72 * We can get the highest test count from testnum.
74 static unsigned long _planned = 0;
75 static unsigned long _failed = 0;
78 * Store the PID of the process that called plan() and only summarize
79 * results when that process exits, so as to not misreport results in forked
82 static pid_t _process = 0;
85 * If true, we're doing lazy planning and will print out the plan based on the
86 * last test number at the end of testing.
91 * If true, the test was aborted by calling bail(). Currently, this is only
92 * used to ensure that we pass a false value to any cleanup functions even if
93 * all tests to that point have passed.
95 static int _aborted = 0;
98 * Registered cleanup functions. These are stored as a linked list and run in
99 * registered order by finish when the test program exits. Each function is
100 * passed a boolean value indicating whether all tests were successful.
102 struct cleanup_func {
103 test_cleanup_func func;
104 test_cleanup_func_with_data func_with_data;
106 struct cleanup_func *next;
108 static struct cleanup_func *cleanup_funcs = NULL;
111 * Registered diag files. Any output found in these files will be printed out
112 * as if it were passed to diag() before any other output we do. This allows
113 * background processes to log to a file and have that output interleaved with
121 struct diag_file *next;
123 static struct diag_file *diag_files = NULL;
126 * Print a specified prefix and then the test description. Handles turning
127 * the argument list into a va_args structure suitable for passing to
128 * print_desc, which has to be done in a macro. Assumes that format is the
129 * argument immediately before the variadic arguments.
131 #define PRINT_DESC(prefix, format) \
133 if (format != NULL) { \
135 printf("%s", prefix); \
136 va_start(args, format); \
137 vprintf(format, args); \
144 * Form a new string by concatenating multiple strings. The arguments must be
145 * terminated by (const char *) 0.
147 * This function only exists because we can't assume asprintf. We can't
148 * simulate asprintf with snprintf because we're only assuming SUSv3, which
149 * does not require that snprintf with a NULL buffer return the required
150 * length. When those constraints are relaxed, this should be ripped out and
151 * replaced with asprintf or a more trivial replacement with snprintf.
154 concat(const char *first, ...)
163 * Find the total memory required. Ensure we don't overflow length. See
164 * the comment for breallocarray for why we're using UINT_MAX here.
166 va_start(args, first);
167 for (string = first; string != NULL; string = va_arg(args, const char *)) {
168 if (length >= UINT_MAX - strlen(string))
169 bail("strings too long in concat");
170 length += strlen(string);
175 /* Create the string. */
176 result = bcalloc_type(length, char);
177 va_start(args, first);
179 for (string = first; string != NULL; string = va_arg(args, const char *)) {
180 memcpy(result + offset, string, strlen(string));
181 offset += strlen(string);
184 result[offset] = '\0';
190 * Helper function for check_diag_files to handle a single line in a diag
193 * The general scheme here used is as follows: read one line of output. If we
194 * get NULL, check for an error. If there was one, bail out of the test
195 * program; otherwise, return, and the enclosing loop will check for EOF.
197 * If we get some data, see if it ends in a newline. If it doesn't end in a
198 * newline, we have one of two cases: our buffer isn't large enough, in which
199 * case we resize it and try again, or we have incomplete data in the file, in
200 * which case we rewind the file and will try again next time.
202 * Returns a boolean indicating whether the last line was incomplete.
205 handle_diag_file_line(struct diag_file *file, fpos_t where)
210 /* Read the next line from the file. */
211 size = file->bufsize > INT_MAX ? INT_MAX : (int) file->bufsize;
212 if (fgets(file->buffer, size, file->file) == NULL) {
213 if (ferror(file->file))
214 sysbail("cannot read from %s", file->name);
219 * See if the line ends in a newline. If not, see which error case we
222 length = strlen(file->buffer);
223 if (file->buffer[length - 1] != '\n') {
226 /* Check whether we ran out of buffer space and resize if so. */
227 if (length < file->bufsize - 1)
230 file->bufsize += BUFSIZ;
232 breallocarray_type(file->buffer, file->bufsize, char);
236 * On either incomplete lines or too small of a buffer, rewind
237 * and read the file again (on the next pass, if incomplete).
238 * It's simpler than trying to double-buffer the file.
240 if (fsetpos(file->file, &where) < 0)
241 sysbail("cannot set position in %s", file->name);
245 /* We saw a complete line. Print it out. */
246 printf("# %s", file->buffer);
252 * Check all registered diag_files for any output. We only print out the
253 * output if we see a complete line; otherwise, we wait for the next newline.
256 check_diag_files(void)
258 struct diag_file *file;
263 * Walk through each file and read each line of output available.
265 for (file = diag_files; file != NULL; file = file->next) {
266 clearerr(file->file);
268 /* Store the current position in case we have to rewind. */
269 if (fgetpos(file->file, &where) < 0)
270 sysbail("cannot get position in %s", file->name);
272 /* Continue until we get EOF or an incomplete line of data. */
274 while (!feof(file->file) && !incomplete) {
275 incomplete = handle_diag_file_line(file, where);
282 * Our exit handler. Called on completion of the test to report a summary of
283 * results provided we're still in the original process. This also handles
284 * printing out the plan if we used plan_lazy(), although that's suppressed if
285 * we never ran a test (due to an early bail, for example), and running any
286 * registered cleanup functions.
291 int success, primary;
292 struct cleanup_func *current;
293 unsigned long highest = testnum - 1;
294 struct diag_file *file, *tmp;
296 /* Check for pending diag_file output. */
299 /* Free the diag_files. */
301 while (file != NULL) {
312 * Determine whether all tests were successful, which is needed before
313 * calling cleanup functions since we pass that fact to the functions.
315 if (_planned == 0 && _lazy)
317 success = (!_aborted && _planned == highest && _failed == 0);
320 * If there are any registered cleanup functions, we run those first. We
321 * always run them, even if we didn't run a test. Don't do anything
322 * except free the diag_files and call cleanup functions if we aren't the
323 * primary process (the process in which plan or plan_lazy was called),
324 * and tell the cleanup functions that fact.
326 primary = (_process == 0 || getpid() == _process);
327 while (cleanup_funcs != NULL) {
328 if (cleanup_funcs->func_with_data) {
329 void *data = cleanup_funcs->data;
331 cleanup_funcs->func_with_data(success, primary, data);
333 cleanup_funcs->func(success, primary);
335 current = cleanup_funcs;
336 cleanup_funcs = cleanup_funcs->next;
342 /* Don't do anything further if we never planned a test. */
346 /* If we're aborting due to bail, don't print summaries. */
350 /* Print out the lazy plan if needed. */
352 if (_lazy && _planned > 0)
353 printf("1..%lu\n", _planned);
355 /* Print out a summary of the results. */
356 if (_planned > highest)
357 diag("Looks like you planned %lu test%s but only ran %lu", _planned,
358 (_planned > 1 ? "s" : ""), highest);
359 else if (_planned < highest)
360 diag("Looks like you planned %lu test%s but ran %lu extra", _planned,
361 (_planned > 1 ? "s" : ""), highest - _planned);
362 else if (_failed > 0)
363 diag("Looks like you failed %lu test%s of %lu", _failed,
364 (_failed > 1 ? "s" : ""), _planned);
365 else if (_planned != 1)
366 diag("All %lu tests successful or skipped", _planned);
368 diag("%lu test successful or skipped", _planned);
373 * Initialize things. Turns on line buffering on stdout and then prints out
374 * the number of tests in the test suite. We intentionally don't check for
375 * pending diag_file output here, since it should really come after the plan.
378 plan(unsigned long count)
380 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
381 sysdiag("cannot set stdout to line buffered");
383 printf("1..%lu\n", count);
387 if (atexit(finish) != 0) {
388 sysdiag("cannot register exit handler");
389 diag("cleanups will not be run");
395 * Initialize things for lazy planning, where we'll automatically print out a
396 * plan at the end of the program. Turns on line buffering on stdout as well.
401 if (setvbuf(stdout, NULL, _IOLBF, BUFSIZ) != 0)
402 sysdiag("cannot set stdout to line buffered");
406 if (atexit(finish) != 0)
407 sysbail("cannot register exit handler to display plan");
412 * Skip the entire test suite and exits. Should be called instead of plan(),
413 * not after it, since it prints out a special plan line. Ignore diag_file
414 * output here, since it's not clear if it's allowed before the plan.
417 skip_all(const char *format, ...)
420 printf("1..0 # skip");
421 PRINT_DESC(" ", format);
428 * Takes a boolean success value and assumes the test passes if that value
429 * is true and fails if that value is false.
432 ok(int success, const char *format, ...)
436 printf("%sok %lu", success ? "" : "not ", testnum++);
439 PRINT_DESC(" - ", format);
446 * Same as ok(), but takes the format arguments as a va_list.
449 okv(int success, const char *format, va_list args)
453 printf("%sok %lu", success ? "" : "not ", testnum++);
456 if (format != NULL) {
458 vprintf(format, args);
469 skip(const char *reason, ...)
473 printf("ok %lu # skip", testnum++);
474 PRINT_DESC(" ", reason);
480 * Report the same status on the next count tests.
483 ok_block(unsigned long count, int success, const char *format, ...)
489 for (i = 0; i < count; i++) {
490 printf("%sok %lu", success ? "" : "not ", testnum++);
493 PRINT_DESC(" - ", format);
501 * Skip the next count tests.
504 skip_block(unsigned long count, const char *reason, ...)
510 for (i = 0; i < count; i++) {
511 printf("ok %lu # skip", testnum++);
512 PRINT_DESC(" ", reason);
519 * Takes two boolean values and requires the truth value of both match.
522 is_bool(int left, int right, const char *format, ...)
528 success = (!!left == !!right);
530 printf("ok %lu", testnum++);
532 diag(" left: %s", !!left ? "true" : "false");
533 diag("right: %s", !!right ? "true" : "false");
534 printf("not ok %lu", testnum++);
537 PRINT_DESC(" - ", format);
544 * Takes two integer values and requires they match.
547 is_int(long left, long right, const char *format, ...)
553 success = (left == right);
555 printf("ok %lu", testnum++);
557 diag(" left: %ld", left);
558 diag("right: %ld", right);
559 printf("not ok %lu", testnum++);
562 PRINT_DESC(" - ", format);
569 * Takes two strings and requires they match (using strcmp). NULL arguments
570 * are permitted and handled correctly.
573 is_string(const char *left, const char *right, const char *format, ...)
580 /* Compare the strings, being careful of NULL. */
582 success = (right == NULL);
583 else if (right == NULL)
586 success = (strcmp(left, right) == 0);
588 /* Report the results. */
590 printf("ok %lu", testnum++);
592 diag(" left: %s", left == NULL ? "(null)" : left);
593 diag("right: %s", right == NULL ? "(null)" : right);
594 printf("not ok %lu", testnum++);
597 PRINT_DESC(" - ", format);
604 * Takes two unsigned longs and requires they match. On failure, reports them
608 is_hex(unsigned long left, unsigned long right, const char *format, ...)
614 success = (left == right);
616 printf("ok %lu", testnum++);
618 diag(" left: %lx", (unsigned long) left);
619 diag("right: %lx", (unsigned long) right);
620 printf("not ok %lu", testnum++);
623 PRINT_DESC(" - ", format);
630 * Takes pointers to a regions of memory and requires that len bytes from each
631 * match. Otherwise reports any bytes which didn't match.
634 is_blob(const void *left, const void *right, size_t len, const char *format,
642 success = (memcmp(left, right, len) == 0);
644 printf("ok %lu", testnum++);
646 const unsigned char *left_c = (const unsigned char *) left;
647 const unsigned char *right_c = (const unsigned char *) right;
649 for (i = 0; i < len; i++) {
650 if (left_c[i] != right_c[i])
651 diag("offset %lu: left %02x, right %02x", (unsigned long) i,
652 left_c[i], right_c[i]);
654 printf("not ok %lu", testnum++);
657 PRINT_DESC(" - ", format);
664 * Bail out with an error.
667 bail(const char *format, ...)
675 printf("Bail out! ");
676 va_start(args, format);
677 vprintf(format, args);
685 * Bail out with an error, appending strerror(errno).
688 sysbail(const char *format, ...)
697 printf("Bail out! ");
698 va_start(args, format);
699 vprintf(format, args);
701 printf(": %s\n", strerror(oerrno));
707 * Report a diagnostic to stderr. Always returns 1 to allow embedding in
708 * compound statements.
711 diag(const char *format, ...)
719 va_start(args, format);
720 vprintf(format, args);
728 * Report a diagnostic to stderr, appending strerror(errno). Always returns 1
729 * to allow embedding in compound statements.
732 sysdiag(const char *format, ...)
741 va_start(args, format);
742 vprintf(format, args);
744 printf(": %s\n", strerror(oerrno));
750 * Register a new file for diag_file processing.
753 diag_file_add(const char *name)
755 struct diag_file *file, *prev;
757 file = bcalloc_type(1, struct diag_file);
758 file->name = bstrdup(name);
759 file->file = fopen(file->name, "r");
760 if (file->file == NULL)
761 sysbail("cannot open %s", name);
762 file->buffer = bcalloc_type(BUFSIZ, char);
763 file->bufsize = BUFSIZ;
764 if (diag_files == NULL)
767 for (prev = diag_files; prev->next != NULL; prev = prev->next)
775 * Remove a file from diag_file processing. If the file is not found, do
776 * nothing, since there are some situations where it can be removed twice
777 * (such as if it's removed from a cleanup function, since cleanup functions
778 * are called after freeing all the diag_files).
781 diag_file_remove(const char *name)
783 struct diag_file *file;
784 struct diag_file **prev = &diag_files;
786 for (file = diag_files; file != NULL; file = file->next) {
787 if (strcmp(file->name, name) == 0) {
801 * Allocate cleared memory, reporting a fatal error with bail on failure.
804 bcalloc(size_t n, size_t size)
810 sysbail("failed to calloc %lu", (unsigned long) (n * size));
816 * Allocate memory, reporting a fatal error with bail on failure.
825 sysbail("failed to malloc %lu", (unsigned long) size);
831 * Reallocate memory, reporting a fatal error with bail on failure.
834 brealloc(void *p, size_t size)
836 p = realloc(p, size);
838 sysbail("failed to realloc %lu bytes", (unsigned long) size);
844 * The same as brealloc, but determine the size by multiplying an element
845 * count by a size, similar to calloc. The multiplication is checked for
848 * We should technically use SIZE_MAX here for the overflow check, but
849 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
850 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
851 * can assume that UINT_MAX <= SIZE_MAX.
853 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
854 * I disbelieve in the existence of such systems and they will have to cope
855 * without overflow checks.)
858 breallocarray(void *p, size_t n, size_t size)
860 if (n > 0 && UINT_MAX / n <= size)
861 bail("reallocarray too large");
864 p = realloc(p, n * size);
866 sysbail("failed to realloc %lu bytes", (unsigned long) (n * size));
872 * Copy a string, reporting a fatal error with bail on failure.
875 bstrdup(const char *s)
881 p = (char *) malloc(len);
883 sysbail("failed to strdup %lu bytes", (unsigned long) len);
890 * Copy up to n characters of a string, reporting a fatal error with bail on
891 * failure. Don't use the system strndup function, since it may not exist and
892 * the TAP library doesn't assume any portability support.
895 bstrndup(const char *s, size_t n)
901 /* Don't assume that the source string is nul-terminated. */
902 for (p = s; (size_t)(p - s) < n && *p != '\0'; p++)
904 length = (size_t)(p - s);
905 copy = (char *) malloc(length + 1);
907 sysbail("failed to strndup %lu bytes", (unsigned long) length);
908 memcpy(copy, s, length);
915 * Locate a test file. Given the partial path to a file, look under
916 * C_TAP_BUILD and then C_TAP_SOURCE for the file and return the full path to
917 * the file. Returns NULL if the file doesn't exist. A non-NULL return
918 * should be freed with test_file_path_free().
921 test_file_path(const char *file)
925 const char *envs[] = {"C_TAP_BUILD", "C_TAP_SOURCE", NULL};
928 for (i = 0; envs[i] != NULL; i++) {
929 base = getenv(envs[i]);
932 path = concat(base, "/", file, (const char *) 0);
933 if (access(path, R_OK) == 0)
943 * Free a path returned from test_file_path(). This function exists primarily
944 * for Windows, where memory must be freed from the same library domain that
945 * it was allocated from.
948 test_file_path_free(char *path)
955 * Create a temporary directory, tmp, under C_TAP_BUILD if set and the current
956 * directory if it does not. Returns the path to the temporary directory in
957 * newly allocated memory, and calls bail on any failure. The return value
958 * should be freed with test_tmpdir_free.
960 * This function uses sprintf because it attempts to be independent of all
961 * other portability layers. The use immediately after a memory allocation
962 * should be safe without using snprintf or strlcpy/strlcat.
970 build = getenv("C_TAP_BUILD");
973 path = concat(build, "/tmp", (const char *) 0);
974 if (access(path, X_OK) < 0)
975 if (mkdir(path, 0777) < 0)
976 sysbail("error creating temporary directory %s", path);
982 * Free a path returned from test_tmpdir() and attempt to remove the
983 * directory. If we can't delete the directory, don't worry; something else
984 * that hasn't yet cleaned up may still be using it.
987 test_tmpdir_free(char *path)
995 register_cleanup(test_cleanup_func func,
996 test_cleanup_func_with_data func_with_data, void *data)
998 struct cleanup_func *cleanup, **last;
1000 cleanup = bcalloc_type(1, struct cleanup_func);
1001 cleanup->func = func;
1002 cleanup->func_with_data = func_with_data;
1003 cleanup->data = data;
1004 cleanup->next = NULL;
1005 last = &cleanup_funcs;
1006 while (*last != NULL)
1007 last = &(*last)->next;
1012 * Register a cleanup function that is called when testing ends. All such
1013 * registered functions will be run by finish.
1016 test_cleanup_register(test_cleanup_func func)
1018 register_cleanup(func, NULL, NULL);
1022 * Same as above, but also allows an opaque pointer to be passed to the cleanup
1026 test_cleanup_register_with_data(test_cleanup_func_with_data func, void *data)
1028 register_cleanup(NULL, func, data);