testsuite: allow for expected failure of tests

Adds a bool to the test struct called 'expected_fail' which can be set
to flip the logic used to determine success and failure. Messaging is
also changed to reflect an unexpected pass or expected fail. This can be
used to write tests which may represent functionality desirable for a
future release.

2 files changed, 34 insertions, 14 deletions

diff --git a/testsuite/testsuite.c b/testsuite/testsuite.c
index 60dfff2..56e73ee 100644
--- a/testsuite/testsuite.c
+++ b/testsuite/testsuite.c
@@ -407,21 +407,40 @@ static inline int test_run_parent(const struct test *t, int fdout[2],
 				WTERMSIG(err), strsignal(WTERMSIG(err)));
 	}
 
-	if (err == 0) {
-		if (matchout)
-			LOG("%sPASSED%s: %s\n",
-				ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
-				t->name);
-		else {
-			ERR("%sFAILED%s: exit ok but outputs do not match: %s\n",
-				ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
-				t->name);
-			err = EXIT_FAILURE;
+	if (t->expected_fail == false) {
+		if (err == 0) {
+			if (matchout)
+				LOG("%sPASSED%s: %s\n",
+					ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+			else {
+				ERR("%sFAILED%s: exit ok but outputs do not match: %s\n",
+					ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+				err = EXIT_FAILURE;
+			}
+		} else
+			ERR("%sFAILED%s: %s\n",
+					ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+	} else {
+		if (err == 0) {
+			if (matchout) {
+				LOG("%sUNEXPECTED PASS%s: %s\n",
+					ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+				err = EXIT_FAILURE;
+			} else
+				LOG("%sEXPECTED FAIL%s: exit ok but outputs do not match: %s\n",
+					ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+		} else {
+			ERR("%sEXPECTED FAIL%s: %s\n",
+					ANSI_HIGHLIGHT_GREEN_ON, ANSI_HIGHLIGHT_OFF,
+					t->name);
+			err = EXIT_SUCCESS;
 		}
-	} else
-		ERR("%sFAILED%s: %s\n",
-				ANSI_HIGHLIGHT_RED_ON, ANSI_HIGHLIGHT_OFF,
-				t->name);
+	}
 
 	return err;
 }
diff --git a/testsuite/testsuite.h b/testsuite/testsuite.h
index b02d73d..9081d84 100644
--- a/testsuite/testsuite.h
+++ b/testsuite/testsuite.h
@@ -48,6 +48,7 @@ struct test {
 	testfunc func;
 	const char *config[_TC_LAST];
 	bool need_spawn;
+	bool expected_fail;
 };