Sustainable C++ Testing

Riddle #1: Exceptional

We have a Widget class, defined by its size and anchor position. For simplicity, all 1-dimensional. The constructor performs basic validations on the parameters and that's the scope of our tests.

#include <stdexcept>
#include <gtest/gtest.h>

class Widget {
public:
  Widget(int _size, int _anchor) : size(_size), anchor(_anchor) {
    if (_size < 0) {
      throw std::invalid_argument("Invalid size");
    }
    if (_anchor < 0) {
      throw std::invalid_argument("Invalid anchor");
    }
  }
  
private:
  const int size, anchor;
};

TEST(WidgetTest, construct_negativeSize_throwsException) {
  const int negativeSize = -1;
  const int testAnchor = 0;
  EXPECT_THROW(Widget(negativeSize, testAnchor), std::invalid_argument);
}

TEST(WidgetTest, construct_negativeAnchor_throwsException) {
  const int testSize = 5;
  const int negativeAnchor = -1;
  EXPECT_THROW(Widget(testSize, negativeAnchor), std::invalid_argument);
}

#include <stdexcept>
#include <gtest/gtest.h>

class Widget {
public:
  Widget(int _size, int _anchor) : size(_size), anchor(_anchor) {
    if (_size < 10) {
      throw std::invalid_argument("Invalid size");
    }
    if (_anchor < 0) {
      throw std::invalid_argument("Invalid anchor");
    }
  }
  
private:
  const int size, anchor;
};

TEST(WidgetTest, construct_negativeSize_throwsException) {
  const int negativeSize = -1;
  const int testAnchor = 0;
  EXPECT_THROW(Widget(negativeSize, testAnchor), std::invalid_argument);
}

TEST(WidgetTest, construct_negativeAnchor_throwsException) {
  const int testSize = 5;
  const int negativeAnchor = -1;

  // FAIL! exception still thrown but not because of negative anchor anymore
  EXPECT_THROW(Widget(testSize, negativeAnchor), std::invalid_argument);
}

#include <stdexcept>
#include <gtest/gtest.h>

class Widget {
public:
  class InvalidSize : public std::invalid_argument { using invalid_argument::invalid_argument; };
  class InvalidAnchor : public std::invalid_argument { using invalid_argument::invalid_argument; };
  
  Widget(int _size, int _anchor) : size(_size), anchor(_anchor) {
    if (_size < 0) {
      throw InvalidSize("Invalid size");
    }
    if (_anchor < 0) {
      throw InvalidAnchor("Invalid anchor");
    }
  }
  
private:
  const int size, anchor;
};

TEST(WidgetTest, construct_negativeSize_throwsException) {
  const int negativeSize = -1;
  const int testAnchor = 0;
  EXPECT_THROW(Widget(negativeSize, testAnchor), Widget::InvalidSize);
}

TEST(WidgetTest, construct_negativeAnchor_throwsException) {
  const int testSize = 5;
  const int negativeAnchor = -1;
  EXPECT_THROW(Widget(testSize, negativeAnchor), Widget::InvalidAnchor);
}

#define EXPECT_THROW_MSG(__statement, __msg) \
try { \
  __statement; \
  FAIL() << "Expected: Statement throws an exception.\n"\
    " Actual: it throws nothing."; \
} catch (const std::exception& e) { \
  ASSERT_TRUE(e.what()); \
  EXPECT_STREQ(__msg, e.what()); \
} catch (...) { \
  FAIL() << "Expected: Statement throws an exception.\n"\
    " Actual: it throws something other than an std::exception."; \
}

#include <stdexcept>
#include <gtest/gtest.h>

class Widget {
public:
  Widget(int _size, int _anchor) : size(_size), anchor(_anchor) {
    if (_size < 0) {
      throw std::invalid_argument("Invalid size");
    }
    if (_anchor < 0) {
      throw std::invalid_argument("Invalid anchor");
    }
  }
  
private:
  const int size, anchor;
};

TEST(WidgetTest, construct_negativeSize_throwsException) {
  const int negativeSize = -1;
  const int testAnchor = 0;
  EXPECT_THROW_MSG(Widget(negativeSize, testAnchor), "Invalid size");
}

TEST(WidgetTest, construct_negativeAnchor_throwsException) {
  const int testSize = 5;
  const int negativeAnchor = -1;
  EXPECT_THROW_MSG(Widget(testSize, negativeAnchor), "Invalid anchor");
}

Can you spot a weakness in the tests?

Give me a hint

What if the validations in the constructor were changed?

Give me a hint

What if the constraints on size were changed to e.g. "no less than 10"?

Reveal the answer

Diagnosis

Certain changes in the validations may get some tests in trouble, even if those changes do not affect the test scope at all.
If we consider a change to the validation on size, to reject values below 10, then construct_negativeAnchor_throwsException will continue detecting an exception, but now thrown for a different reason - size being invalid (new validation) and not the negative anchor (test scope). The test would therefore quietly stop delivering what it promises in its name. Show me
Of course, one could hope the maintainer would review all test cases when applying such change in validations, but for larger classes in larger systems, that's not a very realistic expectation. Instead, maybe there is an inexpensive trick to make the test defend itself against this sort of changes?

Reveal the remedy

Remedy

Use dedicated exception classes for different errors. In some cases it might be beneficial also for the production code itself to differentiate between exception types: if the API user might have a use case for it, e.g. if anchor and size come from different sources and errors in validation need to be treated differently.
Show me

In your expectation, you can not only check the exception type, but also the error message or its part. GTest does not have a built-in macro for that, but you can produce one for your project fairly easily:

#define EXPECT_THROW_MSG(__statement, __msg) \
try { \
  __statement; \
  FAIL() << "Expected: Statement throws an exception.\n"\
    " Actual: it throws nothing."; \
} catch (const std::exception& e) { \
  ASSERT_TRUE(e.what()); \
  EXPECT_STREQ(__msg, e.what()); \
} catch (...) { \
  FAIL() << "Expected: Statement throws an exception.\n"\
    " Actual: it throws something other than an std::exception."; \
}

Show me

Feedback on this riddle? 3c6120687265663d226d61696c746f3a666565646261636b407375737461696e61626c6563707074657374696e672e636f6d3f7375626a6563743d5375737461696e61626c6520432b2b2054657374696e672c20726964646c65233120666565646261636b223e446f2067657420696e20746f756368213c2f613e