TestRunner.h

/*
MIT License
 
Copyright (c) 2018 Brian T. Park
 
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
 
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
 
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
 
#ifndef AUNIT_TEST_RUNNER_H
#define AUNIT_TEST_RUNNER_H
 
#if defined(EPOXY_DUINO)
#include <stdlib.h> // exit()
#endif
#include <stdint.h>
#include <Arduino.h> // SERIAL_PORT_MONITOR, F(), Print
#include "Test.h"
 
// ESP32 does not defined SERIAL_PORT_MONITOR
#ifndef SERIAL_PORT_MONITOR
#define SERIAL_PORT_MONITOR Serial
#endif
 
namespace aunit {
 
class TestRunner {
  public:
    typedef uint16_t TimeoutType;
 
    static void run() {
      getRunner()->runTest();
    }
 
    static void list() {
      getRunner()->listTests();
    }
 
    static void exclude(const char* pattern) {
      getRunner()->setLifeCycleMatchingPattern(
          pattern, Test::kLifeCycleExcluded);
    }
 
    static void exclude(const char* testClass, const char* pattern) {
      getRunner()->setLifeCycleMatchingPattern(testClass, pattern,
          Test::kLifeCycleExcluded);
    }
 
    static void include(const char* pattern) {
      getRunner()->setLifeCycleMatchingPattern(pattern, Test::kLifeCycleNew);
    }
 
    static void include(const char* testClass, const char* pattern) {
      getRunner()->setLifeCycleMatchingPattern(testClass, pattern,
          Test::kLifeCycleNew);
    }
 
    static void excludesub(const char* substring) {
      getRunner()->setLifeCycleMatchingSubstring(
          substring, Test::kLifeCycleExcluded);
    }
 
    static void includesub(const char* substring) {
      getRunner()->setLifeCycleMatchingSubstring(
          substring, Test::kLifeCycleNew);
    }
 
    static void setVerbosity(uint8_t verbosity) {
      getRunner()->setVerbosityFlag(verbosity);
    }
 
    static bool isVerbosity(uint8_t verbosity) {
      return getRunner()->isVerbosityFlag(verbosity);
    }
 
    static void setPrinter(Print* printer);
 
    static void setTimeout(TimeoutType seconds) {
      getRunner()->setRunnerTimeout(seconds);
    }
 
  private:
    static const TimeoutType kTimeoutDefault = 10;
 
    static const uint8_t kMaxPatternLength = 63 + 1;
 
    static TestRunner* getRunner();
 
    static uint16_t countTests();
 
    // Disable copy-constructor and assignment operator
    TestRunner(const TestRunner&) = delete;
    TestRunner& operator=(const TestRunner&) = delete;
 
    TestRunner() {}
 
    void runTest() {
      setupRunner();
 
      // Print initial header if this is the first run.
      if (!mIsRunning) {
        printStartRunner();
        mIsRunning = true;
      }
 
      // If no more test cases, then print out summary of run.
      if (*Test::getRoot() == nullptr) {
        if (!mIsResolved) {
          mEndTime = millis();
          resolveRun();
          mIsResolved = true;
        #if EPOXY_DUINO
          exit((mFailedCount || mExpiredCount) ? 1 : 0);
        #endif
        }
        return;
      }
 
      // If reached the end and there are still test cases left, start from the
      // beginning again.
      if (*mCurrent == nullptr) {
        mCurrent = Test::getRoot();
      }
 
      // Implement a finite state machine that calls the (*mCurrent)->setup() or
      // (*mCurrent)->loop(), then changes the test case's mStatus.
      switch ((*mCurrent)->getLifeCycle()) {
        case Test::kLifeCycleNew:
          // Transfer the verbosity of the TestRunner to the Test.
          (*mCurrent)->enableVerbosity(mVerbosity);
          (*mCurrent)->setup();
 
          // Support assertXxx() statements inside the setup() method by
          // moving to the next lifeCycle state if an assertXxx() did not fail
          // inside the setup().
          if ((*mCurrent)->getLifeCycle() == Test::kLifeCycleNew) {
            (*mCurrent)->setLifeCycle(Test::kLifeCycleSetup);
          }
          break;
        case Test::kLifeCycleExcluded:
          // If a test is excluded, go directly to LifeCycleFinished, without
          // calling setup() or teardown().
          (*mCurrent)->enableVerbosity(mVerbosity);
          (*mCurrent)->setStatus(Test::kStatusSkipped);
          mSkippedCount++;
          (*mCurrent)->setLifeCycle(Test::kLifeCycleFinished);
          break;
        case Test::kLifeCycleSetup:
          {
            // Check for timeout. mTimeout == 0 means infinite timeout. NOTE: It
            // feels like this code should go into the Test::loop() method (like
            // the extra bit of code in TestOnce::loop()) because it seems like
            // we could want the timeout to be configurable on a case by case
            // basis. This would cause the testing() code to move down into a
            // new again() virtual method dispatched from Test::loop(),
            // analogous to once(). But let's keep the code here for now.
            unsigned long now = millis();
            if (mTimeout > 0 && now >= mStartTime + 1000L * mTimeout) {
              (*mCurrent)->expire();
            } else {
              (*mCurrent)->loop();
 
              // If test status is unresolved (i.e. still in kLifeCycleNew
              // state) after loop(), then this is a continuous testing() test
              // case, so skip to the next test. Otherwise, stay on the current
              // test so that the next iteration of runTest() can resolve the
              // current test.
              if ((*mCurrent)->getLifeCycle() == Test::kLifeCycleSetup) {
                // skip to the next one, but keep current test in the list
                mCurrent = (*mCurrent)->getNext();
              }
            }
          }
          break;
        case Test::kLifeCycleAsserted:
          switch ((*mCurrent)->getStatus()) {
            case Test::kStatusSkipped:
              mSkippedCount++;
              break;
            case Test::kStatusPassed:
              mPassedCount++;
              break;
            case Test::kStatusFailed:
              mFailedCount++;
              break;
            case Test::kStatusExpired:
              mExpiredCount++;
              break;
            default:
              // should never get here
              mStatusErrorCount++;
              break;
          }
          (*mCurrent)->teardown();
          (*mCurrent)->setLifeCycle(Test::kLifeCycleFinished);
          break;
        case Test::kLifeCycleFinished:
          (*mCurrent)->resolve();
          // skip to the next one by taking current test out of the list
          *mCurrent = *(*mCurrent)->getNext();
          break;
      }
    }
 
    void listTests() {
      setupRunner();
 
      Print* printer = Printer::getPrinter();
      printer->print(F("TestRunner test count: "));
      printer->println(mCount);
      for (Test** p = Test::getRoot(); (*p) != nullptr; p = (*p)->getNext()) {
        printer->print(F("Test "));
        (*p)->getName().print(printer);
        printer->print(F("; lifeCycle: "));
        printer->println((*p)->getLifeCycle());
      }
    }
 
    void printStartRunner() const;
 
    void resolveRun() const;
 
    void setupRunner() {
      if (mIsSetup) return;
 
      if (! Printer::getPrinter()) {
        Printer::setPrinter(&SERIAL_PORT_MONITOR);
      }
 
    #if EPOXY_DUINO
      processCommandLine();
    #endif
      mIsSetup = true;
      mCount = countTests();
      mCurrent = Test::getRoot();
      mStartTime = millis();
    }
 
    void setVerbosityFlag(uint8_t verbosity) { mVerbosity = verbosity; }
 
    bool isVerbosityFlag(uint8_t verbosity) const {
      return mVerbosity & verbosity;
    }
 
    void setLifeCycleMatchingPattern(const char* pattern, uint8_t lifeCycle);
 
    void setLifeCycleMatchingPattern(const char* testClass, const char* pattern,
        uint8_t lifeCycle);
 
    void setLifeCycleMatchingSubstring(
        const char* substring, uint8_t lifeCycle);
 
    void excludeAll();
 
    void setRunnerTimeout(TimeoutType seconds);
 
  #if EPOXY_DUINO
    enum class FilterType : uint8_t {
      kInclude,
      kExclude,
      kIncludeSub,
      kExcludeSub
    };
 
    void processCommandLine();
 
    int parseFlags(int argc, const char* const* argv);
 
    void processCommaList(const char* commaList, FilterType filterType);
  #endif
 
  private:
    // The current test case is represented by a pointer to a pointer. This
    // allows treating the root node the same as all the other nodes, and
    // simplifies the code traversing the singly-linked list significantly.
    Test** mCurrent = nullptr;
 
    bool mIsResolved = false;
    bool mIsSetup = false;
    bool mIsRunning = false;
    uint8_t mVerbosity = Verbosity::kDefault;
    // True if any include(), exclude(), includesub(), excludesub() was invoked.
    bool hasBeenFiltered = false;
    uint16_t mCount = 0;
    uint16_t mPassedCount = 0;
    uint16_t mFailedCount = 0;
    uint16_t mSkippedCount = 0;
    uint16_t mExpiredCount = 0;
    uint16_t mStatusErrorCount = 0;
    TimeoutType mTimeout = kTimeoutDefault;
    unsigned long mStartTime;
    unsigned long mEndTime;
};
 
}
 
#endif