330 lines
12 KiB
C++
330 lines
12 KiB
C++
// Protocol Buffers - Google's data interchange format
|
|
// Copyright 2008 Google Inc. All rights reserved.
|
|
// https://developers.google.com/protocol-buffers/
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions are
|
|
// met:
|
|
//
|
|
// * Redistributions of source code must retain the above copyright
|
|
// notice, this list of conditions and the following disclaimer.
|
|
// * Redistributions in binary form must reproduce the above
|
|
// copyright notice, this list of conditions and the following disclaimer
|
|
// in the documentation and/or other materials provided with the
|
|
// distribution.
|
|
// * Neither the name of Google Inc. nor the names of its
|
|
// contributors may be used to endorse or promote products derived from
|
|
// this software without specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
// This file defines a protocol for running the conformance test suite
|
|
// in-process. In other words, the suite itself will run in the same process as
|
|
// the code under test.
|
|
//
|
|
// For pros and cons of this approach, please see conformance.proto.
|
|
|
|
#ifndef CONFORMANCE_CONFORMANCE_TEST_H
|
|
#define CONFORMANCE_CONFORMANCE_TEST_H
|
|
|
|
#include <functional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include <google/protobuf/descriptor.h>
|
|
#include <google/protobuf/wire_format_lite.h>
|
|
#include <google/protobuf/util/type_resolver.h>
|
|
#include "conformance.pb.h"
|
|
|
|
namespace conformance {
|
|
class ConformanceRequest;
|
|
class ConformanceResponse;
|
|
} // namespace conformance
|
|
|
|
namespace protobuf_test_messages {
|
|
namespace proto3 {
|
|
class TestAllTypesProto3;
|
|
} // namespace proto3
|
|
} // namespace protobuf_test_messages
|
|
|
|
namespace google {
|
|
namespace protobuf {
|
|
|
|
class ConformanceTestSuite;
|
|
|
|
class ConformanceTestRunner {
|
|
public:
|
|
virtual ~ConformanceTestRunner() {}
|
|
|
|
// Call to run a single conformance test.
|
|
//
|
|
// "input" is a serialized conformance.ConformanceRequest.
|
|
// "output" should be set to a serialized conformance.ConformanceResponse.
|
|
//
|
|
// If there is any error in running the test itself, set "runtime_error" in
|
|
// the response.
|
|
virtual void RunTest(const std::string& test_name,
|
|
const std::string& input,
|
|
std::string* output) = 0;
|
|
};
|
|
|
|
// Test runner that spawns the process being tested and communicates with it
|
|
// over a pipe.
|
|
class ForkPipeRunner : public ConformanceTestRunner {
|
|
public:
|
|
// Note: Run() doesn't take ownership of the pointers inside suites.
|
|
static int Run(int argc, char *argv[],
|
|
const std::vector<ConformanceTestSuite*>& suites);
|
|
|
|
ForkPipeRunner(const std::string& executable,
|
|
const std::vector<std::string>& executable_args)
|
|
: child_pid_(-1),
|
|
executable_(executable),
|
|
executable_args_(executable_args) {}
|
|
|
|
explicit ForkPipeRunner(const std::string& executable)
|
|
: child_pid_(-1), executable_(executable) {}
|
|
|
|
virtual ~ForkPipeRunner() {}
|
|
|
|
void RunTest(const std::string& test_name,
|
|
const std::string& request,
|
|
std::string* response);
|
|
|
|
private:
|
|
void SpawnTestProgram();
|
|
|
|
void CheckedWrite(int fd, const void *buf, size_t len);
|
|
bool TryRead(int fd, void *buf, size_t len);
|
|
void CheckedRead(int fd, void *buf, size_t len);
|
|
|
|
int write_fd_;
|
|
int read_fd_;
|
|
pid_t child_pid_;
|
|
std::string executable_;
|
|
const std::vector<std::string> executable_args_;
|
|
std::string current_test_name_;
|
|
};
|
|
|
|
// Class representing the test suite itself. To run it, implement your own
|
|
// class derived from ConformanceTestRunner, class derived from
|
|
// ConformanceTestSuite and then write code like:
|
|
//
|
|
// class MyConformanceTestSuite : public ConformanceTestSuite {
|
|
// public:
|
|
// void RunSuiteImpl() {
|
|
// // INSERT ACTUAL TESTS.
|
|
// }
|
|
// };
|
|
//
|
|
// class MyConformanceTestRunner : public ConformanceTestRunner {
|
|
// public:
|
|
// static int Run(int argc, char *argv[],
|
|
// ConformanceTestSuite* suite);
|
|
//
|
|
// private:
|
|
// virtual void RunTest(...) {
|
|
// // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE.
|
|
// }
|
|
// };
|
|
//
|
|
// int main() {
|
|
// MyConformanceTestSuite suite;
|
|
// MyConformanceTestRunner::Run(argc, argv, &suite);
|
|
// }
|
|
//
|
|
class ConformanceTestSuite {
|
|
public:
|
|
ConformanceTestSuite()
|
|
: verbose_(false),
|
|
enforce_recommended_(false),
|
|
failure_list_flag_name_("--failure_list") {}
|
|
virtual ~ConformanceTestSuite() {}
|
|
|
|
void SetVerbose(bool verbose) { verbose_ = verbose; }
|
|
|
|
// Whether to require the testee to pass RECOMMENDED tests. By default failing
|
|
// a RECOMMENDED test case will not fail the entire suite but will only
|
|
// generated a warning. If this flag is set to true, RECOMMENDED tests will
|
|
// be treated the same way as REQUIRED tests and failing a RECOMMENDED test
|
|
// case will cause the entire test suite to fail as well. An implementation
|
|
// can enable this if it wants to be strictly conforming to protobuf spec.
|
|
// See the comments about ConformanceLevel below to learn more about the
|
|
// difference between REQUIRED and RECOMMENDED test cases.
|
|
void SetEnforceRecommended(bool value) {
|
|
enforce_recommended_ = value;
|
|
}
|
|
|
|
// Gets the flag name to the failure list file.
|
|
// By default, this would return --failure_list
|
|
std::string GetFailureListFlagName() { return failure_list_flag_name_; }
|
|
|
|
void SetFailureListFlagName(const std::string& failure_list_flag_name) {
|
|
failure_list_flag_name_ = failure_list_flag_name;
|
|
}
|
|
|
|
// Sets the path of the output directory.
|
|
void SetOutputDir(const char* output_dir) {
|
|
output_dir_ = output_dir;
|
|
}
|
|
|
|
// Run all the conformance tests against the given test runner.
|
|
// Test output will be stored in "output".
|
|
//
|
|
// Returns true if the set of failing tests was exactly the same as the
|
|
// failure list.
|
|
// The filename here is *only* used to create/format useful error messages for
|
|
// how to update the failure list. We do NOT read this file at all.
|
|
bool RunSuite(ConformanceTestRunner* runner, std::string* output,
|
|
const std::string& filename,
|
|
conformance::FailureSet* failure_list);
|
|
|
|
protected:
|
|
// Test cases are classified into a few categories:
|
|
// REQUIRED: the test case must be passed for an implementation to be
|
|
// interoperable with other implementations. For example, a
|
|
// parser implementation must accept both packed and unpacked
|
|
// form of repeated primitive fields.
|
|
// RECOMMENDED: the test case is not required for the implementation to
|
|
// be interoperable with other implementations, but is
|
|
// recommended for best performance and compatibility. For
|
|
// example, a proto3 serializer should serialize repeated
|
|
// primitive fields in packed form, but an implementation
|
|
// failing to do so will still be able to communicate with
|
|
// other implementations.
|
|
enum ConformanceLevel {
|
|
REQUIRED = 0,
|
|
RECOMMENDED = 1,
|
|
};
|
|
|
|
class ConformanceRequestSetting {
|
|
public:
|
|
ConformanceRequestSetting(ConformanceLevel level,
|
|
conformance::WireFormat input_format,
|
|
conformance::WireFormat output_format,
|
|
conformance::TestCategory test_category,
|
|
const Message& prototype_message,
|
|
const std::string& test_name,
|
|
const std::string& input);
|
|
virtual ~ConformanceRequestSetting() {}
|
|
|
|
std::unique_ptr<Message> NewTestMessage() const;
|
|
|
|
std::string GetTestName() const;
|
|
|
|
const conformance::ConformanceRequest& GetRequest() const {
|
|
return request_;
|
|
}
|
|
|
|
const ConformanceLevel GetLevel() const {
|
|
return level_;
|
|
}
|
|
|
|
std::string ConformanceLevelToString(ConformanceLevel level) const;
|
|
|
|
void SetPrintUnknownFields(bool print_unknown_fields) {
|
|
request_.set_print_unknown_fields(true);
|
|
}
|
|
|
|
void SetPrototypeMessageForCompare(const Message& message) {
|
|
prototype_message_for_compare_.reset(message.New());
|
|
}
|
|
|
|
protected:
|
|
virtual std::string InputFormatString(conformance::WireFormat format) const;
|
|
virtual std::string OutputFormatString(
|
|
conformance::WireFormat format) const;
|
|
conformance::ConformanceRequest request_;
|
|
|
|
private:
|
|
ConformanceLevel level_;
|
|
::conformance::WireFormat input_format_;
|
|
::conformance::WireFormat output_format_;
|
|
const Message& prototype_message_;
|
|
std::unique_ptr<Message> prototype_message_for_compare_;
|
|
std::string test_name_;
|
|
};
|
|
|
|
bool CheckSetEmpty(const std::set<std::string>& set_to_check,
|
|
const std::string& write_to_file, const std::string& msg);
|
|
std::string WireFormatToString(conformance::WireFormat wire_format);
|
|
|
|
// Parse payload in the response to the given message. Returns true on
|
|
// success.
|
|
virtual bool ParseResponse(
|
|
const conformance::ConformanceResponse& response,
|
|
const ConformanceRequestSetting& setting,
|
|
Message* test_message) = 0;
|
|
|
|
void VerifyResponse(const ConformanceRequestSetting& setting,
|
|
const std::string& equivalent_wire_format,
|
|
const conformance::ConformanceResponse& response,
|
|
bool need_report_success, bool require_same_wire_format);
|
|
|
|
void ReportSuccess(const std::string& test_name);
|
|
void ReportFailure(const std::string& test_name, ConformanceLevel level,
|
|
const conformance::ConformanceRequest& request,
|
|
const conformance::ConformanceResponse& response,
|
|
const char* fmt, ...);
|
|
void ReportSkip(const std::string& test_name,
|
|
const conformance::ConformanceRequest& request,
|
|
const conformance::ConformanceResponse& response);
|
|
|
|
void RunValidInputTest(const ConformanceRequestSetting& setting,
|
|
const std::string& equivalent_text_format);
|
|
void RunValidBinaryInputTest(const ConformanceRequestSetting& setting,
|
|
const std::string& equivalent_wire_format,
|
|
bool require_same_wire_format = false);
|
|
|
|
void RunTest(const std::string& test_name,
|
|
const conformance::ConformanceRequest& request,
|
|
conformance::ConformanceResponse* response);
|
|
|
|
void AddExpectedFailedTest(const std::string& test_name);
|
|
|
|
virtual void RunSuiteImpl() = 0;
|
|
|
|
ConformanceTestRunner* runner_;
|
|
int successes_;
|
|
int expected_failures_;
|
|
bool verbose_;
|
|
bool enforce_recommended_;
|
|
std::string output_;
|
|
std::string output_dir_;
|
|
std::string failure_list_flag_name_;
|
|
std::string failure_list_filename_;
|
|
|
|
// The set of test names that are expected to fail in this run, but haven't
|
|
// failed yet.
|
|
std::set<std::string> expected_to_fail_;
|
|
|
|
// The set of test names that have been run. Used to ensure that there are no
|
|
// duplicate names in the suite.
|
|
std::set<std::string> test_names_;
|
|
|
|
// The set of tests that failed, but weren't expected to.
|
|
std::set<std::string> unexpected_failing_tests_;
|
|
|
|
// The set of tests that succeeded, but weren't expected to.
|
|
std::set<std::string> unexpected_succeeding_tests_;
|
|
|
|
// The set of tests that the testee opted out of;
|
|
std::set<std::string> skipped_;
|
|
};
|
|
|
|
} // namespace protobuf
|
|
} // namespace google
|
|
|
|
#endif // CONFORMANCE_CONFORMANCE_TEST_H
|