From 1ff6bb65a0a845dfe4e29fef01f5f42ddfe4f67a Mon Sep 17 00:00:00 2001 From: Christopher Dunn Date: Fri, 24 Apr 2020 00:58:35 -0500 Subject: [PATCH 1/5] ninja test --- CONTRIBUTING.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08428b6..9c9fc6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,8 +27,13 @@ Then, #LIB_TYPE=static meson --buildtype ${BUILD_TYPE} --default-library ${LIB_TYPE} . build-${LIB_TYPE} ninja -v -C build-${LIB_TYPE} - cd build-${LIB_TYPE} - meson test --no-rebuild --print-errorlogs + + ninja -C build-static/ test + + # Or + #cd build-${LIB_TYPE} + #meson test --no-rebuild --print-errorlogs + sudo ninja install ## Building and testing with other build systems From 411d88fae895d7d2dbc06dcccc80d4f747746c2b Mon Sep 17 00:00:00 2001 From: Christopher Dunn Date: Fri, 24 Apr 2020 01:45:19 -0500 Subject: [PATCH 2/5] Stop checking status; raise instead --- test/runjsontests.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/runjsontests.py b/test/runjsontests.py index dfdeca3..26caf0c 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -62,6 +62,10 @@ def safeReadFile(path): except IOError as e: return '' % (path,e) +class FailError(Exception): + def __init__(self, msg): + super(Exception, self).__init__(msg) + def runAllTests(jsontest_executable_path, input_dir = None, use_valgrind=False, with_json_checker=False, writerClass='StyledWriter'): @@ -161,10 +165,9 @@ def runAllTests(jsontest_executable_path, input_dir = None, print() print('Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), len(failed_tests))) - return 1 + raise FailError(repr(failed_tests)) else: print('All %d tests passed.' % len(tests)) - return 0 def main(): from optparse import OptionParser @@ -187,24 +190,21 @@ def main(): input_path = os.path.normpath(os.path.abspath(args[1])) else: input_path = None - status = runAllTests(jsontest_executable_path, input_path, + runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, writerClass='StyledWriter') - if status: - sys.exit(status) - status = runAllTests(jsontest_executable_path, input_path, + runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, writerClass='StyledStreamWriter') - if status: - sys.exit(status) - status = runAllTests(jsontest_executable_path, input_path, + runAllTests(jsontest_executable_path, input_path, use_valgrind=options.valgrind, with_json_checker=options.with_json_checker, writerClass='BuiltStyledStreamWriter') - if status: - sys.exit(status) if __name__ == '__main__': - main() + try: + main() + except FailError: + sys.exit(1) From 9e23f66f614d23c4c2b540d2d62ce3b2725d422a Mon Sep 17 00:00:00 2001 From: Jordan Bayles Date: Thu, 14 Nov 2019 10:41:25 -0800 Subject: [PATCH 3/5] Issue 1102: Fixup test suite, fix broken tests A recent PR broken the JsonChecker tests by adding support for trailing commas. This didn't end up breaking the build, because those tests aren't run, except locally and only using CMake. This patch fixes the tests by adding exclusions for trailing comma tests, as well as updates Meson to run these tests as part of `ninja test`. See issue #1102. --- meson.build | 10 +++++++++ test/runjsontests.py | 53 ++++++++++++++------------------------------ 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/meson.build b/meson.build index c293203..e2e92ad 100644 --- a/meson.build +++ b/meson.build @@ -105,3 +105,13 @@ test( jsontestrunner, join_paths(meson.current_source_dir(), 'test/data')] ) +test( + 'jsonchecker_jsontestrunner', + python, + args : [ + '-B', + join_paths(meson.current_source_dir(), 'test/runjsontests.py'), + '--with-json-checker', + jsontestrunner, + join_paths(meson.current_source_dir(), 'test/data')] + ) diff --git a/test/runjsontests.py b/test/runjsontests.py index 26caf0c..fb1fb39 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -73,45 +73,26 @@ def runAllTests(jsontest_executable_path, input_dir = None, input_dir = os.path.join(os.getcwd(), 'data') tests = glob(os.path.join(input_dir, '*.json')) if with_json_checker: - all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) - # These tests fail with strict json support, but pass with jsoncpp extra lieniency - """ - Failure details: - * Test ../jsonchecker/fail25.json - Parsing should have failed: - [" tab character in string "] - - * Test ../jsonchecker/fail13.json - Parsing should have failed: - {"Numbers cannot have leading zeroes": 013} - - * Test ../jsonchecker/fail18.json - Parsing should have failed: - [[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]] - - * Test ../jsonchecker/fail8.json - Parsing should have failed: - ["Extra close"]] - - * Test ../jsonchecker/fail7.json - Parsing should have failed: - ["Comma after the close"], - - * Test ../jsonchecker/fail10.json - Parsing should have failed: - {"Extra value after close": true} "misplaced quoted value" - - * Test ../jsonchecker/fail27.json - Parsing should have failed: - ["line - break"] - """ - known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json", - "fail7.json", "fail10.json", "fail27.json" ] - test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ] + all_tests = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) + # These tests fail with strict json support, but pass with JsonCPP's + # extra leniency features. When adding a new exclusion to this list, + # remember to add the test's number and reasoning here: + known = ["fail{}.json".format(n) for n in [ + 4, 9, # fail because we allow trailing commas + 7, # fails because we allow commas after close + 8, # fails because we allow extra close + 10, # fails because we allow extra values after close + 13, # fails because we allow leading zeroes in numbers + 18, # fails because we allow deeply nested values + 25, # fails because we allow tab characters in strings. + 27, # fails because we allow string line breaks + ]] + test_jsonchecker = [ test for test in all_tests + if os.path.basename(test) not in known] else: test_jsonchecker = [] + failed_tests = [] valgrind_path = use_valgrind and VALGRIND_CMD or '' for input_path in tests + test_jsonchecker: From b3492219385ba41f9f8167a68e3f56a2b6cdd7a9 Mon Sep 17 00:00:00 2001 From: Jordan Bayles Date: Thu, 14 Nov 2019 10:52:13 -0800 Subject: [PATCH 4/5] Cleanup test configurations --- meson.build | 5 +++-- test/runjsontests.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/meson.build b/meson.build index e2e92ad..7bacdfc 100644 --- a/meson.build +++ b/meson.build @@ -103,7 +103,7 @@ test( '-B', join_paths(meson.current_source_dir(), 'test/runjsontests.py'), jsontestrunner, - join_paths(meson.current_source_dir(), 'test/data')] + join_paths(meson.current_source_dir(), 'test/data')], ) test( 'jsonchecker_jsontestrunner', @@ -113,5 +113,6 @@ test( join_paths(meson.current_source_dir(), 'test/runjsontests.py'), '--with-json-checker', jsontestrunner, - join_paths(meson.current_source_dir(), 'test/data')] + join_paths(meson.current_source_dir(), 'test/data')], + workdir : join_paths(meson.current_source_dir(), 'test/data'), ) diff --git a/test/runjsontests.py b/test/runjsontests.py index fb1fb39..5496e2c 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -84,7 +84,7 @@ def runAllTests(jsontest_executable_path, input_dir = None, 10, # fails because we allow extra values after close 13, # fails because we allow leading zeroes in numbers 18, # fails because we allow deeply nested values - 25, # fails because we allow tab characters in strings. + 25, # fails because we allow tab characters in strings 27, # fails because we allow string line breaks ]] test_jsonchecker = [ test for test in all_tests From a0b8c3ecb418a8d5fcf992a3250fd3634b5cb563 Mon Sep 17 00:00:00 2001 From: Christopher Dunn Date: Fri, 24 Apr 2020 09:24:22 -0500 Subject: [PATCH 5/5] Do not run colliding tests at same time --- meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/meson.build b/meson.build index 7bacdfc..975d56c 100644 --- a/meson.build +++ b/meson.build @@ -108,6 +108,7 @@ test( test( 'jsonchecker_jsontestrunner', python, + is_parallel : false, args : [ '-B', join_paths(meson.current_source_dir(), 'test/runjsontests.py'),