ddl2cpp command line argument for custom types (#491)

* ddl2cpp command line argument for custom types

- Updated the ddl2cpp script to allow custom/extended types through external csv file
- Had to re-order the script to allow the command line to be parsed before setting up the parser
- Updated README

* Test for the command line argument

- Script test only for now

* Test the custom type argument

- Firs a negative test
- Last a positive test and compile test against the generated output

* Expand the test

- Ensure built in types still work
- Check capitilisation
- Ensure more than one custom works
- Check type with spaces

---------

Co-authored-by: Carel Combrink <carel.combrink@vastech.co.za>

README.md

@@ -239,12 +239,46 @@ Create headers for them with provided Python script:
 ```
 %sqlpp11_dir%/scripts/ddl2cpp ~/temp/MyTable.ddl  ~/temp/MyTable %DatabaseNamespaceForExample%
 ```
-(In case you’re getting notes about unsupported column type take a look at the other datatypes in sqlpp11/data_types. They are not hard to implement.)
+
+In case you’re getting notes about unsupported column type consider:
+  - Take a look at the other datatypes in sqlpp11/data_types. They are not hard to implement.
+  - Use the `--datatype-file` command line argument as described below.
 
 Include generated header (MyTable.h), that’s all.
 
 If you prefer Ruby over Python, you might want to take a look at https://github.com/douyw/sqlpp11gen
 
+Unsupported column types:
+-------------
+__Map unsupported column types to supported column types with a csv file__:
+
+One can use the `--datatype-file` command line argument for the ddl2cpp script to map unsupported column types to supported column types.
+
+The format of the csv file is:
+```
+<dataType>, <col_type1>, <col_type2>
+<dataType>, <col_type3>
+```
+
+Where `<dataType>` is one or more of the following internal types:
+
+  - `Boolean`
+  - `Integer`
+  - `Serial`
+  - `FloatingPoint`
+  - `Text`
+  - `Blob`
+  - `Date`
+  - `DateTime`
+  - `Time`
+
+Example:
+
+```
+Boolean, one_or_zero
+Text, url, uuid
+```
+
 Contact:
 --------
   * Issues at https://github.com/rbock/sqlpp11/issues

scripts/ddl2cpp

@@ -71,16 +71,12 @@ ddlBracedExpression << ddlLeft + ddlExpression + ddlRight
 ddlArguments = pp.Suppress(pp.Group(pp.delimitedList(ddlExpression)))
 ddlFunctionCall << ddlName + ddlLeft + pp.Optional(ddlArguments) + ddlRight
 
-# Column and constraint parsers
+# Data types
 ddlBooleanTypes = [
     "bool",
     "boolean",
 ]
 
-ddlBoolean = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlBooleanTypes, reverse=True))
-).setParseAction(pp.replaceWith("boolean"))
-
 ddlIntegerTypes = [
     "bigint",
     "int",
@@ -92,20 +88,12 @@ ddlIntegerTypes = [
     "smallint",
     "tinyint",
 ]
-ddlInteger = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlIntegerTypes, reverse=True))
-).setParseAction(pp.replaceWith("integer"))
 
 ddlSerialTypes = [
     "bigserial",  # PostgreSQL
     "serial",  # PostgreSQL
     "smallserial",  # PostgreSQL
 ]
-ddlSerial = (
-    pp.Or(map(pp.CaselessLiteral, sorted(ddlSerialTypes, reverse=True)))
-    .setParseAction(pp.replaceWith("integer"))
-    .setResultsName("hasAutoValue")
-)
 
 ddlFloatingPointTypes = [
     "decimal",  # MYSQL
@@ -116,9 +104,6 @@ ddlFloatingPointTypes = [
     "numeric",  # PostgreSQL
     "real",
 ]
-ddlFloatingPoint = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlFloatingPointTypes, reverse=True))
-).setParseAction(pp.replaceWith("floating_point"))
 
 ddlTextTypes = [
     "char",
@@ -136,10 +121,6 @@ ddlTextTypes = [
     "rational", # PostgreSQL pg_rationale extension
 ]
 
-ddlText = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlTextTypes, reverse=True))
-).setParseAction(pp.replaceWith("text"))
-
 ddlBlobTypes = [
     "bytea",
     "tinyblob",
@@ -150,21 +131,10 @@ ddlBlobTypes = [
     "varbinary",  # MYSQL
 ]
 
-ddlBlob = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlBlobTypes, reverse=True))
-).setParseAction(pp.replaceWith("blob"))
-
-
 ddlDateTypes = [
     "date",
 ]
 
-ddlDate = (
-    pp.Or(map(pp.CaselessLiteral, sorted(ddlDateTypes, reverse=True)))
-    .setParseAction(pp.replaceWith("day_point"))
-    .setResultsName("warnTimezone")
-)
-
 ddlDateTimeTypes = [
     "datetime",
     "timestamp",
@@ -173,21 +143,62 @@ ddlDateTimeTypes = [
     "timestamptz",  # PostgreSQL
 ]
 
-ddlDateTime = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlDateTimeTypes, reverse=True))
-).setParseAction(pp.replaceWith("time_point"))
-
 ddlTimeTypes = [
     "time",
     "time without time zone",  # PostgreSQL
     "time with time zone",  # PostgreSQL
 ]
 
+# Init the DLL parser
+def initDllParser():
+    global ddl
+    global ddlType
+    global ddlColumn
+    global ddlConstraint
+    global ddlCreateTable
+    # Column and constraint parsers
+
+    ddlBoolean = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlBooleanTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("boolean"))
+
+    ddlInteger = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlIntegerTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("integer"))
+
+    ddlSerial = (
+        pp.Or(map(pp.CaselessLiteral, sorted(ddlSerialTypes, reverse=True)))
+        .setParseAction(pp.replaceWith("integer"))
+        .setResultsName("hasAutoValue")
+    )
+
+    ddlFloatingPoint = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlFloatingPointTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("floating_point"))
+
+    ddlText = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlTextTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("text"))
+
+
+    ddlBlob = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlBlobTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("blob"))
+
+    ddlDate = (
+        pp.Or(map(pp.CaselessLiteral, sorted(ddlDateTypes, reverse=True)))
+        .setParseAction(pp.replaceWith("day_point"))
+        .setResultsName("warnTimezone")
+    )
+
+    ddlDateTime = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlDateTimeTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("time_point"))
+
     ddlTime = pp.Or(
         map(pp.CaselessLiteral, sorted(ddlTimeTypes, reverse=True))
     ).setParseAction(pp.replaceWith("time_of_day"))
 
-
     ddlUnknown = pp.Word(pp.alphanums).setParseAction(pp.replaceWith("UNKNOWN"))
 
     ddlType = (
@@ -285,7 +296,6 @@ ddl = pp.OneOrMore(pp.Suppress(pp.SkipTo(ddlCreateTable, False)) + ddlCreateTabl
     ddlComment = pp.oneOf(["--", "#"]) + pp.restOfLine
     ddl.ignore(ddlComment)
 
-
 def testBoolean():
     for t in ddlBooleanTypes:
         result = ddlType.parseString(t, parseAll=True)
@@ -423,6 +433,7 @@ def testPrimaryKeyAutoIncrement():
         assert column.hasAutoValue
 
 def testParser():
+    initDllParser()
     testBoolean()
     testInteger()
     testSerial()
@@ -441,10 +452,8 @@ def testParser():
     testPrimaryKeyAutoIncrement()
 
 
-# CODE GENERATOR
+
 # HELPERS
-
-
 def get_include_guard_name(namespace, inputfile):
     val = re.sub("[^A-Za-z0-9]+", "_", namespace + "_" + os.path.basename(inputfile))
     return val.upper()
@@ -485,6 +494,15 @@ def setArgumentBool(s, bool_value):
     var_name = first_lower(re.sub("(\s|-|[0-9])(\S)", repl_func_for_args, s))
     globals()[var_name] = bool_value
 
+def loadExtendedTypesFile(filename):
+    import csv
+    with open(filename, newline='') as csvfile:
+        reader = csv.DictReader(csvfile, fieldnames=["baseType"], restkey="extendedTypes", delimiter=',')
+        for row in reader:
+            var_values = [clean_val for value in row['extendedTypes'] if (clean_val := value.strip(" \"'"))]
+            if var_values:
+                var_name = f"ddl{row['baseType']}Types"
+                globals()[var_name].extend(var_values)
 
 def escape_if_reserved(name):
     reserved_names = [
@@ -524,6 +542,10 @@ def endHeader(header, nsList):
 def help_message():
     arg_string = ""
     pad = 0
+
+    # The dataTypeFileArg is handled differently from the normal optionalArgs
+    # and only added to the list here to make use of the formatting of the help.
+    optionalArgs[dataTypeFileArg] = f"path to a csv that contains custom datatype mappings. The format is '{dataTypeFileArg}=path/to/file.csv' (See the README)."
     for argument in list(optionalArgs.keys()):
         if len(argument) > pad:
             pad = len(argument)
@@ -560,7 +582,7 @@ noTimestampWarning = False
 autoId = False
 identityNaming = False
 splitTables = False
-
+dataTypeFileArg = "--datatype-file"
 
 def createHeader():
     global noTimestampWarning
@@ -575,6 +597,9 @@ def createHeader():
             if arg in list(optionalArgs.keys()):
                 setArgumentBool(arg, True)
                 firstPositional += 1
+            if dataTypeFileArg in arg:
+                loadExtendedTypesFile(arg.split('=')[1])
+                firstPositional += 1
             else:
                 pass
 
@@ -590,6 +615,8 @@ def createHeader():
     pathToHeader = sys.argv[firstPositional + 1] + ("/" if splitTables else ".h")
     namespace = sys.argv[firstPositional + 2]
 
+    initDllParser()
+
     try:
         tableCreations = ddl.parseFile(pathToDdl)
     except pp.ParseException as e:
@@ -738,8 +765,9 @@ def createHeader():
         print("Error: unsupported datatypes.")
         print("Possible solutions:")
         print("A) Implement this datatype (examples: sqlpp11/data_types)")
-        print("B) Extend/upgrade ddl2cpp (edit types map)")
+        print(f"B) Use the '{dataTypeFileArg}' command line argument to map the type to a known type (example: README)")
-        print("C) Raise an issue on github")
+        print("C) Extend/upgrade ddl2cpp (edit types map)")
+        print("D) Raise an issue on github")
         sys.exit(10)  # return non-zero error code, we might need it for automation
 
 

tests/scripts/CMakeLists.txt

@@ -84,5 +84,30 @@ if (${Python3_Interpreter_FOUND})
                 "${sqlpp.scripts.generated.sample.include}.h")
             target_link_libraries(sqlpp.scripts.compiled.${sample_name} PRIVATE sqlpp11)
         endforeach()
+
+        set(custom_type_sql "ddl2cpp_sample_good_custom_type")
+        include_directories(${CMAKE_CURRENT_BINARY_DIR})
+        add_test(NAME sqlpp11.scripts.ddl2cpp.bad_custom_types
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/../../scripts/ddl2cpp"
+                    "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+                    "${CMAKE_CURRENT_BINARY_DIR}/fail"
+                    test)
+        set_tests_properties(sqlpp11.scripts.ddl2cpp.bad_custom_types PROPERTIES
+            PASS_REGULAR_EXPRESSION "Error: unsupported datatypes.")
+
+        set(sqlpp.scripts.generated.custom_type_sql.include "${CMAKE_CURRENT_BINARY_DIR}/${custom_type_sql}")
+        add_custom_command(
+            OUTPUT "${sqlpp.scripts.generated.custom_type_sql.include}.h"
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/../../scripts/ddl2cpp"
+                    "--datatype-file=${CMAKE_CURRENT_LIST_DIR}/custom_types.csv"
+                    "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+                    "${sqlpp.scripts.generated.custom_type_sql.include}"
+                    test
+            DEPENDS "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+            VERBATIM)
+
+        add_executable(sqlpp.scripts.compiled.${custom_type_sql} ${custom_type_sql}.cpp
+            "${sqlpp.scripts.generated.custom_type_sql.include}.h")
+        target_link_libraries(sqlpp.scripts.compiled.${custom_type_sql} PRIVATE sqlpp11)
     endif()
 endif()

tests/scripts/custom_types.csv

@@ -0,0 +1,9 @@
+Boolean, CustomBooleanType
+Integer, CustomIntegerType, SPECIAL INT
+Serial, CustomSerialType
+FloatingPoint, CustomFloatingPointType
+Text, CustomTextType, another_text_type
+Blob, CustomBlobType
+Date, CustomDateType
+DateTime, CustomDateTimeType
+Time, CustomTimeType

tests/scripts/ddl2cpp_sample_good_custom_type.cpp

@@ -0,0 +1,32 @@
+
+#include <sqlpp11/chrono.h>
+#include <ddl2cpp_sample_good_custom_type.h>
+
+int main()
+{
+  test::TabFoo tab_foo;
+  tab_foo.myBoolean = true;
+  tab_foo.myInteger = 5;
+  tab_foo.mySerial = 10;
+  tab_foo.myFloatingPoint = 12.34;
+  tab_foo.myText = "test";
+  tab_foo.myBlob = "blob";
+  tab_foo.myDate = sqlpp::chrono::day_point{};
+  tab_foo.myDateTime = std::chrono::system_clock::now();
+  tab_foo.myTime = std::chrono::seconds{10};
+  // Special cases
+  tab_foo.mySecondText = "another text";
+  tab_foo.myTypeWithSpaces = 20;
+  // Capitalisation
+  tab_foo.capBoolean = false;
+  // Build in types
+  tab_foo.builtinBoolean = true;
+  tab_foo.builtinInteger = 5;
+  tab_foo.builtinSerial = 10;
+  tab_foo.builtinFloatingPoint = 12.34;
+  tab_foo.builtinText = "test";
+  tab_foo.builtinBlob = "blob";
+  tab_foo.builtinDate = sqlpp::chrono::day_point{};
+  tab_foo.builtinDateTime = std::chrono::system_clock::now();
+  tab_foo.builtinTime = std::chrono::seconds{10};
+}

tests/scripts/ddl2cpp_sample_good_custom_type.sql

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2013-2015, Roland Bock
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+CREATE TABLE tab_foo
+(
+	myBoolean CustomBooleanType,
+	myInteger CustomIntegerType,
+	mySerial CustomSerialType,
+	myFloatingPoint CustomFloatingPointType,
+	myText CustomTextType,
+	myBlob CustomBlobType,
+	myDate CustomDateType,
+	myDateTime CustomDateTimeType,
+	myTime CustomTimeType,
+	-- Some more special cases
+	mySecondText another_text_type,
+	myTypeWithSpaces SPECIAL INT,
+	-- Checking capitalisation of types
+	capBoolean CUSTOMBOOLEANTYPE,
+	-- Ensuring built in types still function
+	builtinBoolean BOOLEAN,
+	builtinInteger INTEGER,
+	builtinSerial SERIAL,
+	builtinFloatingPoint NUMERIC,
+	builtinText TEXT,
+	builtinBlob BINARY,
+	builtinDate DATE,
+	builtinDateTime TIMESTAMPTZ,
+	builtinTime TIME WITH TIME ZONE
+
+) WITH SYSTEM VERSIONING; -- enable System-Versioning for this table