ddl2cpp command line argument for custom types (#491)

* ddl2cpp command line argument for custom types - Updated the ddl2cpp script to allow custom/extended types through external csv file - Had to re-order the script to allow the command line to be parsed before setting up the parser - Updated README * Test for the command line argument - Script test only for now * Test the custom type argument - Firs a negative test - Last a positive test and compile test against the generated output * Expand the test - Ensure built in types still work - Check capitilisation - Ensure more than one custom works - Check type with spaces --------- Co-authored-by: Carel Combrink <carel.combrink@vastech.co.za>
2024-11-15 12:29:41 +08:00 · 2023-06-22 07:06:00 +02:00 · 2023-06-22 07:06:00 +02:00 · babd420ecb
commit babd420ecb
parent eac9a6e5e3
6 changed files with 317 additions and 135 deletions
--- a/README.md
+++ b/README.md
@ -239,12 +239,46 @@ Create headers for them with provided Python script:
 ```
 %sqlpp11_dir%/scripts/ddl2cpp ~/temp/MyTable.ddl  ~/temp/MyTable %DatabaseNamespaceForExample%
 ```
-(In case you’re getting notes about unsupported column type take a look at the other datatypes in sqlpp11/data_types. They are not hard to implement.)
+
+In case you’re getting notes about unsupported column type consider:
+  - Take a look at the other datatypes in sqlpp11/data_types. They are not hard to implement.
+  - Use the `--datatype-file` command line argument as described below.

 Include generated header (MyTable.h), that’s all.

 If you prefer Ruby over Python, you might want to take a look at https://github.com/douyw/sqlpp11gen

+Unsupported column types:
+-------------
+__Map unsupported column types to supported column types with a csv file__:
+
+One can use the `--datatype-file` command line argument for the ddl2cpp script to map unsupported column types to supported column types.
+
+The format of the csv file is:
+```
+<dataType>, <col_type1>, <col_type2>
+<dataType>, <col_type3>
+```
+
+Where `<dataType>` is one or more of the following internal types:
+
+  - `Boolean`
+  - `Integer`
+  - `Serial`
+  - `FloatingPoint`
+  - `Text`
+  - `Blob`
+  - `Date`
+  - `DateTime`
+  - `Time`
+
+Example:
+
+```
+Boolean, one_or_zero
+Text, url, uuid
+```
+
 Contact:
 --------
  * Issues at https://github.com/rbock/sqlpp11/issues
--- a/scripts/ddl2cpp
+++ b/scripts/ddl2cpp
@ -71,16 +71,12 @@ ddlBracedExpression << ddlLeft + ddlExpression + ddlRight
 ddlArguments = pp.Suppress(pp.Group(pp.delimitedList(ddlExpression)))
 ddlFunctionCall << ddlName + ddlLeft + pp.Optional(ddlArguments) + ddlRight

-# Column and constraint parsers
+# Data types
 ddlBooleanTypes = [
    "bool",
    "boolean",
 ]

-ddlBoolean = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlBooleanTypes, reverse=True))
-).setParseAction(pp.replaceWith("boolean"))
-
 ddlIntegerTypes = [
    "bigint",
    "int",
@ -92,20 +88,12 @@ ddlIntegerTypes = [
    "smallint",
    "tinyint",
 ]
-ddlInteger = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlIntegerTypes, reverse=True))
-).setParseAction(pp.replaceWith("integer"))

 ddlSerialTypes = [
    "bigserial",  # PostgreSQL
    "serial",  # PostgreSQL
    "smallserial",  # PostgreSQL
 ]
-ddlSerial = (
-    pp.Or(map(pp.CaselessLiteral, sorted(ddlSerialTypes, reverse=True)))
-    .setParseAction(pp.replaceWith("integer"))
-    .setResultsName("hasAutoValue")
-)

 ddlFloatingPointTypes = [
    "decimal",  # MYSQL
@ -116,9 +104,6 @@ ddlFloatingPointTypes = [
    "numeric",  # PostgreSQL
    "real",
 ]
-ddlFloatingPoint = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlFloatingPointTypes, reverse=True))
-).setParseAction(pp.replaceWith("floating_point"))

 ddlTextTypes = [
    "char",
@ -136,10 +121,6 @@ ddlTextTypes = [
    "rational", # PostgreSQL pg_rationale extension
 ]

-ddlText = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlTextTypes, reverse=True))
-).setParseAction(pp.replaceWith("text"))
-
 ddlBlobTypes = [
    "bytea",
    "tinyblob",
@ -150,21 +131,10 @@ ddlBlobTypes = [
    "varbinary",  # MYSQL
 ]

-ddlBlob = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlBlobTypes, reverse=True))
-).setParseAction(pp.replaceWith("blob"))
-
-
 ddlDateTypes = [
    "date",
 ]

-ddlDate = (
-    pp.Or(map(pp.CaselessLiteral, sorted(ddlDateTypes, reverse=True)))
-    .setParseAction(pp.replaceWith("day_point"))
-    .setResultsName("warnTimezone")
-)
-
 ddlDateTimeTypes = [
    "datetime",
    "timestamp",
@ -173,24 +143,65 @@ ddlDateTimeTypes = [
    "timestamptz",  # PostgreSQL
 ]

-ddlDateTime = pp.Or(
-    map(pp.CaselessLiteral, sorted(ddlDateTimeTypes, reverse=True))
-).setParseAction(pp.replaceWith("time_point"))
-
 ddlTimeTypes = [
    "time",
    "time without time zone",  # PostgreSQL
    "time with time zone",  # PostgreSQL
 ]

-ddlTime = pp.Or(
+# Init the DLL parser
+def initDllParser():
+    global ddl
+    global ddlType
+    global ddlColumn
+    global ddlConstraint
+    global ddlCreateTable
+    # Column and constraint parsers
+
+    ddlBoolean = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlBooleanTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("boolean"))
+
+    ddlInteger = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlIntegerTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("integer"))
+
+    ddlSerial = (
+        pp.Or(map(pp.CaselessLiteral, sorted(ddlSerialTypes, reverse=True)))
+        .setParseAction(pp.replaceWith("integer"))
+        .setResultsName("hasAutoValue")
+    )
+
+    ddlFloatingPoint = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlFloatingPointTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("floating_point"))
+
+    ddlText = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlTextTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("text"))
+
+
+    ddlBlob = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlBlobTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("blob"))
+
+    ddlDate = (
+        pp.Or(map(pp.CaselessLiteral, sorted(ddlDateTypes, reverse=True)))
+        .setParseAction(pp.replaceWith("day_point"))
+        .setResultsName("warnTimezone")
+    )
+
+    ddlDateTime = pp.Or(
+        map(pp.CaselessLiteral, sorted(ddlDateTimeTypes, reverse=True))
+    ).setParseAction(pp.replaceWith("time_point"))
+
+    ddlTime = pp.Or(
        map(pp.CaselessLiteral, sorted(ddlTimeTypes, reverse=True))
-).setParseAction(pp.replaceWith("time_of_day"))
+    ).setParseAction(pp.replaceWith("time_of_day"))

+    ddlUnknown = pp.Word(pp.alphanums).setParseAction(pp.replaceWith("UNKNOWN"))

-ddlUnknown = pp.Word(pp.alphanums).setParseAction(pp.replaceWith("UNKNOWN"))
-
-ddlType = (
+    ddlType = (
        ddlBoolean
        | ddlInteger
        | ddlSerial
@ -201,33 +212,33 @@ ddlType = (
        | ddlDate
        | ddlTime
        | ddlUnknown
-)
+    )

-ddlUnsigned = pp.CaselessLiteral("UNSIGNED").setResultsName("isUnsigned")
-ddlDigits = "," + pp.Word(pp.nums)
-ddlWidth = ddlLeft + pp.Word(pp.nums) + pp.Optional(ddlDigits) + ddlRight
-ddlTimezone = (
+    ddlUnsigned = pp.CaselessLiteral("UNSIGNED").setResultsName("isUnsigned")
+    ddlDigits = "," + pp.Word(pp.nums)
+    ddlWidth = ddlLeft + pp.Word(pp.nums) + pp.Optional(ddlDigits) + ddlRight
+    ddlTimezone = (
        (pp.CaselessLiteral("with") | pp.CaselessLiteral("without"))
        + pp.CaselessLiteral("time")
        + pp.CaselessLiteral("zone")
-)
+    )

-ddlNotNull = pp.Group(
+    ddlNotNull = pp.Group(
        pp.CaselessLiteral("NOT") + pp.CaselessLiteral("NULL")
-).setResultsName("notNull")
-ddlDefaultValue = pp.CaselessLiteral("DEFAULT").setResultsName("hasDefaultValue")
+    ).setResultsName("notNull")
+    ddlDefaultValue = pp.CaselessLiteral("DEFAULT").setResultsName("hasDefaultValue")

-ddlAutoKeywords = [
+    ddlAutoKeywords = [
        "AUTO_INCREMENT",
        "AUTOINCREMENT",
        "SMALLSERIAL",
        "SERIAL",
        "BIGSERIAL",
        "GENERATED",
-]
-ddlAutoValue = pp.Or(map(pp.CaselessLiteral, sorted(ddlAutoKeywords, reverse=True)))
+    ]
+    ddlAutoValue = pp.Or(map(pp.CaselessLiteral, sorted(ddlAutoKeywords, reverse=True)))

-ddlConstraintKeywords = [
+    ddlConstraintKeywords = [
        "CONSTRAINT",
        "PRIMARY",
        "FOREIGN",
@ -237,13 +248,13 @@ ddlConstraintKeywords = [
        "UNIQUE",
        "CHECK",
        "PERIOD",
-]
-ddlConstraint = pp.Group(
+    ]
+    ddlConstraint = pp.Group(
        pp.Or(map(pp.CaselessLiteral, sorted(ddlConstraintKeywords, reverse=True)))
        + ddlExpression
-).setResultsName("isConstraint")
+    ).setResultsName("isConstraint")

-ddlColumn = pp.Group(
+    ddlColumn = pp.Group(
        ddlName("name")
        + ddlType("type")
        + pp.Suppress(pp.Optional(ddlWidth))
@ -257,16 +268,16 @@ ddlColumn = pp.Group(
            | pp.Suppress(pp.OneOrMore(pp.Or(map(pp.CaselessLiteral, sorted(ddlConstraintKeywords, reverse=True)))))
            | pp.Suppress(ddlExpression)
        )
-)
+    )

-# CREATE TABLE parser
-ddlIfNotExists = pp.Group(
+    # CREATE TABLE parser
+    ddlIfNotExists = pp.Group(
        pp.CaselessLiteral("IF") + pp.CaselessLiteral("NOT") + pp.CaselessLiteral("EXISTS")
-).setResultsName("ifNotExists")
-ddlOrReplace = pp.Group(
+    ).setResultsName("ifNotExists")
+    ddlOrReplace = pp.Group(
        pp.CaselessLiteral("OR") + pp.CaselessLiteral("REPLACE")
-).setResultsName("orReplace")
-ddlCreateTable = pp.Group(
+    ).setResultsName("orReplace")
+    ddlCreateTable = pp.Group(
        pp.CaselessLiteral("CREATE")
        + pp.Suppress(pp.Optional(ddlOrReplace))
        + pp.CaselessLiteral("TABLE")
@ -277,14 +288,13 @@ ddlCreateTable = pp.Group(
            "columns"
        )
        + ddlRight
-).setResultsName("create")
-# ddlString.setDebug(True) #uncomment to debug pyparsing
+    ).setResultsName("create")
+    # ddlString.setDebug(True) #uncomment to debug pyparsing

-ddl = pp.OneOrMore(pp.Suppress(pp.SkipTo(ddlCreateTable, False)) + ddlCreateTable)
-
-ddlComment = pp.oneOf(["--", "#"]) + pp.restOfLine
-ddl.ignore(ddlComment)
+    ddl = pp.OneOrMore(pp.Suppress(pp.SkipTo(ddlCreateTable, False)) + ddlCreateTable)

+    ddlComment = pp.oneOf(["--", "#"]) + pp.restOfLine
+    ddl.ignore(ddlComment)

 def testBoolean():
    for t in ddlBooleanTypes:
@ -423,6 +433,7 @@ def testPrimaryKeyAutoIncrement():
        assert column.hasAutoValue

 def testParser():
+    initDllParser()
    testBoolean()
    testInteger()
    testSerial()
@ -441,10 +452,8 @@ def testParser():
    testPrimaryKeyAutoIncrement()


-# CODE GENERATOR
+
 # HELPERS
-
-
 def get_include_guard_name(namespace, inputfile):
    val = re.sub("[^A-Za-z0-9]+", "_", namespace + "_" + os.path.basename(inputfile))
    return val.upper()
@ -485,6 +494,15 @@ def setArgumentBool(s, bool_value):
    var_name = first_lower(re.sub("(\s|-|[0-9])(\S)", repl_func_for_args, s))
    globals()[var_name] = bool_value

+def loadExtendedTypesFile(filename):
+    import csv
+    with open(filename, newline='') as csvfile:
+        reader = csv.DictReader(csvfile, fieldnames=["baseType"], restkey="extendedTypes", delimiter=',')
+        for row in reader:
+            var_values = [clean_val for value in row['extendedTypes'] if (clean_val := value.strip(" \"'"))]
+            if var_values:
+                var_name = f"ddl{row['baseType']}Types"
+                globals()[var_name].extend(var_values)

 def escape_if_reserved(name):
    reserved_names = [
@ -524,6 +542,10 @@ def endHeader(header, nsList):
 def help_message():
    arg_string = ""
    pad = 0
+
+    # The dataTypeFileArg is handled differently from the normal optionalArgs
+    # and only added to the list here to make use of the formatting of the help.
+    optionalArgs[dataTypeFileArg] = f"path to a csv that contains custom datatype mappings. The format is '{dataTypeFileArg}=path/to/file.csv' (See the README)."
    for argument in list(optionalArgs.keys()):
        if len(argument) > pad:
            pad = len(argument)
@ -560,7 +582,7 @@ noTimestampWarning = False
 autoId = False
 identityNaming = False
 splitTables = False
-
+dataTypeFileArg = "--datatype-file"

 def createHeader():
    global noTimestampWarning
@ -575,6 +597,9 @@ def createHeader():
            if arg in list(optionalArgs.keys()):
                setArgumentBool(arg, True)
                firstPositional += 1
+            if dataTypeFileArg in arg:
+                loadExtendedTypesFile(arg.split('=')[1])
+                firstPositional += 1
            else:
                pass

@ -590,6 +615,8 @@ def createHeader():
    pathToHeader = sys.argv[firstPositional + 1] + ("/" if splitTables else ".h")
    namespace = sys.argv[firstPositional + 2]

+    initDllParser()
+
    try:
        tableCreations = ddl.parseFile(pathToDdl)
    except pp.ParseException as e:
@ -738,8 +765,9 @@ def createHeader():
        print("Error: unsupported datatypes.")
        print("Possible solutions:")
        print("A) Implement this datatype (examples: sqlpp11/data_types)")
-        print("B) Extend/upgrade ddl2cpp (edit types map)")
-        print("C) Raise an issue on github")
+        print(f"B) Use the '{dataTypeFileArg}' command line argument to map the type to a known type (example: README)")
+        print("C) Extend/upgrade ddl2cpp (edit types map)")
+        print("D) Raise an issue on github")
        sys.exit(10)  # return non-zero error code, we might need it for automation


--- a/tests/scripts/CMakeLists.txt
+++ b/tests/scripts/CMakeLists.txt
@ -84,5 +84,30 @@ if (${Python3_Interpreter_FOUND})
                "${sqlpp.scripts.generated.sample.include}.h")
            target_link_libraries(sqlpp.scripts.compiled.${sample_name} PRIVATE sqlpp11)
        endforeach()
+
+        set(custom_type_sql "ddl2cpp_sample_good_custom_type")
+        include_directories(${CMAKE_CURRENT_BINARY_DIR})
+        add_test(NAME sqlpp11.scripts.ddl2cpp.bad_custom_types
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/../../scripts/ddl2cpp"
+                    "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+                    "${CMAKE_CURRENT_BINARY_DIR}/fail"
+                    test)
+        set_tests_properties(sqlpp11.scripts.ddl2cpp.bad_custom_types PROPERTIES
+            PASS_REGULAR_EXPRESSION "Error: unsupported datatypes.")
+
+        set(sqlpp.scripts.generated.custom_type_sql.include "${CMAKE_CURRENT_BINARY_DIR}/${custom_type_sql}")
+        add_custom_command(
+            OUTPUT "${sqlpp.scripts.generated.custom_type_sql.include}.h"
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/../../scripts/ddl2cpp"
+                    "--datatype-file=${CMAKE_CURRENT_LIST_DIR}/custom_types.csv"
+                    "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+                    "${sqlpp.scripts.generated.custom_type_sql.include}"
+                    test
+            DEPENDS "${CMAKE_CURRENT_LIST_DIR}/${custom_type_sql}.sql"
+            VERBATIM)
+
+        add_executable(sqlpp.scripts.compiled.${custom_type_sql} ${custom_type_sql}.cpp
+            "${sqlpp.scripts.generated.custom_type_sql.include}.h")
+        target_link_libraries(sqlpp.scripts.compiled.${custom_type_sql} PRIVATE sqlpp11)
    endif()
 endif()
--- a/tests/scripts/custom_types.csv
+++ b/tests/scripts/custom_types.csv
@ -0,0 +1,9 @@
+Boolean, CustomBooleanType
+Integer, CustomIntegerType, SPECIAL INT
+Serial, CustomSerialType
+FloatingPoint, CustomFloatingPointType
+Text, CustomTextType, another_text_type
+Blob, CustomBlobType
+Date, CustomDateType
+DateTime, CustomDateTimeType
+Time, CustomTimeType
--- a/tests/scripts/ddl2cpp_sample_good_custom_type.cpp
+++ b/tests/scripts/ddl2cpp_sample_good_custom_type.cpp
@ -0,0 +1,32 @@
+
+#include <sqlpp11/chrono.h>
+#include <ddl2cpp_sample_good_custom_type.h>
+
+int main()
+{
+  test::TabFoo tab_foo;
+  tab_foo.myBoolean = true;
+  tab_foo.myInteger = 5;
+  tab_foo.mySerial = 10;
+  tab_foo.myFloatingPoint = 12.34;
+  tab_foo.myText = "test";
+  tab_foo.myBlob = "blob";
+  tab_foo.myDate = sqlpp::chrono::day_point{};
+  tab_foo.myDateTime = std::chrono::system_clock::now();
+  tab_foo.myTime = std::chrono::seconds{10};
+  // Special cases
+  tab_foo.mySecondText = "another text";
+  tab_foo.myTypeWithSpaces = 20;
+  // Capitalisation
+  tab_foo.capBoolean = false;
+  // Build in types
+  tab_foo.builtinBoolean = true;
+  tab_foo.builtinInteger = 5;
+  tab_foo.builtinSerial = 10;
+  tab_foo.builtinFloatingPoint = 12.34;
+  tab_foo.builtinText = "test";
+  tab_foo.builtinBlob = "blob";
+  tab_foo.builtinDate = sqlpp::chrono::day_point{};
+  tab_foo.builtinDateTime = std::chrono::system_clock::now();
+  tab_foo.builtinTime = std::chrono::seconds{10};
+}
--- a/tests/scripts/ddl2cpp_sample_good_custom_type.sql
+++ b/tests/scripts/ddl2cpp_sample_good_custom_type.sql
@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2013-2015, Roland Bock
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *   Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above copyright notice, this
+ *   list of conditions and the following disclaimer in the documentation and/or
+ *   other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+CREATE TABLE tab_foo
+(
+	myBoolean CustomBooleanType,
+	myInteger CustomIntegerType,
+	mySerial CustomSerialType,
+	myFloatingPoint CustomFloatingPointType,
+	myText CustomTextType,
+	myBlob CustomBlobType,
+	myDate CustomDateType,
+	myDateTime CustomDateTimeType,
+	myTime CustomTimeType,
+	-- Some more special cases
+	mySecondText another_text_type,
+	myTypeWithSpaces SPECIAL INT,
+	-- Checking capitalisation of types
+	capBoolean CUSTOMBOOLEANTYPE,
+	-- Ensuring built in types still function
+	builtinBoolean BOOLEAN,
+	builtinInteger INTEGER,
+	builtinSerial SERIAL,
+	builtinFloatingPoint NUMERIC,
+	builtinText TEXT,
+	builtinBlob BINARY,
+	builtinDate DATE,
+	builtinDateTime TIMESTAMPTZ,
+	builtinTime TIME WITH TIME ZONE
+
+) WITH SYSTEM VERSIONING; -- enable System-Versioning for this table