Refactor bindgen

.github/workflows/test.yml

@@ -63,6 +63,7 @@ jobs:
 
       - name: Run Bindgen Test
         run: |
+          sh bundle.sh
           sh bindgen-test.sh install
           sh bindgen-test.sh run
 

package.json

@@ -17,7 +17,7 @@
     "typescript": "^5.3.2"
   },
   "dependencies": {
-    "@dylibso/xtp-bindgen": "1.0.0-rc.11",
+    "@dylibso/xtp-bindgen": "1.0.0-rc.13",
     "ejs": "^3.1.10"
   }
 }

src/index.ts

@@ -1,62 +1,77 @@
 import ejs from "ejs";
-import { getContext, helpers, Property } from "@dylibso/xtp-bindgen";
+import { 
+  helpers, 
+  getContext, 
+  ObjectType, 
+  EnumType, 
+  ArrayType, 
+  XtpNormalizedType, 
+  XtpTyped 
+} from "@dylibso/xtp-bindgen"
 
-function toPythonType(property: Property): string {
-  let tp
+function pythonTypeName(s: string): string {
+  return helpers.snakeToPascalCase(s);
+}
+
+function pythonFunctionName(s: string): string {
+  return helpers.camelToSnakeCase(s);
+}
+
+function isOptional(type: String): boolean {
+  return type.startsWith('Optional[')
+}
 
-  if (property.$ref) {
-    tp = property.$ref.name
-  } else {
-    switch (property.type) {
-      case "string":
-        if (property.format === "date-time") {
-          tp = "datetime"
-        } else {
-          tp = "str"
-        }
-        break
-      case "number":
-        // @ts-ignore
-        if (property.contentType === "application/json") {
-          tp = "str"
-        } else if (property.format === "float" || property.format === "double") {
-          tp = "float"
-        } else {
-          tp = "int"
-        }
-        break
-      case "integer":
-        // @ts-ignore
-        if (property.contentType === "application/json") {
-          tp = "str"
-        } else {
-          tp = "int"
-        }
-        break
-      case "boolean":
-        tp = "bool"
-        break
-      case "object":
-        tp = "dict"
-        break
-      case "array":
-        if (!property.items) {
-          tp = "list"
-        } else {
-          tp = `List[${toPythonType(property.items as Property)}]`
-        }
-        break
-      case "buffer":
-        tp = "bytes"
-        break
-      default:
-        throw new Error("Can't convert property to Python type: " + property.type);
-    }
+function toPythonTypeX(type: XtpNormalizedType): string {
+  const opt = (t: string) => {
+    return type.nullable ? `Optional[${t}]` : t
   }
+  switch (type.kind) {
+    case 'string':
+      return opt('str');
+    case 'int32':
+      return opt('int');
+    case 'float':
+      return opt('float');
+    case 'double':
+      return opt('float')
+    case 'byte':
+      return opt('byte');
+    case 'date-time':
+      return opt("datetime");
+    case 'boolean':
+      return opt('bool');
+    case 'array':
+      const arrayType = type as ArrayType
+      return opt(`List[${toPythonTypeX(arrayType.elementType)}]`);
+    case 'buffer':
+      return opt('bytes'); 
+    case 'map':
+      // TODO: improve typing of dicts
+      return opt('dict');
+    case 'object':
+      return opt(pythonTypeName((type as ObjectType).name));
+    case 'enum':
+      return opt(pythonTypeName((type as EnumType).name));
+    default:
+      throw new Error("Can't convert XTP type to Python type: " + type)
+  }
+}
+
+
+function toPythonType(property: XtpTyped): string {
+  let t = toPythonTypeX(property.xtpType);
+  if (isOptional(t)) return t;
+  return `Optional[${t}]`;
+}
 
-  if (!tp) throw new Error("Cant convert property to Python type: " + property.type)
-  if (!property.nullable && !property.required) return tp
-  return `Optional[${tp}]`
+function toPythonParamType(property: XtpTyped): string {
+  let t = toPythonTypeX(property.xtpType);
+  // We need to represent bare int/float as a str in Python for now,
+  // there may be some updates to the encoder we can make to handle
+  // this case at some point
+  t = t.replace('int', 'str');
+  t = t.replace('float', 'str');
+  return t;
 }
 
 export function render() {
@@ -65,6 +80,9 @@ export function render() {
     ...helpers,
     ...getContext(),
     toPythonType,
+    toPythonParamType,
+    pythonTypeName,
+    pythonFunctionName
   };
 
   const output = ejs.render(tmpl, ctx);

template/plugin/__init__.py.ejs

@@ -1,10 +1,12 @@
 # THIS FILE WAS GENERATED BY `xtp-python-bindgen`. DO NOT EDIT.
 
+from typing import Optional, List # noqa: F401
+from datetime import datetime # noqa: F401
 import extism # pyright: ignore
 import plugin
 
 <% if (Object.values(schema.schemas).length > 0){ %>
-from pdk_types import <%- Object.values(schema.schemas).map(schema => schema.name).join(", ") %> # noqa: F401 
+from pdk_types import <%- Object.values(schema.schemas).map(schema => pythonTypeName(schema.name)).join(", ") %> # noqa: F401 
 <% } %>
 
 # Imports
@@ -13,13 +15,13 @@ from pdk_types import <%- Object.values(schema.schemas).map(schema => schema.nam
 <% if (hasComment(imp)) -%>
 #<%- imp.name %> <%- formatCommentBlock(imp.description, "# ") %>
 <% if (hasComment(imp.input)) { -%>
-# It takes input of <%- toPythonType(imp.input) %> (<%- formatCommentLine(imp.input.description) %>)
+# It takes input of <%- toPythonParamType(imp.input) %> (<%- formatCommentLine(imp.input.description) %>)
 <% } -%>
 <% if (hasComment(imp.output)) { -%>
-# And it returns an output <%- toPythonType(imp.output) %> (<%- formatCommentLine(imp.output.description) %>)
+# And it returns an output <%- toPythonParamType(imp.output) %> (<%- formatCommentLine(imp.output.description) %>)
 <% } -%>
 @extism.import_fn("extism:host/user", "<%- imp.name %>")
-def <%- camelToSnakeCase(imp.name) %>(<% if (imp.input) { -%>input: <%- toPythonType(imp.input) %><%} -%>) <% if (imp.output) { %> -> <%- toPythonType(imp.output) %><% } %>: # pyright: ignore [reportReturnType]
+def <%- pythonFunctionName(imp.name) %>(<% if (imp.input) { -%>input: <%- toPythonParamType(imp.input) %><%} -%>) <% if (imp.output) { %> -> <%- toPythonParamType(imp.output) %><% } %>: # pyright: ignore [reportReturnType]
   pass
 <% }) %>
 
@@ -32,7 +34,7 @@ def <%- camelToSnakeCase(imp.name) %>(<% if (imp.input) { -%>input: <%- toPython
 <% } -%>
 @extism.plugin_fn
 def <%- ex.name %>():
-  res = plugin.<%- camelToSnakeCase(ex.name) %>(<% if (ex.input) { %> extism.input(<%- toPythonType(ex.input) %>) <% } %>)
+  res = plugin.<%- pythonFunctionName(ex.name) %>(<% if (ex.input) { %> extism.input(<%- toPythonParamType(ex.input) %>) <% } %>)
   extism.output(res)
 
 <% }) %>

template/plugin/pdk_imports.py.ejs

@@ -1,22 +1,24 @@
 # THIS FILE WAS GENERATED BY `xtp-python-bindgen`. DO NOT EDIT.
 
+from typing import Optional, List # noqa: F401
+from datetime import datetime # noqa: F401
 import extism # noqa: F401 # pyright: ignore
 
 <% if (Object.values(schema.schemas).length > 0) { %>
-from pdk_types import <%- Object.values(schema.schemas).map(schema => schema.name).join(", ") %> # noqa: F401
+from pdk_types import <%- Object.values(schema.schemas).map(schema => pythonTypeName(schema.name)).join(", ") %> # noqa: F401
 <% } %>
 
 <% schema.imports.forEach(imp => { %>
 <% if (hasComment(imp)) -%>
 #<%- imp.name %> <%- formatCommentBlock(imp.description, "# ") %>
 <% if (hasComment(imp.input)) { -%>
-# It takes input of <%- toPythonType(imp.input) %> (<%- formatCommentLine(imp.input.description) %>)
+# It takes input of <%- toPythonParamType(imp.input) %> (<%- formatCommentLine(imp.input.description) %>)
 <% } -%>
 <% if (hasComment(imp.output)) { -%>
-# And it returns an output <%- toPythonType(imp.output) %> (<%- formatCommentLine(imp.output.description) %>)
+# And it returns an output <%- toPythonParamType(imp.output) %> (<%- formatCommentLine(imp.output.description) %>)
 <% } -%>
 @extism.import_fn("extism:host/user", "<%- imp.name %>")
-def <%- camelToSnakeCase(imp.name) %>(<% if (imp.input) { -%>input: <%- toPythonType(imp.input) %><%} -%>) <% if (imp.output) { %> -> <%- toPythonType(imp.output) %><% } %>: # pyright: ignore [reportReturnType]
+def <%- pythonFunctionName(imp.name) %>(<% if (imp.input) { -%>input: <%- toPythonParamType(imp.input) %><%} -%>) <% if (imp.output) { %> -> <%- toPythonParamType(imp.output) %><% } %>: # pyright: ignore [reportReturnType]
   pass
 <% }) %>
 

template/plugin/pdk_types.py.ejs

@@ -10,27 +10,27 @@ import extism # noqa: F401 # pyright: ignore
 
 <% Object.values(schema.schemas).forEach(schema => { %>
     <% if (schema.enum) { %>
-class <%- capitalize(schema.name) %>(Enum):
+class <%- pythonTypeName(schema.name) %>(Enum):
 <% schema.enum.forEach(variant => { -%>
-  <%- capitalize(variant) %> = "<%- variant %>"
+  <%- pythonTypeName(variant) %> = "<%- variant %>"
 <% }) %>
     <% } else { %>
 @dataclass
-class <%- capitalize(schema.name) %>(extism.Json):
+class <%- pythonTypeName(schema.name) %>(extism.Json):
 <% schema.properties.forEach(p => { -%>
-<% if (p.description) { -%>
-  # <%- formatCommentBlock(p.description, "# ") %>
-<% } -%>
-<% if (!p.nullable) {%>
+<% if (!p.nullable || p.required) {%>
+  <% if (p.description) { -%>
+    # <%- formatCommentBlock(p.description, "# ") %>
+  <% } -%>
   <%- p.name %>: <%- toPythonType(p) %>
 <% } %>
 <% }) %>
 
 <% schema.properties.forEach(p => { -%>
-<% if (p.description) { -%>
-  # <%- formatCommentBlock(p.description, "# ") %>
-<% } -%>
-<% if (p.nullable) {%>
+<% if (p.nullable && !p.required) {%>
+  <% if (p.description) { -%>
+    # <%- formatCommentBlock(p.description, "# ") %>
+  <% } -%>
   <%- p.name %>: <%- toPythonType(p) %> = None
 <% } %>
 <% }) %>

template/plugin/plugin.py.ejs

@@ -1,17 +1,20 @@
+from typing import Optional, List # noqa: F401
+from datetime import datetime # noqa: F401
 import extism # noqa: F401 # pyright: ignore
+
 <% if (Object.values(schema.schemas).length > 0) { %>
-from pdk_types import <%- Object.values(schema.schemas).map(schema => schema.name).join(", ") %> # noqa: F401
+from pdk_types import <%- Object.values(schema.schemas).map(schema => pythonTypeName(schema.name)).join(", ") %> # noqa: F401
 <% } %>
 <% if (schema.imports.length > 0) { %>
-from pdk_imports import <%- schema.imports.map(schema => camelToSnakeCase(schema.name)).join(", ") %> # noqa: F401
+from pdk_imports import <%- schema.imports.map(schema => pythonFunctionName(schema.name)).join(", ") %> # noqa: F401
 <% } %>
 from typing import List, Optional # noqa: F401
 
 <% schema.exports.forEach(ex => { -%>
 <% if (hasComment(ex)) { -%>
 # <%- formatCommentBlock(ex.description, "# ") %>
 <% } -%>
-def <%- camelToSnakeCase(ex.name) %>(<% if (ex.input) { %>input: <%- toPythonType(ex.input) %> <% } %>) <% if (ex.output) {%>-> <%- toPythonType(ex.output) %><%}%>:
+def <%- pythonFunctionName(ex.name) %>(<% if (ex.input) { %>input: <%- toPythonParamType(ex.input) %> <% } %>) <% if (ex.output) {%>-> <%- toPythonParamType(ex.output) %><%}%>:
 <% if (featureFlags['stub-with-code-samples'] && codeSamples(ex, 'python').length > 0) { -%>
 	<%- codeSamples(ex, 'python')[0].source %>
 <% } else { -%>