{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Creating Node Definitions \n", "\n", "This notebook will examine the creation of definitions for standard or custom libraries.\n", "\n", "Aspects covered include:\n", "1. Creating \"robust\" definitions from nodegraph implementations.\n", "2. Creating efficient definitions from source code implementations.\n", "3. \"Publishing\" definitions to libraries, including the core MaterialX libraries.\n", "\n", "## Core Library Setup\n", "\n", "The basic setup will use the core MaterialX library as well as the nodegraph utilities found in \n", "mtlxutils\n", "\n", "As with other notebooks, we require the loading in the of the standard libraries and the creation\n", "of a working document." ] }, { "cell_type": "code", "execution_count": 149, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded 750 standard library definitions for MaterialX version 1.39.0\n" ] } ], "source": [ "# MaterialX and MaterialX utilities\n", "import MaterialX as mx\n", "import mtlxutils.mxfile as mxf\n", "from mtlxutils.mxnodegraph import MtlxNodeGraph as mxg\n", "# For markdown display\n", "from IPython.display import display_markdown\n", "import re, os\n", "\n", "# Version check\n", "from mtlxutils.mxbase import *\n", "haveVersion1387 = haveVersion(1, 38, 7) \n", "if not haveVersion1387:\n", " print(\"** Warning: Recommended minimum version is 1.38.7 for tutorials. Have version: \", mx.__version__)\n", "\n", "stdlib = mx.createDocument()\n", "searchPath = mx.getDefaultDataSearchPath()\n", "libraryFolders = mx.getDefaultDataLibraryFolders()\n", "libFiles = None\n", "try:\n", " libFiles = mx.loadLibraries(libraryFolders, searchPath, stdlib)\n", "\n", " print('Loaded %s standard library definitions for MaterialX version %s' % (len(stdlib.getNodeDefs()), mx.__version__))\n", "except mx.Exception as err:\n", " print('Failed to load standard library definitions: \"', err, '\"')\n", "\n", "doc = mx.createDocument()\n", "doc.importLibrary(stdlib)\n", "\n", "# Write predicate\n", "def skipLibraryElement(elem):\n", " return not elem.hasSourceUri()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Moving Comments into Node Definitions" ] }, { "cell_type": "code", "execution_count": 150, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "> Transfer comments to nodedefs for file:: stdlib_defs.mtlx\n", "> Transferred 646 comments to nodedefs\n", "> Write transferred document to file: data/stdlib_defs_doc_transfer.mtlx\n" ] }, { "data": { "text/markdown": [ "> [Download stdlib_defs_doc_transfer.mtlx](data/stdlib_defs_doc_transfer.mtlx)" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def sanitizeXMLString(xmlString):\n", " # Add more here as needed\n", " replacements = {\n", " '<': '<',\n", " '>': '>',\n", " '&': '&',\n", " '\"': '"',\n", " \"'\": ''',\n", " '\\n': ' ',\n", " '\\r': ' ',\n", " '\\t': ' ', # Using 4 spaces for tabs\n", " '\\\\': ' '\n", " }\n", "\n", " # Replace each invalid character with its & equivalent or space\n", " for char, replacement in replacements.items():\n", " xmlString = xmlString.replace(char, replacement)\n", "\n", " # Strip out leading and trailing whitespace \n", " xmlString = re.sub(r'\\s+', ' ', xmlString) \n", " xmlString = xmlString.lstrip().strip()\n", " return xmlString\n", "\n", "def transferCommentsToNodeDefs(libFile):\n", " \n", " readOptions = mx.XmlReadOptions()\n", " readOptions.readComments = True\n", " readOptions.readNewlines = True \n", " readOptions.upgradeVersion = False\n", "\n", " outputDoc = mx.createDocument()\n", " mx.readFromXmlFile(outputDoc, libFile, mx.FileSearchPath(), readOptions) \n", "\n", " # Extract out comments and nodedefs\n", " currentComment = []\n", " children = outputDoc.getChildren()\n", " for child in children:\n", " if child.getCategory() == 'comment':\n", " docstring = child.getAttribute('doc')\n", " if len(docstring) > 0:\n", " docstring = sanitizeXMLString(docstring)\n", " # Replace end .. with .\n", " if docstring.endswith('..'):\n", " docstring = docstring[:-1]\n", " currentComment.append(['comment', docstring, child.getName() ])\n", " elif child.getCategory() == 'nodedef':\n", " currentComment.append(['nodedef', child.getName()])\n", "\n", " strippedComments = []\n", " # Heuristic to find comments for nodedefs:\n", " # 1. Accumulate nodedefs until a comment is found\n", " # 2. Add an association between the comment and nodedefs\n", " # 3. Skip if a comment is found immediately before a comment\n", " # 4. Keep track of comments to remove\n", " hitComment = False\n", " nodedefList = []\n", " removeComments = []\n", " for i in range(len(currentComment)-1, -1, -1):\n", " if not hitComment and currentComment[i][0] == 'comment':\n", " if len(nodedefList) > 0:\n", " # Keep track of comments to remove.\n", " # Add [ nodedef, comment ] pair \n", " removeComments.append(currentComment[i][2])\n", " for nodedef in nodedefList:\n", " strippedComments.append([nodedef, currentComment[i][1]])\n", " nodedefList.clear();\n", " hitComment = True\n", " elif currentComment[i][0] == 'nodedef':\n", " nodedefList.append(currentComment[i][1])\n", " hitComment = False\n", "\n", " # Apply comments to nodedefs:\n", " # 1. Find nodedefs\n", " # 2. Add new comments to existing comments\n", " numStrippedComments = len(strippedComments)\n", " if numStrippedComments == 0:\n", " print('No comments to transfer')\n", " return outputDoc, numStrippedComments\n", " \n", " for i in range(numStrippedComments):\n", "\n", " nodedef = outputDoc.getChild(strippedComments[i][0])\n", " if nodedef is None:\n", " print('Cannot find nodedef:', strippedComments[i][0])\n", " continue\n", " currentDoc = nodedef.getAttribute('doc')\n", " newDoc = strippedComments[i][1]\n", " if len(currentDoc) > 0:\n", " newDoc = newDoc + \" \" + currentDoc\n", " nodedef.setAttribute('doc', newDoc)\n", "\n", " # Remove comments\n", " for i in range(len(removeComments)):\n", " outputDoc.removeChild(removeComments[i])\n", "\n", " return outputDoc, numStrippedComments\n", "\n", "# remove duplicates from libFiles\n", "transferredDoc = None\n", "libFiles = list(dict.fromkeys(libFiles))\n", "for libFile in libFiles:\n", " if 'stdlib_defs.mtlx' in libFile:\n", "\n", " print('> Transfer comments to nodedefs for file::', os.path.basename(libFile))\n", " transferredDoc, numStripped = transferCommentsToNodeDefs(libFile)\n", " if numStripped > 0:\n", " print('> Transferred %d comments to nodedefs' % numStripped)\n", " else:\n", " print('> No comments to transfer')\n", " break\n", "\n", "if transferredDoc is None:\n", " print('> Failed to transfer comments to nodedefs')\n", "else:\n", " print('> Write transferred document to file: data/stdlib_defs_doc_transfer.mtlx')\n", " mx.writeToXmlFile(transferredDoc, 'data/stdlib_defs_doc_transfer.mtlx')\n", " writeToMarkdown('> [Download stdlib_defs_doc_transfer.mtlx](data/stdlib_defs_doc_transfer.mtlx)')" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## 1. Creating Definitions from Compound Graphs\n", "\n", "Creating definitions from compound node graphs is the easiest way to create new definitions without worrying about shading language implementations if the language is supported. \n", "\n", "The basic logic for publishing from a nodegraph entails:\n", "* Making a copy a given compound graph\n", "* Turning the copy into a functional graph\n", "* Creating a definition based on the input and output interfaces on the functional graph\n", "* Adding in additional meta-data tagging as outlined in the definitions learning section:\n", " * Node group\n", " * Version\n", " * Namespace\n", " * UI properties\n", "* Creating a reference between the definition and the functional graph" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Helpers\n", "\n", "There is currently a helper interface on Document called `addNodeDefFromGraph()` that encapsulates the required logic for the most part. It does not:\n", "* Handle creating of definitions which inherit from each other, nor \n", "* Update versioning on existing definitions with different versions.\n", "\n", "The 1.38.7 version of the utility has some issues with which are being looked at. These issues can be addressed by patching the results.\n", "\n", "The following is a simple utility wrapper sets up the creation parameters. It calls `adNodeDefFromGraph()` and returns both the definition (`nodedef`) and the functional graph created." ] }, { "cell_type": "code", "execution_count": 151, "metadata": {}, "outputs": [], "source": [ "def createDefinitionAndFunctionalGraph(nodeGraph, cparam):\n", " '''\n", " Example of creating a definition. This has a fixed version, nodegroup and graph names. \n", "\n", " Arguments:\n", " - nodeGraph : the compound node graph \n", " - cparam : a set of node definition parameters keyed by semantic names.\n", "\n", " Returns:\n", " - Node definition and functional node graph\n", " '''\n", " version_major, version_minor, version_patch = mx.getVersionIntegers()\n", " if version_major >=1 and version_minor >= 39:\n", " print('Using 1.39 or later API to create definition...')\n", " definition = doc.addNodeDefFromGraph(nodeGraph, cparam['nodedefName'], cparam['category'], cparam['nodegraphName'])\n", " if len(cparam['version']) > 0:\n", " definition.setVersionString(cparam['version'])\n", " if cparam['defaultversion'] != None:\n", " definition.setDefaultVersion(cparam['defaultversion'])\n", " if len(cparam['nodegroup']) > 0:\n", " definition.setNodeGroup(cparam['nodegroup'])\n", " else:\n", " definition = doc.addNodeDefFromGraph(nodeGraph, cparam['nodedefName'],\n", " cparam['category'], cparam['version'], cparam['defaultversion'], \n", " cparam['nodegroup'], cparam['nodegraphName'])\n", " funcgraph = doc.getNodeGraph(cparam['nodegraphName'])\n", "\n", " return definition, funcgraph" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "The `getNodeGroups()` helper scans for existing node group names defined by the standard library using the interface Document.getNodeDefs() to get all of the definitions, and NodeDef.getNodeGroup() to find any specified node group.\n", "\n", "Using these group names allows:\n", "* Any associated group semantic meanings to be discoverable by code generation. For example `texture2d` and `pbr` have semantic meanings. \n", "* For naming consistency avoiding excessive partitioning into too many disparate groups.\n", "\n", "Integrations may which to run this type of logic to examine for existing node groups independently from \n", "definition creation workflows. " ] }, { "cell_type": "code", "execution_count": 152, "metadata": {}, "outputs": [], "source": [ "def getNodeGroups(library):\n", " '''\n", " Find all the existing node group names on definitions \n", "\n", " Inputs:\n", " - library : Definition library which defines the node groups.\n", " '''\n", " nodeGroups = set()\n", " for nd in library.getNodeDefs():\n", " group = nd.getNodeGroup()\n", " if group:\n", " nodeGroups.add(group)\n", "\n", " return nodeGroups\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "The helper is used below to print out the available node groups in the standard libraries:" ] }, { "cell_type": "code", "execution_count": 153, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Existing node groups on definitions:\n", " - adjustment\n", " - application\n", " - channel\n", " - colortransform\n", " - compositing\n", " - conditional\n", " - convolution2d\n", " - geometric\n", " - global\n", " - light\n", " - material\n", " - math\n", " - npr\n", " - organization\n", " - pbr\n", " - procedural\n", " - procedural2d\n", " - procedural3d\n", " - shader\n", " - texture2d\n", " - texture3d\n", " - translation\n" ] } ], "source": [ "print('Existing node groups on definitions:')\n", "nodeGroups = getNodeGroups(stdlib)\n", "for ng in sorted(nodeGroups):\n", " print(' - %s' % ng)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "The `findCompoundGraphs()` help scans a document and returns a list of compound `nodegraphs` in a document.\n", "\n", "**To differentiate between a compound and an functional graph**, the API interface \n", "NodeGraph.getNodeDef() can be used.\n", "If a non empty definition is returned from `getNodeDef()` then the graph is a functional graph.\n", "\n", "A list of library file names is passed to filter out any compound graphs that were loaded in from a definition\n", "library." ] }, { "cell_type": "code", "execution_count": 154, "metadata": {}, "outputs": [], "source": [ "def findCompoundGraphs(doc, libFiles):\n", " '''\n", " Search for compound graphs in a document. Skips any graphs found in \n", " library files (passed in a a list of source URIs)\n", " '''\n", " compoundGraphs = []\n", "\n", " nodeGraphs = doc.getNodeGraphs()\n", " for nodeGraph in nodeGraphs:\n", " # Skip any nodegraph which is from a library\n", " if nodeGraph.getSourceUri() in libFiles:\n", " continue\n", "\n", " # Skip functional graphs\n", " if nodeGraph.getNodeDef():\n", " continue\n", "\n", " compoundGraphs.append(nodeGraph)\n", " return compoundGraphs" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Associations Stored In Implementations\n", "\n", "The association between a definition and a functional graph can be stored in an implementation as of version `1.38.5`.\n", "It is possible to find these implementations by comparing the definition name with the implementation's `nodedef` attribute name, by calling Interface.getNodeDefString() to see if there is a definition match.\n", "> At time of writing the node graph query interface is not exposed in the Python API but the attribute can be queried directly. Note that an Implementation is derived from Interface.\n", "\n", "The helper `getImplementationForNodedef()` shows this logic. In the example we are looking for Autodesk `standard surface` which uses implementations for definition / functional graph associations for different definition versions. Both are queried for below.\n", "\n", "> Note that source code associations always used implementations for associations as there is no \n", "mechanism for source code to reference back to it's definition as there is no explicit \"source code\" element in \n", "MaterialX.\n" ] }, { "cell_type": "code", "execution_count": 155, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found implementation: IMPL_standard_surface_surfaceshader_101, definition ND_standard_surface_surfaceshader and graph NG_standard_surface_surfaceshader_100\n", "Found implementation: IMPL_standard_surface_surfaceshader_100, definition ND_standard_surface_surfaceshader_100 and graph NG_standard_surface_surfaceshader_100\n" ] } ], "source": [ "def getAllImplementations(doc):\n", " '''\n", " Print out all implementations\n", " '''\n", " for impl in doc.getImplementations():\n", " print(impl)\n", "\n", "def getImplementationForNodedef(doc, definition):\n", " '''\n", " Get an implemenation which matches a definition which is implemented\n", " as a functional nodegraph\n", " '''\n", " if not definition or not doc:\n", " return None\n", " for impl in doc.getImplementations():\n", " # Missing getNodeGraphString() expose in Python API \n", " if impl.getNodeDefString() == definition.getName() and impl.getAttribute('nodegraph'): #impl.getNodeGraphString() \n", " return impl\n", " return None\n", "\n", "# Look for two versions of standard surface\n", "definition = doc.getNodeDef('ND_standard_surface_surfaceshader')\n", "definition_old = doc.getNodeDef('ND_standard_surface_surfaceshader_100')\n", "\n", "# Find the implementations for each definition\n", "impl = getImplementationForNodedef(doc, definition)\n", "if impl:\n", " print('Found implementation: %s, definition %s and graph %s' % (impl.getName(), impl.getNodeDefString(),\n", " impl.getAttribute('nodegraph')))\n", "else:\n", " print('Failed to find implementation element for definition %s' % (definition.getName()))\n", "\n", "impl = getImplementationForNodedef(doc, definition_old)\n", "if impl:\n", " print('Found implementation: %s, definition %s and graph %s' % (impl.getName(), impl.getNodeDefString(),\n", " impl.getAttribute('nodegraph')))\n", "else:\n", " print('Failed to find implementation element for definition %s' % (definition.getName())) " ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "\n", "### Extracting Out A Definition\n", "\n", "Once a new definition created, we will want to export the definitions and functional graphs (and implementations) into either a new document or an existing definition library document. To do so the contents need to be copied into the desired document.\n", "\n", "The help function `addDefinitionToDocument()` will copy a definition, functional graph pair either 1 or 2 separate documents. As there is no \"copy\" function from one document to another, an empty definition and graph needs to be created first using Document.addNodeDef() and \n", "Document.addNodeGraph() respectively. and then the contents copied over using the \n", "Element.copyContentFrom() interface respectively.\n", "\n", "If the association between the functional graph and definition:\n", "* resides on node graph, this will be copied over when `copyContentFrom()` is called on the graph.\n", "* is stored on a separate `implementation` instead then that must also be copied by creating a new implementationusing \n", "Document.addImplementation()\n", "and copying it's contents over.\n" ] }, { "cell_type": "code", "execution_count": 156, "metadata": {}, "outputs": [], "source": [ "\n", "def addDefinitionToDocument(definition, funcgraph, defDoc, graphDoc=None, defComment='', graphComment=''):\n", " '''\n", " Copy a definition and functional node graph to a new document.\n", " If there is a implementation which associates the definition and graph\n", " copyy that as well.\n", "\n", " Arguments:\n", " - definition : nodedef to copy\n", " - funcgraph : Functional graph to copy\n", " - defDoc : Destination document for definition\n", " - graphDoc: Optional destination document for functional graph\n", " - defComment : Optional comment to prepend to the destination document's definition\n", " - graphComment : Optional comment to prepend to the destination document's functional graph\n", " '''\n", " if definition and funcgraph:\n", "\n", " # Add definition comment\n", " if defComment:\n", " comment = defDoc.addChildOfCategory('comment')\n", " comment.setDocString(defComment) \n", "\n", " # Create a new definition, and copy the content over. Make sure\n", " # to use the existing name and category.\n", " nodeDef = defDoc.addNodeDef(definition.getName(), '', definition.getCategory())\n", " nodeDef.copyContentFrom(definition) \n", "\n", " if not graphDoc:\n", " graphDoc = defDoc\n", "\n", " # Add graph comment\n", " if graphComment:\n", " comment = graphDoc.addChildOfCategory('comment')\n", " comment.setDocString(graphComment) \n", "\n", " # Create a new graph and copy the contents over. This will result in a functional graph.\n", " # Use the definiton document if no graph document specified\n", " newGraph = graphDoc.addNodeGraph(funcgraph.getName())\n", " newGraph.copyContentFrom(funcgraph)\n", "\n", " # If an implementation exists, copy that over as well. This will be added to \n", " # node graph document if a separate one is specified.\n", " impl = getImplementationForNodedef(definition.getDocument(), definition)\n", " if impl:\n", " newImpl = graphDoc.addImplementation()\n", " newImpl.copyContentFrom(impl) " ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Some additional utilities are proved to write the contents of the new definition document for display." ] }, { "cell_type": "code", "execution_count": 157, "metadata": {}, "outputs": [], "source": [ "def writeDocToString(doc):\n", " writeOptions = mx.XmlWriteOptions()\n", " writeOptions.writeXIncludeEnable = False\n", " writeOptions.elementPredicate = mxf.MtlxFile.skipLibraryElement\n", "\n", " documentContents = mx.writeToXmlString(doc, writeOptions)\n", " return documentContents\n", "\n", "def writeDocToMarkdown(documentContents):\n", " display_markdown('```xml\\n' + documentContents + '\\n```\\n', raw=True)\n", "\n", "def writeToMarkdown(val):\n", " display_markdown(val, raw=True) " ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### Example\n", "\n", "As an example we will load in an example compound graph and use it to create a definition. The Python utility (`createdefinition`) encapsulates this logic and provides a command line interface for various options.\n", "\n", "The main logic loads in an example file, creates new definition/functional graph pairs and writes them to\n", "separate separate document(s).\n", "\n", "For the 1.38.7 version API, the following creation parameters are set:\n", "\n", "* A `category` name. \n", " * It is **very important to choose a unique name** for this as it is the name of the element which\n", "could be used in a shared definition library. \n", " * In general you want something that takes into account the signature of the graph interface as well as any key definition attributes such as version. \n", " * A sample identifier creator is provided called `generateIdentifier()` which takes in a nodegraph. An alternative version takes in a list of outputs: `generateIdentifierFromOutputs()`. *This is useful for source code definitions as described later on.* \n", " * There are no strict guidelines for category name but users should use `createValidName()` to ensure a valid element name is used.\n", "\n", "* The \"standard\" prefixes of `ND_` and `NG_` are used as node definition and node graph name prefixes and codified in the `createDefinitionIdentifier()` and `createGraphIdentifier()` utilities. An `IMPL_` can be used if the association is set using a separate implementation element. \n", "\n", "* A version is always added. For initial definitions the version number of `1.0` and the flag is set to indicate that this is the default version.\n", "\n", "* A node group is always added. This setting is difficult to infer based on just the compound node graph so can be a user speified choice based on the available groups returned from `getNodeGroups()` or a new custom one." ] }, { "cell_type": "code", "execution_count": 158, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using 1.39 or later API to create definition...\n" ] }, { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def generateIdentifier(category, version, nodeGraph):\n", " '''\n", " Utility to generate a unique identifier for a definition. Takes into account\n", " category, version and a node graphs signature.\n", " '''\n", " outputTypes = []\n", " for output in nodeGraph.getOutputs():\n", " outputTypes.append(output.getType())\n", " return generateIdentifierFromOutputs(category, version, outputTypes)\n", "\n", "def generateIdentifierFromOutputs(category, version, outputTypes):\n", " '''\n", " Utility to generate a unique identifier for a definition. Takes into account\n", " category, version and list of output types.\n", " '''\n", " identifier = category\n", "\n", " if version:\n", " identifier = identifier + '_' + version\n", "\n", " for outputType in outputTypes:\n", " identifier = identifier + '_' + outputType\n", "\n", " return mx.createValidName(identifier)\n", "\n", "def createDefinitionIdentifier(identifier):\n", " ''' \n", " Create the definition element id\n", " '''\n", " nodedefName = 'ND_' + identifier\n", " return nodedefName\n", "\n", "def createGraphIdentifier(identifier):\n", " ''' \n", " Create the functional node graph element id\n", " '''\n", " nodegraphName = 'NG_' + identifier\n", " return nodegraphName\n", "\n", "def createImplIdentifier(identifier):\n", " ''' \n", " Create the implementation element id\n", " '''\n", " nodegraphName = 'IMPL_' + identifier\n", " return nodegraphName\n", "\n", "# Read in an example with a compound graph\n", "doc, libFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "mx.readFromXmlFile(doc, mx.FilePath('./data/test_procedural.mtlx'))\n", "\n", "# Determine the node group and version\n", "availableGroupNames = getNodeGroups(doc)\n", "nodegroup = 'texture2d' if 'texture2d' in availableGroupNames else list(availableGroupNames)[0]\n", "version = '1.0'\n", "isDefaultVersion = True\n", "\n", "compoundGraphs = findCompoundGraphs(doc, libFiles)\n", "for nodeGraph in compoundGraphs: \n", " cparam = {}\n", " # Set the category name. Just use the nodegraph name for now\n", " category = nodeGraph.getName().lower()\n", " # Create a new identifier\n", " id = generateIdentifier(category, version, nodeGraph)\n", " # Get definition and graph name\n", " cparam['nodedefName'] = createDefinitionIdentifier(id)\n", " cparam['nodegraphName'] = createGraphIdentifier(id) \n", " cparam['category'] = category\n", " cparam['version'] = version\n", " cparam['defaultversion'] = isDefaultVersion\n", " cparam['nodegroup'] = nodegroup\n", "\n", " # Create new definition and functional graph\n", " definition, funcgraph = createDefinitionAndFunctionalGraph(nodeGraph, cparam)\n", "\n", " # Copy the definition to a destination document(s)\n", " defDoc = mx.createDocument()\n", " comment = ' Node: <' + category + '> '\n", " addDefinitionToDocument(definition, funcgraph, defDoc, defDoc, comment, comment)\n", " if defDoc:\n", " documentContents = writeDocToString(defDoc)\n", " writeDocToMarkdown(documentContents)\n", " break" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 1.38.7 Definition Patching\n", "\n", "For 1.38.7 we need to patch the result. This includes \n", "* Setting the documentation string and adding any namespace.\n", "* Moving the interface inputs from the graph to the definition. ( Note that this is checked during Document `validate()` and will produce warnings if the functional graph has any inputs ). \n", "* Removing any undesired attributes on nodes and ports.\n", "Both the definition and functional graph need to be patched.\n", "\n", "The utility `patchDefinition` encapsulates this logic." ] }, { "cell_type": "code", "execution_count": 159, "metadata": {}, "outputs": [], "source": [ "def patchDefinition(definition, funcgraph, documentation, namespace):\n", " if documentation:\n", " definition.setDocString(documentation)\n", " if namespace:\n", " definition.setNamespace(namespace)\n", " funcgraph.setNamespace(namespace)\n", "\n", " if not funcgraph:\n", " return\n", "\n", " for graphChild in funcgraph.getChildren():\n", " graphChild.removeAttribute('xpos')\n", " graphChild.removeAttribute('ypos')\n", "\n", " filterAttributes = { 'nodegraph', 'nodename', 'channels', 'interfacename', 'xpos', 'ypos' }\n", "\n", " # Transfer input interface from the graph to the nodedef\n", " for input in funcgraph.getInputs():\n", " nodeDefInput = definition.addInput(input.getName(), input.getType())\n", " if nodeDefInput:\n", " nodeDefInput.copyContentFrom(input)\n", " for filterAttribute in filterAttributes:\n", " nodeDefInput.removeAttribute(filterAttribute);\n", " nodeDefInput.setSourceUri('')\n", " input.setInterfaceName(nodeDefInput.getName())\n", "\n", " # Remove interface from the nodegraph\n", " for input in funcgraph.getInputs():\n", " funcgraph.removeInput(input.getName())\n", "\n", " # Copy the output interface from the graph to the nodedef\n", " for output in funcgraph.getOutputs():\n", " nodeDefOutput = definition.getOutput(output.getName())\n", " if nodeDefOutput:\n", " definition.removeOutput(output.getName())\n", " definition.addOutput(output.getName(), output.getType())\n", " if nodeDefOutput:\n", " nodeDefOutput.copyContentFrom(output)\n", " for filterAttribute in filterAttributes:\n", " nodeDefOutput.removeAttribute(filterAttribute)\n", " nodeDefOutput.setSourceUri('') \n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Running with the patch results in the new corrected definition:" ] }, { "cell_type": "code", "execution_count": 160, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using 1.39 or later API to create definition...\n" ] }, { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Run definition creation again with patching logic\n", "doc, libFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "mx.readFromXmlFile(doc, mx.FilePath('./data/test_procedural.mtlx'))\n", "\n", "compoundGraphs = findCompoundGraphs(doc, libFiles)\n", "for nodeGraph in compoundGraphs: \n", " cparam = {}\n", " # Set the category name. Just use the nodegraph name for now\n", " category = nodeGraph.getName().lower()\n", " # Create a new identifier\n", " id = generateIdentifier(category, version, nodeGraph)\n", " # Get definition and graph name\n", " cparam['nodedefName'] = createDefinitionIdentifier(id)\n", " cparam['nodegraphName'] = createGraphIdentifier(id) \n", " cparam['category'] = category\n", " cparam['version'] = version\n", " cparam['defaultversion'] = isDefaultVersion\n", " cparam['nodegroup'] = nodegroup\n", "\n", " # Create new definition and functional graph\n", " definition, funcgraph = createDefinitionAndFunctionalGraph(nodeGraph, cparam)\n", "\n", " # Add documentation and namespace as well as patch up definition and functional graph\n", " documentation = 'Documentation for new definition: ' + nodeGraph.getName()\n", " namespace = 'mynamespace'\n", " patchDefinition(definition, funcgraph, documentation, namespace)\n", "\n", " # Copy the definition to a destination document(s)\n", " defDoc = mx.createDocument()\n", " graphDoc = None\n", " defComment = ' Definition: nodeGraph.getName() '\n", " graphComment = ' Functional graph for definition: nodeGraph.getName() '\n", " \n", " addDefinitionToDocument(definition, funcgraph, defDoc, graphDoc, defComment, graphComment)\n", " if defDoc:\n", " documentContents = writeDocToString(defDoc)\n", " writeDocToMarkdown(documentContents)\n", " break" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Creating Definitions from Source Code\n", "\n", "When creating a custom source node in MaterialX there are basically three things that needs to be created:\n", "\n", "1. A `` element specifying the signature of the node.\n", "2. An `` element that tells the code generator where it can find the source code for the node, for a particular target/language. You need one such element for each target you want to support. For the standard library: GLSL (and Vulkan, ESSL variants), OSL, MDL, and MSL should be supported. \n", "3. The source code for the node." ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 2.1 Creating the Interface\n", "\n", "There are no specific tools to directly creation `nodedef` interfaces. We will use a simple example which just adds 2 colors together:\n", "```xml\n", " \n", " \n", " \n", " \n", " \n", " \n", " ```\n", "Note that it is a good practice to have a default routing from the input to the output if a node instance is disabled (is a pass-through).\n", "\n", "This can be done by setting the `mx.Output.DEFAULT_INPUT_ATTRIBUTE` (`defaultnput`) attribute on an output. Note that it is only valid to set this on an output in a definition. In this case the default value for the output is the input \"in1\".\n", "\n", "Below is some sample code to create a source code definition interface using the helper `addSourceNodeDefinition()`. It reuses the unique identifier creation logic via `generateIdentifierFromOutputs()` for consistency.\n", "\n", "The definitions interface is manually populated it with the desired inputs and outputs. Of note is that the inputs must have default values in order to pass validation, and the\n", "output type is `color3` which is incorporated into the identifier as with the functional graph logic." ] }, { "cell_type": "code", "execution_count": 161, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def addSourceNodeDefinition(doc, cparam):\n", " '''\n", " Add a node definition which uses the standard naming convention.\n", " No definition is created if a node of the same name already exists in the document \n", " '''\n", " id = generateIdentifierFromOutputs(cparam['category'], cparam['version'], cparam['outputs'])\n", " nodedefName = createDefinitionIdentifier(id) \n", "\n", " existingDef = doc.getChild(nodedefName)\n", " if existingDef:\n", " return None\n", "\n", " newDef = doc.addChildOfCategory('nodedef', nodedefName)\n", " newDef.setVersionString(cparam['version'])\n", " newDef.setNodeGroup(cparam['nodegroup'])\n", " newDef.setNodeString(cparam['category'])\n", " return newDef\n", "\n", "# Create a working document and add a nodedef\n", "doc, libFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "\n", "cparam = {}\n", "category = nodeGraph.getName().lower()\n", "id = generateIdentifier(category, version, nodeGraph)\n", "cparam['nodedefName'] = createDefinitionIdentifier(id)\n", "cparam['category'] = 'myadd_explicit'\n", "cparam['version'] = '1.0'\n", "cparam['defaultversion'] = True\n", "cparam['nodegroup'] = 'math'\n", "cparam['outputs'] = ['color3']\n", "\n", "output_type = 'color3'\n", "input_type = 'color3'\n", "comment = doc.addChildOfCategory('comment')\n", "comment.setDocString(' Definition of a simple node , adding two colors. ')\n", "newDef = addSourceNodeDefinition(doc, cparam)\n", "\n", "# Add some inputs and outputs, making sure to set values for the inputs\n", "inputs = [ [\"in1\", input_type, \"1.0, 0.0, 0.0\"], [\"in2\", input_type, \"0.0, 0.0, 0.0\"] ]\n", "outputs = [ [\"out\", output_type, \"in1\"] ]\n", "for input in inputs:\n", " newInput = newDef.addInput(input[0], input[1])\n", " newInput.setValueString(input[2])\n", "for output in outputs:\n", " newOutput = newDef.addOutput(output[0], output[1])\n", " if output[2]:\n", " newOutput.setAttribute(mx.Output.DEFAULT_INPUT_ATTRIBUTE, output[2])\n", "\n", "documentContents = writeDocToString(doc)\n", "writeDocToMarkdown(documentContents)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "\n", "As an alternative, the interface could be created as a compound `nodegraph` by first using existing graph editing tools and then create a definition based on it. \n", "\n", "For example this graph was created interactively in the MaterialX Graph Editor:\n", "```xml\n", "\n", " \n", " \n", " \n", "\n", "```\n", "A utility called `copyValueElements()` is used to copy inputs and outputs over.\n", "> Note that `copyValueElements()` replaces the attributes on the node so these need to be cached and restored." ] }, { "cell_type": "code", "execution_count": 162, "metadata": {}, "outputs": [], "source": [ "def copyGraphInterface(newDef, refNodeGraph):\n", " '''\n", " Create a source code definition with the interface being provided by a reference node graph\n", " '''\n", " # Copy the children over from the nodegraph. Cache and restore attributes on the nodedef\n", " # which get written over when the copy occurs\n", " newDefAttrs = newDef.getAttributeNames()\n", " newDefAttrValues = {}\n", " for newDefAttr in newDefAttrs:\n", " attr = newDef.getAttribute(newDefAttr)\n", " newDefAttrValues[newDefAttr] = attr\n", " newDef.copyContentFrom(refNodeGraph)\n", " for newDefAttr in newDefAttrs:\n", " newDef.setAttribute(newDefAttr, newDefAttrValues[newDefAttr])\n", "\n", " # Filter out undesired attributes including connections and ui position\n", " filterAttributes = { 'nodegraph', 'nodename', 'channels', 'interfacename', 'xpos', 'ypos' }\n", " for child in newDef.getChildren():\n", " for filterAttribute in filterAttributes:\n", " child.removeAttribute(filterAttribute)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "As in the previous example the default input value is set manually. It cannot be set compound nodegraph before hand as this is considered to be invalid." ] }, { "cell_type": "code", "execution_count": 163, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Create new workgin document\n", "sourceCodeDoc, libFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "\n", "# Read in reference nodegraph\n", "refdoc, reflibFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "mx.readFromXmlFile(refdoc, mx.FilePath('./data/myadd_compound_graph.mtlx'))\n", "\n", "# Create a new empty definition\n", "cparam = {}\n", "category = nodeGraph.getName().lower()\n", "id = generateIdentifier(category, version, nodeGraph)\n", "cparam['nodedefName'] = createDefinitionIdentifier(id)\n", "cparam['category'] = 'myadd'\n", "cparam['version'] = '1.0'\n", "cparam['defaultversion'] = True\n", "cparam['nodegroup'] = 'math'\n", "cparam['outputs'] = ['color3']\n", "\n", "comment = sourceCodeDoc.addChildOfCategory('comment')\n", "comment.setDocString(' Definition of a simple node , adding two colors. ')\n", "inline_definition = addSourceNodeDefinition(sourceCodeDoc, cparam)\n", "\n", "# Copy the interface from a reference graph\n", "refNodeGraph = refdoc.getNodeGraph('myadd')\n", "if refNodeGraph:\n", "\n", " copyGraphInterface(inline_definition, refNodeGraph)\n", " for child in inline_definition.getOutputs():\n", " child.setAttribute(mx.Output.DEFAULT_INPUT_ATTRIBUTE, 'in1')\n", "\n", "if sourceCodeDoc:\n", " docString = writeDocToString(sourceCodeDoc)\n", " writeDocToMarkdown(docString)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 2.2 Creating Implementation Elements\n", "\n", "To support the \"standar\"d shading languages an implementation will be made for each target.\n", "At this point a choice needs to be made on whether the code can be inlined or not. \n", "* If it can then the `implementation` element will hold the code in it's `sourcecode` attribute. \n", "* If not then additional source code files need to be created and a `file` and `function` attribute need to be added. \n", "\n", "1. The helper `getTargetDefs()` will find all the \"standard\" targets defined. \n", "2. The helper `createImplementations()` will create on implementation per target and set up the association to the definition using `Implementation.setNodeDef()`, and the target using `Implementation.setTarget()`.\n", "\n", "For consistency we reuse the identifier for the definition but modify it's prefix to be the \"standard\" one of 'IMPL_'\n", "The `target` is added as a post-fix to the identifier to disambiguate by target. For example:\n", "```\n", "ND_myadd_1_0_color3\n", "```\n", "ends up with the following implementation identifiers:\n", "```\n", "IMPL_myadd_1_0_color3_genglsl for target: genglsl\n", "IMPL_myadd_1_0_color3_genmdl for target: genmdl\n", "IMPL_myadd_1_0_color3_genmsl for target: genmsl\n", "IMPL_myadd_1_0_color3_genosl for target: genosl\n", "```\n", "\n", "> At time of writing there has been no instances of requiring implementations to be versioned." ] }, { "cell_type": "code", "execution_count": 164, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Created implementations for definition: ND_myadd_1_0_color3\n", "- IMPL_myadd_1_0_color3_genglsl. Target:genglsl\n", "- IMPL_myadd_1_0_color3_genmdl. Target:genmdl\n", "- IMPL_myadd_1_0_color3_genmsl. Target:genmsl\n", "- IMPL_myadd_1_0_color3_genosl. Target:genosl\n" ] } ], "source": [ "def getTargetDefs(doc):\n", " targets = []\n", " for element in doc.getChildren():\n", " if element.getCategory() == 'targetdef':\n", " if element.getName() != 'essl':\n", " targets.append(element.getName())\n", " return targets\n", "\n", "def createImplementations(doc, nodedef, targets):\n", " '''\n", " Create implementation elements for a set of targets based on a given definition (nodedef).\n", " All implementation names are of the form:\n", " \n", " IM___\n", " \n", " '''\n", " # Reuse the same id signature as for the nodedef but replace the prefix.\n", " implName = createImplIdentifier(nodedef.getName().removeprefix('ND_'))\n", " category = nodedef.getNodeString() \n", " type = nodedef.getType()\n", " impls = []\n", " for target in targets:\n", " comment = doc.addChildOfCategory('comment')\n", " comment.setDocString(' Implementation of <%s> for target %s and type %s ' % (category, target, type))\n", " impl = doc.addImplementation(implName + '_' + target)\n", " impl.setNodeDef(nodedef)\n", " impl.setTarget(target)\n", " impls.append(impl)\n", "\n", " return impls\n", "\n", "# Create the implementations for all targets based on the nodedef\n", "inlined_doc = mx.createDocument()\n", "inlined_doc.copyContentFrom(sourceCodeDoc)\n", "targets = getTargetDefs(inlined_doc)\n", "for nodedef in inlined_doc.getNodeDefs():\n", " if nodedef.hasSourceUri():\n", " continue\n", " inlined_impls = createImplementations(inlined_doc, nodedef, targets)\n", " break\n", "\n", "print('Created implementations for definition: %s' % nodedef.getName())\n", "for impl in inlined_impls:\n", " print('- %s. Target:%s' % (impl.getName(), impl.getTarget()))" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "### 2.3 Adding Source Code\n", "\n", "#### 2.3.1 Adding Inlined Source Code\n", "\n", "For this implementation example we will first show inlined code which uses tokens to represent arguments. Tokens use '{{' and '}}' delimiters.\n", "\n", "In this example the code logic is:\n", "```\n", " '{{in1}} + {{in2}}'\n", " ```\n", " where `in1` corresponds to the nodedef input `in1` and `in2` to the nodedef input `in2`\n", "\n", " The help `setImplementationSourceCode_v1()` will find all implementations for a definitions and it's targets and set the inlineded coud." ] }, { "cell_type": "code", "execution_count": 165, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def setImplementationSourceCode_v1(doc, nodedef, targets, sourceCode):\n", " '''\n", " Set the same inlined source code for all targets of a nodedef. \n", " '''\n", " #category = nodedef.getNodeString() \n", " #type = nodedef.getType()\n", " implName = createImplIdentifier(nodedef.getName().removeprefix('ND_'))\n", " #implName = 'IM_' + category + '_' + type \n", " for target, code in zip(targets, sourceCode):\n", " impl = doc.getImplementation(implName + '_' + target)\n", " if impl:\n", " impl.setAttribute('sourcecode', code)\n", "\n", "\n", "\n", "# Set the source code for all targets based on the nodedef.\n", "# In this case all inline implementations are identical.\n", "sourceCode = [ '{{in1}} + {{in2}}' ]\n", "sourceCode = sourceCode * len(targets)\n", "for nodedef in inlined_doc.getNodeDefs():\n", " if nodedef.hasSourceUri():\n", " continue\n", " setImplementationSourceCode_v1(inlined_doc, nodedef, targets, sourceCode)\n", "\n", "docString = mxf.MtlxFile.writeDocumentToString(inlined_doc, mxf.MtlxFile.skipLibraryElement)\n", "writeDocToMarkdown(docString)\n", "\n", "mxf.MtlxFile.writeDocumentToFile(inlined_doc, './data/myadd_definition.mtlx', mxf.MtlxFile.skipLibraryElement)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "#### 2.3.2 Adding Source Code Files For Implementations\n", "\n", "If the code cannot be inlined then a new function name is required, with the general guideline to prefix the function name with the string `mx_` followed by catagory and type. For consistency the file names for source code will use the same convention.\n", "\n", "Thus for this example:\n", "* `mx_myadd_color3` is the function name and\n", "* `mx_myadd_color3.` is used for the shader name, where `` is the native shading language suffix name (e.g. `osl` for the OSL shading language or `msl` for Metal) \n", "\n", "The utility function `setImplementationSourceCode_v1()` is extended to differentiate between inline and file source code and called `setImplementationSourceCode()`. The API interfaces Implementation.setFunction() and\n", "Implementation.setFile() " ] }, { "cell_type": "code", "execution_count": 166, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "def setImplementationSourceCode(doc, nodedef, targets, sourceCode, inlined):\n", " '''\n", " Add source code references.\n", " - If inlined then the code is embedded in the `sourcecode` attribute\n", " - If stored in a file then the filename is set using setFile() and the function set using setFunction()\n", " '''\n", " type = nodedef.getType()\n", " category = nodedef.getNodeString()\n", " implName = createImplIdentifier(nodedef.getName().removeprefix('ND_'))\n", " #implName = 'IM_' + category + '_' + type \n", "\n", " # Set inlined code\n", " if inlined:\n", " for target, code in zip(targets, sourceCode):\n", " impl = doc.getImplementation(implName + '_' + target)\n", " if impl:\n", " impl.setAttribute('sourcecode', code)\n", " # Set file / function code reference\n", " else:\n", " functionName = 'mx_' + category + '_' + type\n", " for target, code in zip(targets, sourceCode):\n", " impl = doc.getImplementation(implName + '_' + target)\n", " if impl:\n", " fileName = functionName\n", " impl.setFunction(functionName)\n", " fileExtension = target.removeprefix('gen')\n", " fileName = functionName + '.' + fileExtension\n", " impl.setFile(fileName)\n", "\n", " # Note: A possible option to add here would be to create the actual source files.\n", "\n", "filesource_doc = mx.createDocument()\n", "filesource_doc.copyContentFrom(sourceCodeDoc)\n", "\n", "targets = getTargetDefs(filesource_doc)\n", "for nodedef in filesource_doc.getNodeDefs():\n", " if nodedef.hasSourceUri():\n", " continue\n", " createImplementations(filesource_doc, nodedef, targets)\n", "\n", " sourceCode = [ 'placeholder text']\n", " sourceCode = sourceCode * len(targets)\n", " setImplementationSourceCode(filesource_doc, nodedef, targets, sourceCode, False)\n", "\n", " break\n", "\n", "docString = mxf.MtlxFile.writeDocumentToString(filesource_doc, mxf.MtlxFile.skipLibraryElement)\n", "writeDocToMarkdown(docString)\n", "\n", "mxf.MtlxFile.writeDocumentToFile(filesource_doc, './data/myadd_definition_file.mtlx', mxf.MtlxFile.skipLibraryElement)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## 3. Adding Definitions To A \"Library\"\n", "\n", "When a definition is refined to the point where it can be made available generally, there are a few choices to how they are organized and where they will reside. For instance a new definition could either be part of a custom library or potentially contributed back the the MaterialX standard libraries.\n", "\n", "### 3.1 \"Standard\" Library Organization\n", "\n", "Below shows a layout for how the standard libraries are organized on the left.\n", "\n", "\n", "\n", "For example if we consider the grouping on the left to be `stdlib`, then it is composed of:\n", "* A separate file `stdlib_defs.mtlx` containing all definitions (`_defs`)\n", "* A separate file `stdlib_ng.mtlx` containing all Functional node graph implementations. (`_ng`)\n", "* A separate file containing all per-target source code implementation reference. Files are of in per-target sub-folders and of the form: `/stdlib__impl.mtlx` files. For example `genglsl/stdlib_genglsl_impl.mtlx` is the implementation file for GLSL (genglsl target)).\n", "\n", "This structure is repeated for the pbr library: `pbrlib`.\n", "\n", "Higher level functional nodegraph implementation-only libraries such as `bxdf` are built on top of `pbrilb` and `stdlib`. \n", "\n", "### 3.2 Custom Library Organization\n", "In the diagram we show a custom library (on the right) which reflects the \"standard\" libraries. \n", "\n", "There are however many choices as shown on the far right.\n", "\n", "For instance functional nodegraphs could be kept separate from other graphs, and/or they could be kept within the file as definition or as two separate files. The same holds true implementations and implementation files. For instance the implementation, functional nodegraph and definition could all reside in the same file as a self-contained grouping.\n", "\n", "### 3.3 Semantic Groupings\n", "Different attributes could be used for organization such as `category`, `node group`, `namespace`and `version`.\n", "\n", "### 3.4 Library Identification and Dependencies\n", "Note that there is no formal concept of a library and thus no concept of library or definition dependencies. For example `stdlib` is just a folder name where definitions reside. The definitions themselves have no reference to a given library identifier.\n", "\n", "If a definition from the `bxdf` library is created without loading in `stdlib` and / or `pbrlib` then this dependency may only be detected at graph evaluation time (e.g. for code generation)\n", "\n", "Also as noted, `include` dependencies are **not** recommended to be specified explicitly as they are file references. There is no concept of library identifier dependence." ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "\n", "### 3.5 Adding A Custom Definition to the \"Standard\" library\n", "\n", "A desirable feature is to be able to add definitions into the MaterialX standard libraries.\n", "As definitions have no delineation within a file (beyond an XML comment string), an initial option is to just \n", "append the definitions, implementations, and functional graphs into the appropriate files.\n", "\n", "At time of writing, utilities to aid in this process are under discussion / design currently, with a possible recommended\n", "workflow forth-coming. Note that version `1.38.8` is the minimum version to be able to preserve comments and\n", "newlines properly on XML load and save.\n", "\n", "#### 3.5.1 Handling Definitions with Functional Graph Implementations \n", "In this example, we will take the nodegraph from the Marble example and produce separate documents using the\n", "`addDefinitionToDocument()`.\n" ] }, { "cell_type": "code", "execution_count": 167, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Using 1.39 or later API to create definition...\n" ] }, { "data": { "text/markdown": [ "##### Definition document" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "##### Functional Graph document" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "```xml\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", "\n", "```\n" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "doc, libFiles, status = mxf.MtlxFile.createWorkingDocument()\n", "mx.readFromXmlFile(doc, 'data/standard_surface_marble_solid.mtlx')\n", "\n", "nodeGraph = doc.getNodeGraph('NG_marble1')\n", "\n", "category = 'mymarble'\n", "\n", "cparam = {}\n", "cparam['nodedefName'] = 'ND_' + category\n", "cparam['category'] = category\n", "cparam['version'] = '1.0'\n", "cparam['defaultversion'] = True\n", "cparam['nodegroup'] = 'texture2d'\n", "cparam['nodegraphName'] = 'NG_' + category \n", "definition, funcgraph = createDefinitionAndFunctionalGraph(nodeGraph, cparam)\n", "\n", "# Add documentation and namespace as well as patch up definition and functional graph\n", "documentation = 'Custom marble texture defintion: ' + category\n", "namespace = ''\n", "patchDefinition(definition, funcgraph, documentation, namespace)\n", "\n", "defDoc = mx.createDocument()\n", "graphDoc = mx.createDocument()\n", "addDefinitionToDocument(definition, funcgraph, defDoc, graphDoc, 'Custom marble definition: : mymarble ', 'Functional graph implementation of custom marble: mymarble ')\n", "\n", "writeToMarkdown('##### Definition document')\n", "documentContents = writeDocToString(defDoc)\n", "writeDocToMarkdown(documentContents)\n", "\n", "writeToMarkdown('##### Functional Graph document')\n", "documentContents = writeDocToString(graphDoc)\n", "writeDocToMarkdown(documentContents)\n", "\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ " If the `stdlib` files are used instead then the definition and graph will be appended to\n", "existing files.\n", "\n", "The first thing to discover is what are the relevant library files. The helper\n", "`getStandardLibraryFilePaths()` will return the location of the definition file, nodegraph file, and\n", "any implementation files per target." ] }, { "cell_type": "code", "execution_count": 168, "metadata": {}, "outputs": [], "source": [ "def getStandardLibraryFilePaths(library, targets=[]):\n", " '''\n", " Get file paths based on a \"standard\" library configuration\n", " ''' \n", " DEFS_POSTFIX = '_defs'\n", " GRAPH_POSTFIX = '_ng'\n", " MTLX_EXTENSION = 'mtlx'\n", " IMPL_POSTFIX = '_impl'\n", "\n", " rootFilePath = mx.FilePath(library)\n", "\n", " defFilePath = mx.FilePath(library + DEFS_POSTFIX)\n", " defFilePath.addExtension(MTLX_EXTENSION)\n", " defFilePath = rootFilePath / defFilePath\n", "\n", " graphFilePath = mx.FilePath(library + GRAPH_POSTFIX)\n", " graphFilePath.addExtension(MTLX_EXTENSION)\n", " graphFilePath = rootFilePath / graphFilePath\n", "\n", " implFilePaths = []\n", " for target in targets:\n", " targetRoot = mx.FilePath(target)\n", " targetPath = mx.FilePath(library + '_' + target + IMPL_POSTFIX)\n", " targetPath.addExtension(MTLX_EXTENSION)\n", " targetPath = rootFilePath / targetRoot / targetPath\n", " implFilePaths.append(targetPath)\n", "\n", " return defFilePath, graphFilePath, implFilePaths\n", "\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "This helper is used to find all the relevant files for `stdlib` relative to the default `libraries` folder:" ] }, { "cell_type": "code", "execution_count": 169, "metadata": {}, "outputs": [ { "data": { "text/markdown": [ "### File Paths for Library: stdlib " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Definition File: stdlib\\stdlib_defs.mtlx" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Functional Graph File: stdlib\\stdlib_ng.mtlx" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Target( genglsl ) implementation file: stdlib\\genglsl\\stdlib_genglsl_impl.mtlx" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Target( genmdl ) implementation file: stdlib\\genmdl\\stdlib_genmdl_impl.mtlx" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Target( genmsl ) implementation file: stdlib\\genmsl\\stdlib_genmsl_impl.mtlx" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/markdown": [ "* Target( genosl ) implementation file: stdlib\\genosl\\stdlib_genosl_impl.mtlx" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Find the files used for `stdlib`\n", "targets = getTargetDefs(doc)\n", "libraryName = 'stdlib'\n", "defFilePath, graphFilePath, implFilePaths = getStandardLibraryFilePaths(libraryName, targets)\n", "writeToMarkdown('### File Paths for Library: %s ' % libraryName)\n", "writeToMarkdown('* Definition File: %s' % defFilePath.asString())\n", "writeToMarkdown('* Functional Graph File: %s' % graphFilePath.asString())\n", "for implPath, target in zip(implFilePaths, targets): \n", " writeToMarkdown('* Target( %s ) implementation file: %s' % (target, implPath.asString()))" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "With these file names available we can load in these documents, append to them and write them back out.\n", "Note that we turn on preservation of both comments and newlines so as to not lose any of the original formatting.\n", "( Newline preservation is available as of version 1.38.8 )." ] }, { "cell_type": "code", "execution_count": 170, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Functional graph: NG_mymarble added to: libraries\\stdlib\\stdlib_ng.mtlx\n", "Definition: ND_mymarble added to: libraries\\stdlib\\stdlib_defs.mtlx\n" ] } ], "source": [ "# Get the relative library file names\n", "targets = getTargetDefs(doc)\n", "libraryName = 'stdlib'\n", "defFilePath, graphFilePath, implFilePaths = getStandardLibraryFilePaths(libraryName, targets)\n", "\n", "# Get the default `libraries` location to use as a root for the relative file paths\n", "defaultLibFolder = mx.getDefaultDataLibraryFolders()\n", "defaultSearchPath = mx.getDefaultDataSearchPath()\n", "\n", "# Read in files relative to default library search path\n", "defDoc = mx.createDocument()\n", "defFilePath = mx.FilePath(defaultLibFolder[0]) / defFilePath\n", "mx.readFromXmlFile(defDoc, defFilePath, defaultSearchPath)\n", "\n", "graphDoc = mx.createDocument()\n", "graphFilePath = mx.FilePath(defaultLibFolder[0]) / graphFilePath\n", "mx.readFromXmlFile(graphDoc, graphFilePath, defaultSearchPath)\n", "\n", "# Append the definitions and functional graph to each document\n", "addDefinitionToDocument(definition, funcgraph, defDoc, graphDoc, 'Custom marble definition: : mymarble ', ' Functional graph implementation of custom marble: mymarble ')\n", "\n", "# Examine the document\n", "writeOptions = mx.XmlWriteOptions()\n", "writeOptions.writeXIncludeEnable = False\n", "writeOptions.elementPredicate = mxf.MtlxFile.skipLibraryElement\n", "\n", "documentContents = mx.writeToXmlString(defDoc, writeOptions)\n", "text = '
Standard Libray Definitions with New Definiont\\n\\n' + '```xml\\n' + documentContents + '```\\n' + '
\\n' \n", "# Commented out for performance reasons. Uncomment to set file\n", "#display_markdown(text , raw=True) \n", "\n", "documentContents = mx.writeToXmlString(graphDoc, writeOptions)\n", "text = '
Standard Libray Graphs with New Graph\\n\\n' + '```xml\\n' + documentContents + '```\\n' + '
\\n' \n", "# Commented out for performance reasons. Uncomment to set file\n", "# display_markdown(text , raw=True) \n", "\n", "# Confirm existence and clean-up\n", "findGraph = graphDoc.getNodeGraph(funcgraph.getName())\n", "if findGraph:\n", " print('Functional graph: %s added to: %s' % (funcgraph.getName(), graphFilePath.asString()))\n", " graphDoc.removeChild(funcgraph.getName())\n", " findGraph = graphDoc.getNodeGraph(funcgraph.getName())\n", "\n", "findDef = defDoc.getNodeDef(definition.getName())\n", "if findDef:\n", " print('Definition: %s added to: %s' % (definition.getName(), defFilePath.asString()))\n", " defDoc.removeChild(definition.getName())\n", " findDef = defDoc.getNodeGraph(definition.getName())\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "#### 3.5.2 : Handling Definitions with Source Code Implementation \n", "\n", "In this example we take the previous custom `myadd` definition and add it's definition to the `stdlib` definition file and\n", "add all of it's implementations to eh appropriate per target implementation files.\n", "\n", "The help `addSourceDefinitionToDocument()` handles the additions." ] }, { "cell_type": "code", "execution_count": 171, "metadata": {}, "outputs": [], "source": [ "def addSourceDefinitionToDocument(definition, impls, defDoc, implDocs, defComment='', implComment=''):\n", " '''\n", " Copy a definition and implementations to a new document.\n", " \n", " '''\n", " if definition and impls and defDoc and implDocs:\n", "\n", " # Add definition comment\n", " if defComment:\n", " comment = defDoc.addChildOfCategory('comment')\n", " comment.setDocString(defComment) \n", "\n", " # Create a new definition, and copy the content over. Make sure\n", " # to use the existing name and category.\n", " newDef = defDoc.addNodeDef(definition.getName(), '', definition.getCategory())\n", " newDef.copyContentFrom(definition) \n", "\n", " # Add implementations to appropriate implementation documents\n", " for impl, implDoc in zip(impls, implDocs):\n", "\n", " if not implDoc:\n", " continue\n", "\n", " # Add impl comment\n", " if implComment:\n", " comment = implDoc.addChildOfCategory('comment')\n", " comment.setDocString(implComment) \n", "\n", " # Create a new graph and copy the contents over. This will result in a functional graph.\n", " # Use the definiton document if no graph document specified\n", " newImpl = implDoc.addImplementation(impl.getName())\n", " newImpl.copyContentFrom(impl)\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Use the help, each definition with inline implementations previously created is added to `stdlib`. Definitions with non-inlined implementations would use the same logic as the only difference is the source code references stored on the attributes of the implementations." ] }, { "cell_type": "code", "execution_count": 172, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "- Definition: ND_myadd_1_0_color3 added to: libraries\\stdlib\\stdlib_defs.mtlx\n", "- Source implementation: IMPL_myadd_1_0_color3_genglsl added to file: libraries\\stdlib\\genglsl\\stdlib_genglsl_impl.mtlx\n", "- Source implementation: IMPL_myadd_1_0_color3_genmdl added to file: libraries\\stdlib\\genmdl\\stdlib_genmdl_impl.mtlx\n", "- Source implementation: IMPL_myadd_1_0_color3_genmsl added to file: libraries\\stdlib\\genmsl\\stdlib_genmsl_impl.mtlx\n", "- Source implementation: IMPL_myadd_1_0_color3_genosl added to file: libraries\\stdlib\\genosl\\stdlib_genosl_impl.mtlx\n" ] } ], "source": [ "\n", "if inlined_impls:\n", " implDocs = []\n", " implDocPaths = []\n", " skipped_targets = []\n", " for implFilePath, target in zip(implFilePaths, targets):\n", " implFilePath = mx.FilePath(defaultLibFolder[0]) / implFilePath\n", " implDocPaths.append(implFilePath)\n", " implDoc = mx.createDocument()\n", " try:\n", " mx.readFromXmlFile(implDoc, implFilePath, defaultSearchPath)\n", " implDocs.append(implDoc)\n", " except mx.ExceptionFileMissing as err:\n", " implDocs.append(implDoc)\n", " skipped_targets.append(target)\n", " print('No target (%s) impl file to append to %s' % (target, implFilePath.asString()))\n", " \n", " # Add the definition to the definition document and implementations to the implementation documents. \n", " addSourceDefinitionToDocument(inline_definition, inlined_impls, defDoc, implDocs, 'Custom add definition (mxadd)', 'Custom add implementation (mxadd)')\n", "\n", " # Examine the definition document\n", " documentContents = mx.writeToXmlString(defDoc, writeOptions)\n", " text = '
Standard Libray Definitions with New Definition\\n\\n' + '```xml\\n' + documentContents + '```\\n' + '
\\n' \n", " # Uncommented out due to performance of displaying text. Uncomment to set actual files.\n", " #display_markdown(text , raw=True) \n", "\n", " findDef = defDoc.getNodeDef(inline_definition.getName())\n", " if findDef:\n", " print('- Definition: %s added to: %s' % (inline_definition.getName(), defFilePath.asString()))\n", " defDoc.removeChild(inline_definition.getName())\n", " findDef = defDoc.getNodeGraph(inline_definition.getName())\n", "\n", " # Examine the implementation documents\n", " for implDoc, inline_impl, target, implPath in zip(implDocs, inlined_impls, targets, implDocPaths):\n", " if target in skipped_targets:\n", " continue\n", "\n", " documentContents = mx.writeToXmlString(implDoc, writeOptions)\n", " text = '
Implementation for target ' + target + '\\n\\n' + '```xml\\n' + documentContents + '```\\n' + '
\\n' \n", " # Uncommented out due to performance of displaying text. Uncomment to set actual files.\n", " #display_markdown(text , raw=True) \n", "\n", " implName = inline_impl.getName()\n", " if implDoc.getImplementation(implName):\n", " print('- Source implementation: %s added to file: %s' % (implName, implPath.asString()))\n", " implDoc.removeChild(implName)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.10" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }