Refactor Graph de/serialization

bin/meshroom_batch

@@ -154,10 +154,10 @@ with meshroom.core.graph.GraphModification(graph):
     # initialize template pipeline
     loweredPipelineTemplates = dict((k.lower(), v) for k, v in meshroom.core.pipelineTemplates.items())
     if args.pipeline.lower() in loweredPipelineTemplates:
-        graph.load(loweredPipelineTemplates[args.pipeline.lower()], setupProjectFile=False, publishOutputs=True if args.output else False)
+        graph.initFromTemplate(loweredPipelineTemplates[args.pipeline.lower()], publishOutputs=True if args.output else False)
     else:
         # custom pipeline
-        graph.load(args.pipeline, setupProjectFile=False, publishOutputs=True if args.output else False)
+        graph.initFromTemplate(args.pipeline, publishOutputs=True if args.output else False)
 
     def parseInputs(inputs, uniqueInitNode):
         """Utility method for parsing the input and inputRecursive arguments."""

meshroom/core/graphIO.py

@@ -0,0 +1,228 @@
+from enum import Enum
+from typing import Any, TYPE_CHECKING, Union
+
+import meshroom
+from meshroom.core import Version
+from meshroom.core.attribute import Attribute, GroupAttribute, ListAttribute
+from meshroom.core.node import Node
+
+if TYPE_CHECKING:
+    from meshroom.core.graph import Graph
+
+
+class GraphIO:
+    """Centralize Graph file keys and IO version."""
+
+    __version__ = "2.0"
+
+    class Keys(object):
+        """File Keys."""
+
+        # Doesn't inherit enum to simplify usage (GraphIO.Keys.XX, without .value)
+        Header = "header"
+        NodesVersions = "nodesVersions"
+        ReleaseVersion = "releaseVersion"
+        FileVersion = "fileVersion"
+        Graph = "graph"
+
+    class Features(Enum):
+        """File Features."""
+
+        Graph = "graph"
+        Header = "header"
+        NodesVersions = "nodesVersions"
+        PrecomputedOutputs = "precomputedOutputs"
+        NodesPositions = "nodesPositions"
+
+    @staticmethod
+    def getFeaturesForVersion(fileVersion: Union[str, Version]) -> tuple["GraphIO.Features", ...]:
+        """Return the list of supported features based on a file version.
+
+        Args:
+            fileVersion (str, Version): the file version
+
+        Returns:
+            tuple of GraphIO.Features: the list of supported features
+        """
+        if isinstance(fileVersion, str):
+            fileVersion = Version(fileVersion)
+
+        features = [GraphIO.Features.Graph]
+        if fileVersion >= Version("1.0"):
+            features += [
+                GraphIO.Features.Header,
+                GraphIO.Features.NodesVersions,
+                GraphIO.Features.PrecomputedOutputs,
+            ]
+
+        if fileVersion >= Version("1.1"):
+            features += [GraphIO.Features.NodesPositions]
+
+        return tuple(features)
+
+
+class GraphSerializer:
+    """Standard Graph serializer."""
+
+    def __init__(self, graph: "Graph") -> None:
+        self._graph = graph
+
+    def serialize(self) -> dict:
+        """
+        Serialize the Graph.
+        """
+        return {
+            GraphIO.Keys.Header: self.serializeHeader(),
+            GraphIO.Keys.Graph: self.serializeContent(),
+        }
+
+    @property
+    def nodes(self) -> list[Node]:
+        return self._graph.nodes
+
+    def serializeHeader(self) -> dict:
+        """Build and return the graph serialization header.
+
+        The header contains metadata about the graph, such as the:
+            - version of the software used to create it.
+            - version of the file format.
+            - version of the nodes types used in the graph.
+            - template flag.
+
+        Args:
+            nodes: (optional) The list of nodes to consider for node types versions - use all nodes if not specified.
+            template: Whether the graph is going to be serialized as a template.
+        """
+        header: dict[str, Any] = {}
+        header[GraphIO.Keys.ReleaseVersion] = meshroom.__version__
+        header[GraphIO.Keys.FileVersion] = GraphIO.__version__
+        header[GraphIO.Keys.NodesVersions] = self._getNodeTypesVersions()
+        return header
+
+    def _getNodeTypesVersions(self) -> dict[str, str]:
+        """Get registered versions of each node types in `nodes`, excluding CompatibilityNode instances."""
+        nodeTypes = set([node.nodeDesc.__class__ for node in self.nodes if isinstance(node, Node)])
+        nodeTypesVersions = {
+            nodeType.__name__: meshroom.core.nodeVersion(nodeType, "0.0") for nodeType in nodeTypes
+        }
+        # Sort them by name (to avoid random order changing from one save to another).
+        return dict(sorted(nodeTypesVersions.items()))
+
+    def serializeContent(self) -> dict:
+        """Graph content serialization logic."""
+        return {node.name: self.serializeNode(node) for node in sorted(self.nodes, key=lambda n: n.name)}
+
+    def serializeNode(self, node: Node) -> dict:
+        """Node serialization logic."""
+        return node.toDict()
+
+
+class TemplateGraphSerializer(GraphSerializer):
+    """Serializer for serializing a graph as a template."""
+
+    def serializeHeader(self) -> dict:
+        header = super().serializeHeader()
+        header["template"] = True
+        return header
+
+    def serializeNode(self, node: Node) -> dict:
+        """Adapt node serialization to template graphs.
+
+        Instead of getting all the inputs and internal attribute keys, only get the keys of
+        the attributes whose value is not the default one.
+        The output attributes, UIDs, parallelization parameters and internal folder are
+        not relevant for templates, so they are explicitly removed from the returned dictionary.
+        """
+        # For now, implemented as a post-process to update the default serialization.
+        nodeData = super().serializeNode(node)
+
+        inputKeys = list(nodeData["inputs"].keys())
+
+        internalInputKeys = []
+        internalInputs = nodeData.get("internalInputs", None)
+        if internalInputs:
+            internalInputKeys = list(internalInputs.keys())
+
+        for attrName in inputKeys:
+            attribute = node.attribute(attrName)
+            # check that attribute is not a link for choice attributes
+            if attribute.isDefault and not attribute.isLink:
+                del nodeData["inputs"][attrName]
+
+        for attrName in internalInputKeys:
+            attribute = node.internalAttribute(attrName)
+            # check that internal attribute is not a link for choice attributes
+            if attribute.isDefault and not attribute.isLink:
+                del nodeData["internalInputs"][attrName]
+
+        # If all the internal attributes are set to their default values, remove the entry
+        if len(nodeData["internalInputs"]) == 0:
+            del nodeData["internalInputs"]
+
+        del nodeData["outputs"]
+        del nodeData["uid"]
+        del nodeData["internalFolder"]
+        del nodeData["parallelization"]
+
+        return nodeData
+
+
+class PartialGraphSerializer(GraphSerializer):
+    """Serializer to serialize a partial graph (a subset of nodes)."""
+
+    def __init__(self, graph: "Graph", nodes: list[Node]):
+        super().__init__(graph)
+        self._nodes = nodes
+
+    @property
+    def nodes(self) -> list[Node]:
+        """Override to consider only the subset of nodes."""
+        return self._nodes
+
+    def serializeNode(self, node: Node) -> dict:
+        """Adapt node serialization to partial graph serialization."""
+        # NOTE: For now, implemented as a post-process to the default serialization.
+        nodeData = super().serializeNode(node)
+
+        # Override input attributes with custom serialization logic, to handle attributes
+        # connected to nodes that are not in the list of nodes to serialize.
+        for attributeName in nodeData["inputs"]:
+            nodeData["inputs"][attributeName] = self._serializeAttribute(node.attribute(attributeName))
+
+        # Clear UID for non-compatibility nodes, as the custom attribute serialization
+        # can be impacting the UID by removing connections to missing nodes.
+        if not node.isCompatibilityNode:
+            del nodeData["uid"]
+
+        return nodeData
+
+    def _serializeAttribute(self, attribute: Attribute) -> Any:
+        """
+        Serialize `attribute` (recursively for list/groups) and deal with attributes being connected
+        to nodes that are not part of the partial list of nodes to serialize.
+        """
+        # If the attribute is connected to a node that is not in the list of nodes to serialize,
+        # the link expression should not be serialized.
+        if attribute.isLink and attribute.getLinkParam().node not in self.nodes:
+            # If part of a list, this entry can be discarded.
+            if isinstance(attribute.root, ListAttribute):
+                return None
+            # Otherwise, return the default value for this attribute.
+            return attribute.defaultValue()
+
+        if isinstance(attribute, ListAttribute):
+            # Recusively serialize each child of the ListAttribute, skipping those for which the attribute
+            # serialization logic above returns None.
+            return [
+                exportValue
+                for child in attribute
+                if (exportValue := self._serializeAttribute(child)) is not None
+            ]
+
+        if isinstance(attribute, GroupAttribute):
+            # Recursively serialize each child of the group attribute.
+            return {name: self._serializeAttribute(child) for name, child in attribute.value.items()}
+
+        return attribute.getExportValue()
+
+

meshroom/core/node.py

@@ -1668,7 +1668,17 @@ def attributeDescFromValue(attrName, value, isOutput):
         elif isinstance(value, float):
             return desc.FloatParam(range=None, **params)
         elif isinstance(value, str):
-            if isOutput or os.path.isabs(value) or Attribute.isLinkExpression(value):
+            if isOutput or os.path.isabs(value):
+                return desc.File(**params)
+            elif Attribute.isLinkExpression(value):
+                # Do not consider link expression as a valid default desc value.
+                # When the link expression is applied and transformed to an actual link,
+                # the systems resets the value using `Attribute.resetToDefaultValue` to indicate
+                # that this link expression has been handled.
+                # If the link expression is stored as the default value, it will never be cleared,
+                # leading to unexpected behavior where the link expression on a CompatibilityNode
+                # could be evaluated several times and/or incorrectly.
+                params["value"] = ""
                 return desc.File(**params)
             else:
                 return desc.StringParam(**params)
@@ -1851,113 +1861,3 @@ def upgrade(self):
     canUpgrade = Property(bool, canUpgrade.fget, constant=True)
     issueDetails = Property(str, issueDetails.fget, constant=True)
 
-
-def nodeFactory(nodeDict, name=None, template=False, uidConflict=False):
-    """
-    Create a node instance by deserializing the given node data.
-    If the serialized data matches the corresponding node type description, a Node instance is created.
-    If any compatibility issue occurs, a NodeCompatibility instance is created instead.
-
-    Args:
-        nodeDict (dict): the serialization of the node
-        name (str): (optional) the node's name
-        template (bool): (optional) true if the node is part of a template, false otherwise
-        uidConflict (bool): (optional) true if a UID conflict has been detected externally on that node
-
-    Returns:
-        BaseNode: the created node
-    """
-    nodeType = nodeDict["nodeType"]
-
-    # Retro-compatibility: inputs were previously saved as "attributes"
-    if "inputs" not in nodeDict and "attributes" in nodeDict:
-        nodeDict["inputs"] = nodeDict["attributes"]
-        del nodeDict["attributes"]
-
-    # Get node inputs/outputs
-    inputs = nodeDict.get("inputs", {})
-    internalInputs = nodeDict.get("internalInputs", {})
-    outputs = nodeDict.get("outputs", {})
-    version = nodeDict.get("version", None)
-    internalFolder = nodeDict.get("internalFolder", None)
-    position = Position(*nodeDict.get("position", []))
-    uid = nodeDict.get("uid", None)
-
-    compatibilityIssue = None
-
-    nodeDesc = None
-    try:
-        nodeDesc = meshroom.core.nodesDesc[nodeType]
-    except KeyError:
-        # Unknown node type
-        compatibilityIssue = CompatibilityIssue.UnknownNodeType
-
-    # Unknown node type should take precedence over UID conflict, as it cannot be resolved
-    if uidConflict and nodeDesc:
-        compatibilityIssue = CompatibilityIssue.UidConflict
-
-    if nodeDesc and not uidConflict:  # if uidConflict, there is no need to look for another compatibility issue
-        # Compare serialized node version with current node version
-        currentNodeVersion = meshroom.core.nodeVersion(nodeDesc)
-        # If both versions are available, check for incompatibility in major version
-        if version and currentNodeVersion and Version(version).major != Version(currentNodeVersion).major:
-            compatibilityIssue = CompatibilityIssue.VersionConflict
-        # In other cases, check attributes compatibility between serialized node and its description
-        else:
-            # Check that the node has the exact same set of inputs/outputs as its description, except
-            # if the node is described in a template file, in which only non-default parameters are saved;
-            # do not perform that check for internal attributes because there is no point in
-            # raising compatibility issues if their number differs: in that case, it is only useful
-            # if some internal attributes do not exist or are invalid
-            if not template and (sorted([attr.name for attr in nodeDesc.inputs 
-                                         if not isinstance(attr, desc.PushButtonParam)]) != sorted(inputs.keys()) or
-                                 sorted([attr.name for attr in nodeDesc.outputs if not attr.isDynamicValue]) !=
-                                 sorted(outputs.keys())):
-                compatibilityIssue = CompatibilityIssue.DescriptionConflict
-
-            # Check whether there are any internal attributes that are invalidating in the node description: if there
-            # are, then check that these internal attributes are part of nodeDict; if they are not, a compatibility
-            # issue must be raised to warn the user, as this will automatically change the node's UID
-            if not template:
-                invalidatingIntInputs = []
-                for attr in nodeDesc.internalInputs:
-                    if attr.invalidate:
-                        invalidatingIntInputs.append(attr.name)
-                for attr in invalidatingIntInputs:
-                    if attr not in internalInputs.keys():
-                        compatibilityIssue = CompatibilityIssue.DescriptionConflict
-                        break
-
-            # Verify that all inputs match their descriptions
-            for attrName, value in inputs.items():
-                if not CompatibilityNode.attributeDescFromName(nodeDesc.inputs, attrName, value):
-                    compatibilityIssue = CompatibilityIssue.DescriptionConflict
-                    break
-            # Verify that all internal inputs match their description
-            for attrName, value in internalInputs.items():
-                if not CompatibilityNode.attributeDescFromName(nodeDesc.internalInputs, attrName, value):
-                    compatibilityIssue = CompatibilityIssue.DescriptionConflict
-                    break
-            # Verify that all outputs match their descriptions
-            for attrName, value in outputs.items():
-                if not CompatibilityNode.attributeDescFromName(nodeDesc.outputs, attrName, value):
-                    compatibilityIssue = CompatibilityIssue.DescriptionConflict
-                    break
-
-    if compatibilityIssue is None:
-        node = Node(nodeType, position, uid=uid, **inputs, **internalInputs, **outputs)
-    else:
-        logging.debug("Compatibility issue detected for node '{}': {}".format(name, compatibilityIssue.name))
-        node = CompatibilityNode(nodeType, nodeDict, position, compatibilityIssue)
-        # Retro-compatibility: no internal folder saved
-        # can't spawn meaningful CompatibilityNode with precomputed outputs
-        # => automatically try to perform node upgrade
-        if not internalFolder and nodeDesc:
-            logging.warning("No serialized output data: performing automatic upgrade on '{}'".format(name))
-            node = node.upgrade()
-        # If the node comes from a template file and there is a conflict, it should be upgraded anyway unless it is
-        # an "unknown node type" conflict (in which case the upgrade would fail)
-        elif template and compatibilityIssue is not CompatibilityIssue.UnknownNodeType:
-            node = node.upgrade()
-
-    return node