Support JSON Marshalling augments to compressed schemas (#572)

* Correctly marshal JSON for augments in compressed containers.

* Extend `findSchemaPath` and `findMapPaths` to return a path of module names that correspond with the paths.
  e.g. "(shadow-)path:config/name|name" -> "(shadow-)module:modA/modB|modA"
* Removed a small existing hack whereby error from `InstantiatingModule` was being ignored.
* Change `structJSON` to use the corresponding modules when determining whether to prefix node names with corresponding module names.
* Added helper `structTagToLibModules` sister to existing `structTagToLibPaths` to extract modules tag.

Testing:
* Added compressed augment unit test for `findMapPaths` and JSON marshalling.
* Added integration test for generating a compressed augment.
* Other integration tests updated to test `module` and `shadow-module` generation.

Other Testing Changes:
* Changed tests for `findSchemaPath` and `findMapPaths` to create `yang.Entry` objects using `Modules.Process` instead of manually creating them.
* Populated `.Modules` field for other tests in order to avoid errors due to above-mentioned hack being removed.

* Move function to util and refactor structJSON.

* Update goyang to v0.2.9

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/google/go-cmp v0.5.5
 	github.com/kylelemons/godebug v1.1.0
 	github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be
-	github.com/openconfig/goyang v0.2.8
+	github.com/openconfig/goyang v0.2.9
 	github.com/openconfig/gribi v0.1.1-0.20210423184541-ce37eb4ba92f
 	github.com/pmezard/go-difflib v1.0.0
 	google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d

go.sum

@@ -44,8 +44,8 @@ github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be h1:VEK8utxoyZu/hkp
 github.com/openconfig/gnmi v0.0.0-20200508230933-d19cebf5e7be/go.mod h1:M/EcuapNQgvzxo1DDXHK4tx3QpYM/uG4l591v33jG2A=
 github.com/openconfig/goyang v0.0.0-20200115183954-d0a48929f0ea/go.mod h1:dhXaV0JgHJzdrHi2l+w0fZrwArtXL7jEFoiqLEdmkvU=
 github.com/openconfig/goyang v0.2.2/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8=
-github.com/openconfig/goyang v0.2.8 h1:TAdUM9k+MP3GccppsrvJhDEnOA7zpp7hpakKI7qczPU=
-github.com/openconfig/goyang v0.2.8/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8=
+github.com/openconfig/goyang v0.2.9 h1:Z95LskKYk6nBYOxHtmJCu3YEKlr3pJLWG1tYAaNh3yU=
+github.com/openconfig/goyang v0.2.9/go.mod h1:vX61x01Q46AzbZUzG617vWqh/cB+aisc+RrNkXRd3W8=
 github.com/openconfig/gribi v0.1.1-0.20210423184541-ce37eb4ba92f h1:8vRtC+y0xh9BYPrEGf/jG/paYXiDUJ6P8iYt5rCVols=
 github.com/openconfig/gribi v0.1.1-0.20210423184541-ce37eb4ba92f/go.mod h1:OoH46A2kV42cIXGyviYmAlGmn6cHjGduyC2+I9d/iVs=
 github.com/openconfig/ygot v0.6.0/go.mod h1:o30svNf7O0xK+R35tlx95odkDmZWS9JyWWQSmIhqwAs=

testdata/modules/openconfig-simple-augment.yang

@@ -15,5 +15,9 @@ module openconfig-simple-augment {
       }
     }
   }
+
+  augment "/t:native/state" {
+    leaf b { type string; }
+  }
 }
 

ygen/directory.go

@@ -137,67 +137,92 @@ func GetOrderedDirectories(directory map[string]*Directory) ([]string, map[strin
 // of a Directory. The Field is specified as a name in order to guarantee its
 // existence before processing.
 func FindSchemaPath(parent *Directory, fieldName string, absolutePaths bool) ([]string, error) {
-	return findSchemaPath(parent, fieldName, false, absolutePaths)
+	schemaPaths, _, err := findSchemaPath(parent, fieldName, false, absolutePaths)
+	return schemaPaths, err
 }
 
 // findSchemaPath finds the relative or absolute schema path of a given field
-// of a Directory, or the shadowed field path (field duplicated and
-// deprioritized via compression) of a Directory. The Field is specified as a
-// name in order to guarantee its existence before processing.
+// of a Directory, or the shadowed field path (i.e. field duplicated and
+// deprioritized via compression) of a Directory. The first returned slice
+// contains the names of the path elements, and the second contains the
+// corresponding module names for each path element's resident namespace. The
+// Field is specified as a name in order to guarantee its existence before
+// processing.
 // NOTE: If shadowSchemaPaths is true, no error is returned if fieldName is not found.
-func findSchemaPath(parent *Directory, fieldName string, shadowSchemaPaths, absolutePaths bool) ([]string, error) {
+func findSchemaPath(parent *Directory, fieldName string, shadowSchemaPaths, absolutePaths bool) ([]string, []string, error) {
 	field, ok := parent.Fields[fieldName]
 	if shadowSchemaPaths {
 		if field, ok = parent.ShadowedFields[fieldName]; !ok {
-			return nil, nil
+			return nil, nil, nil
 		}
 	}
 	if !ok {
-		return nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): field name %q does not exist in Directory %s", shadowSchemaPaths, fieldName, parent.Path)
+		return nil, nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): field name %q does not exist in Directory %s", shadowSchemaPaths, fieldName, parent.Path)
 	}
 	fieldSlicePath := util.SchemaPathNoChoiceCase(field)
+	var fieldSliceModules []string
+	for _, e := range util.SchemaEntryPathNoChoiceCase(field) {
+		im, err := e.InstantiatingModule()
+		if err != nil {
+			return nil, nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): cannot find instantiating module for field %q in Directory %s: %v", shadowSchemaPaths, fieldName, parent.Path, err)
+		}
+		fieldSliceModules = append(fieldSliceModules, im)
+	}
 
 	if absolutePaths {
-		return append([]string{""}, fieldSlicePath[1:]...), nil
+		return append([]string{""}, fieldSlicePath[1:]...), append([]string{""}, fieldSliceModules[1:]...), nil
 	}
 	// Return the elements that are not common between the two paths.
 	// Since the field is necessarily a child of the parent, then to
 	// determine those elements of the field's path that are not contained
 	// in the parent's, we walk from index X of the field's path (where X
 	// is the number of elements in the path of the parent).
 	if len(fieldSlicePath) < len(parent.Path) {
-		return nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): field %v is not a valid child of %v", shadowSchemaPaths, fieldSlicePath, parent.Path)
+		return nil, nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): field %v is not a valid child of %v", shadowSchemaPaths, fieldSlicePath, parent.Path)
 	}
-	return fieldSlicePath[len(parent.Path)-1:], nil
+	return fieldSlicePath[len(parent.Path)-1:], fieldSliceModules[len(parent.Path)-1:], nil
 }
 
-// findMapPaths takes an input field name for a parent Directory and calculates the set of schemapaths that it represents.
-// If absolutePaths is set, the paths are absolute otherwise they are relative to the parent. If
+// findMapPaths takes an input field name for a parent Directory and calculates
+// the set of schema paths it represents, as well as the corresponding module
+// names for each schema path element's resident namespace.
+// If absolutePaths is set, the paths are absolute; otherwise, they are relative to the parent. If
 // the input entry is a key to a list, and is of type leafref, then the corresponding target leaf's
 // path is also returned. If shadowSchemaPaths is set, then the path of the
 // field deprioritized via compression is returned instead of the prioritized paths.
 // The first returned path is the path of the direct child, with the shadow
 // child's path afterwards, and the key leafref, if any, last.
-func findMapPaths(parent *Directory, fieldName string, compressPaths, shadowSchemaPaths, absolutePaths bool) ([][]string, error) {
-	childPath, err := findSchemaPath(parent, fieldName, shadowSchemaPaths, absolutePaths)
+func findMapPaths(parent *Directory, fieldName string, compressPaths, shadowSchemaPaths, absolutePaths bool) ([][]string, [][]string, error) {
+	childPath, childModulePath, err := findSchemaPath(parent, fieldName, shadowSchemaPaths, absolutePaths)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	var mapPaths [][]string
+	var mapPaths, mapModulePaths [][]string
 	if childPath != nil {
 		mapPaths = append(mapPaths, childPath)
 	}
+	if childModulePath != nil {
+		mapModulePaths = append(mapModulePaths, childModulePath)
+	}
 	// Only for compressed data schema paths for list fields do we have the
 	// possibility for a direct leafref path as a second path for the field.
 	if !compressPaths || parent.ListAttr == nil {
-		return mapPaths, nil
+		return mapPaths, mapModulePaths, nil
 	}
 
 	field, ok := parent.Fields[fieldName]
 	if !ok {
-		return nil, fmt.Errorf("field name %s does not exist in Directory %s", fieldName, parent.Path)
+		return nil, nil, fmt.Errorf("field name %s does not exist in Directory %s", fieldName, parent.Path)
 	}
 	fieldSlicePath := util.SchemaPathNoChoiceCase(field)
+	var fieldSliceModules []string
+	for _, e := range util.SchemaEntryPathNoChoiceCase(field) {
+		im, err := e.InstantiatingModule()
+		if err != nil {
+			return nil, nil, fmt.Errorf("FindSchemaPath(shadowSchemaPaths:%v): cannot find instantiating module for field %q in Directory %s: %v", shadowSchemaPaths, fieldName, parent.Path, err)
+		}
+		fieldSliceModules = append(fieldSliceModules, im)
+	}
 
 	// Handle specific issue of compressed path schemas, where a key of the
 	// parent list is a leafref to this leaf.
@@ -210,7 +235,7 @@ func findMapPaths(parent *Directory, fieldName string, compressPaths, shadowSche
 		// leafref leaf within the schema as well as the target of the
 		// leafref.
 		if k.Parent == nil || k.Parent.Parent == nil || k.Parent.Parent.Dir[k.Name] == nil || k.Parent.Parent.Dir[k.Name].Type == nil {
-			return nil, fmt.Errorf("invalid compressed schema, could not find the key %s or the grandparent of %s", k.Name, k.Path())
+			return nil, nil, fmt.Errorf("invalid compressed schema, could not find the key %s or the grandparent of %s", k.Name, k.Path())
 		}
 
 		// If a key of the list is a leafref that points to the field,
@@ -223,15 +248,19 @@ func findMapPaths(parent *Directory, fieldName string, compressPaths, shadowSche
 			// list, since the YANG specification enforces that keys are direct
 			// children of the list.
 			keyPath := []string{fieldSlicePath[len(fieldSlicePath)-1]}
+			keyModulePath := []string{fieldSliceModules[len(fieldSliceModules)-1]}
 			if absolutePaths {
 				// If absolute paths are required, then the 'config' or 'state' container needs to be omitted from
 				// the complete path for the secondary mapping.
 				keyPath = append([]string{""}, fieldSlicePath[1:len(fieldSlicePath)-2]...)
 				keyPath = append(keyPath, fieldSlicePath[len(fieldSlicePath)-1])
+				keyModulePath = append([]string{""}, fieldSliceModules[1:len(fieldSliceModules)-2]...)
+				keyModulePath = append(keyModulePath, fieldSliceModules[len(fieldSliceModules)-1])
 			}
 			mapPaths = append(mapPaths, keyPath)
+			mapModulePaths = append(mapModulePaths, keyModulePath)
 			break
 		}
 	}
-	return mapPaths, nil
+	return mapPaths, mapModulePaths, nil
 }