Structured data source driver (#5165)

* proto: add structured data source types

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

* make gen

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

* Add structured data source

This commit adds the first version of the structured data source.

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

* go mod tidy

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

* Wire struct data source to DS machinery

This commit connects the new structured data source to the data source
machinery.

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

* Add structured DS tests

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

---------

Signed-off-by: Adolfo García Veytia (puerco) <[email protected]>

go.mod

@@ -346,7 +346,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect

internal/datasources/factory.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/mindersec/minder/internal/datasources/rest"
+	"github.com/mindersec/minder/internal/datasources/structured"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
 	v1datasources "github.com/mindersec/minder/pkg/datasources/v1"
 )
@@ -24,6 +25,8 @@ func BuildFromProtobuf(ds *minderv1.DataSource) (v1datasources.DataSource, error
 	}
 
 	switch ds.GetDriver().(type) {
+	case *minderv1.DataSource_Structured:
+		return structured.NewStructDataSource(ds.GetStructured())
 	case *minderv1.DataSource_Rest:
 		return rest.NewRestDataSource(ds.GetRest())
 	default:

internal/datasources/service/convert.go

@@ -33,6 +33,8 @@ func dataSourceDBToProtobuf(ds db.DataSource, dsfuncs []db.DataSourcesFunction)
 	dsfType := dsfuncs[0].Type
 
 	switch dsfType {
+	case v1datasources.DataSourceDriverStruct:
+		return dataSourceStructDBToProtobuf(outds, dsfuncs)
 	case v1datasources.DataSourceDriverRest:
 		return dataSourceRestDBToProtobuf(outds, dsfuncs)
 	default:
@@ -60,3 +62,27 @@ func dataSourceRestDBToProtobuf(ds *minderv1.DataSource, dsfuncs []db.DataSource
 
 	return ds, nil
 }
+
+func dataSourceStructDBToProtobuf(ds *minderv1.DataSource, dsfuncs []db.DataSourcesFunction) (*minderv1.DataSource, error) {
+	structured := &minderv1.StructDataSource{
+		Def: make(map[string]*minderv1.StructDataSource_Def, len(dsfuncs)),
+	}
+
+	for _, dsf := range dsfuncs {
+		key := dsf.Name
+		dsfToParse := &minderv1.StructDataSource_Def{
+			Path: &minderv1.StructDataSource_Def_Path{},
+		}
+		if err := protojson.Unmarshal(dsf.Definition, dsfToParse); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal data source definition for %s: %w", key, err)
+		}
+
+		structured.Def[key] = dsfToParse
+	}
+
+	ds.Driver = &minderv1.DataSource_Structured{
+		Structured: structured,
+	}
+
+	return ds, nil
+}

internal/datasources/service/service.go

@@ -431,6 +431,23 @@ func addDataSourceFunctions(
 	projectID uuid.UUID,
 ) error {
 	switch drv := ds.GetDriver().(type) {
+	case *minderv1.DataSource_Structured:
+		for name, def := range drv.Structured.GetDef() {
+			defBytes, err := protojson.Marshal(def)
+			if err != nil {
+				return fmt.Errorf("failed to marshal structured data definition: %w", err)
+			}
+
+			if _, err := tx.AddDataSourceFunction(ctx, db.AddDataSourceFunctionParams{
+				DataSourceID: dsID,
+				ProjectID:    projectID,
+				Name:         name,
+				Type:         v1datasources.DataSourceDriverStruct,
+				Definition:   defBytes,
+			}); err != nil {
+				return fmt.Errorf("failed to create data source function: %w", err)
+			}
+		}
 	case *minderv1.DataSource_Rest:
 		for name, def := range drv.Rest.GetDef() {
 			defBytes, err := protojson.Marshal(def)

internal/datasources/structured/decoders.go

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package structured
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/pelletier/go-toml/v2"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	// ErrNoInput triggers if a decoder is called without input
+	ErrNoInput = errors.New("unable to decode, no input defined")
+)
+
+// This file contains various decoders that can be used to decode structured data
+
+// jsonDecoder decodes JSON data
+type jsonDecoder struct{}
+
+func (*jsonDecoder) Parse(r io.Reader) (any, error) {
+	if r == nil {
+		return nil, ErrNoInput
+	}
+	var res any
+	dec := json.NewDecoder(r)
+	if err := dec.Decode(&res); err != nil {
+		return nil, fmt.Errorf("decoding json data: %w", err)
+	}
+	return res, nil
+}
+
+func (*jsonDecoder) Extensions() []string {
+	return []string{"json"}
+}
+
+// yamlDecoder opens yaml
+type yamlDecoder struct{}
+
+func (*yamlDecoder) Parse(r io.Reader) (any, error) {
+	if r == nil {
+		return nil, ErrNoInput
+	}
+	var res any
+	dec := yaml.NewDecoder(r)
+	if err := dec.Decode(&res); err != nil {
+		return nil, fmt.Errorf("decoding yaml data: %w", err)
+	}
+	return res, nil
+}
+
+func (*yamlDecoder) Extensions() []string {
+	return []string{"yaml", "yml"}
+}
+
+type tomlDecoder struct{}
+
+func (*tomlDecoder) Parse(r io.Reader) (any, error) {
+	if r == nil {
+		return nil, ErrNoInput
+	}
+	var res any
+	dec := toml.NewDecoder(r)
+	if err := dec.Decode(&res); err != nil {
+		return nil, fmt.Errorf("decoding toml data: %w", err)
+	}
+	return res, nil
+}
+
+func (*tomlDecoder) Extensions() []string {
+	return []string{"toml"}
+}

internal/datasources/structured/decoders_test.go

@@ -0,0 +1,110 @@
+// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package structured
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestJsonDecoder(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name    string
+		data    []byte
+		mustErr bool
+		expect  any
+	}{
+		{name: "normal", data: []byte(`{"a":1, "b":"abc"}`), mustErr: false},
+		{name: "invalid_json", data: []byte(`a 1`), mustErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var b bytes.Buffer
+			_, err := b.Write(tc.data)
+			require.NoError(t, err)
+
+			dec := jsonDecoder{}
+			res, err := dec.Parse(&b)
+			if tc.mustErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, res)
+		})
+		dec := jsonDecoder{}
+		_, err := dec.Parse(nil)
+		require.Error(t, err)
+
+	}
+}
+
+func TestYamlDecoder(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name    string
+		data    []byte
+		mustErr bool
+		expect  any
+	}{
+		{name: "normal", data: []byte("---\na: 1\nb:\n  - \"Hey\"\n  - \"Bye\"\n"), mustErr: false},
+		{name: "invalid_yaml", data: []byte("  a 1\na: 2\n"), mustErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var b bytes.Buffer
+			_, err := b.Write(tc.data)
+			require.NoError(t, err)
+
+			dec := yamlDecoder{}
+			res, err := dec.Parse(&b)
+			if tc.mustErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, res)
+		})
+		dec := yamlDecoder{}
+		_, err := dec.Parse(nil)
+		require.Error(t, err)
+
+	}
+}
+
+func TestTomlDecoder(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		name    string
+		data    []byte
+		mustErr bool
+		expect  any
+	}{
+		{name: "normal", data: []byte("title = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\""), mustErr: false},
+		{name: "invalid_toml", data: []byte("  a 1\na: 2\n"), mustErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			var b bytes.Buffer
+			_, err := b.Write(tc.data)
+			require.NoError(t, err)
+
+			dec := tomlDecoder{}
+			res, err := dec.Parse(&b)
+			if tc.mustErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, res)
+		})
+		dec := tomlDecoder{}
+		_, err := dec.Parse(nil)
+		require.Error(t, err)
+
+	}
+}