update repo/indexer to support custom repo property

application/catalog/application.go

@@ -7,12 +7,12 @@ import (
 	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/whosonfirst/go-whosonfirst-findingaid"
 	"github.com/whosonfirst/go-whosonfirst-findingaid/application"
-	"log"
+	_ "log"
 	"net/url"
 )
 
 var cache_uri string
-var indexer_uri string
+var iterator_uri string
 var findingaid_uri string
 
 type CatalogApplication struct {
@@ -29,9 +29,9 @@ func (app *CatalogApplication) DefaultFlagSet(ctx context.Context) (*flag.FlagSe
 	fs := flagset.NewFlagSet("catalog")
 
 	fs.StringVar(&cache_uri, "cache-uri", "file:///tmp/", "A valid whosonfirst/go-cache URI string.")
-	fs.StringVar(&indexer_uri, "indexer-uri", "repo://", "A valid whosonfirst/go-whosonfirst-iterate URI string.")
+	fs.StringVar(&iterator_uri, "iterator-uri", "repo://", "A valid whosonfirst/go-whosonfirst-iterate URI string.")
 
-	fs.StringVar(&findingaid_uri, "findingaid-uri", "repo://?cache={cache_uri}&indexer={indexer_uri}", "A valid whosonfirst/go-whosonfirst-findingaid URI string.")
+	fs.StringVar(&findingaid_uri, "findingaid-uri", "repo://?cache={cache_uri}&iterator={iterator_uri}", "A valid whosonfirst/go-whosonfirst-findingaid URI string.")
 
 	return fs, nil
 }
@@ -63,20 +63,18 @@ func (app *CatalogApplication) RunWithFlagSet(ctx context.Context, fs *flag.Flag
 		return fmt.Errorf("Failed to parse findingaid URI, %v", err)
 	}
 
-	log.Println("WHAT", indexer_uri)
-
 	fa_q := fa_uri.Query()
 
 	if fa_q.Get("cache") == "{cache_uri}" {
 		fa_q["cache"] = []string{cache_uri}
 	}
 
-	if fa_q.Get("indexer") == "{indexer_uri}" {
-		fa_q["indexer"] = []string{indexer_uri}
+	if fa_q.Get("iterator") == "{iterator_uri}" {
+		fa_q["iterator"] = []string{iterator_uri}
 	}
 
-	if fa_q.Get("indexer") == "" {
-		return fmt.Errorf("Missing '-indexer-uri' flag.")
+	if fa_q.Get("iterator") == "" {
+		return fmt.Errorf("Missing '-iterator-uri' flag.")
 	}
 
 	fa_uri.RawQuery = fa_q.Encode()

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/sfomuseum/go-flags v0.8.2
 	github.com/tidwall/gjson v1.8.0
+	github.com/tidwall/sjson v1.1.7
 	github.com/whosonfirst/go-cache v0.5.0
 	github.com/whosonfirst/go-cache-blob v0.2.0
 	github.com/whosonfirst/go-ioutil v1.0.0

go.sum

@@ -324,6 +324,8 @@ github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
 github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
 github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/tidwall/sjson v1.1.7 h1:sgVPwu/yygHJ2m1pJDLgGM/h+1F5odx5Q9ljG3imRm8=
+github.com/tidwall/sjson v1.1.7/go.mod h1:w/yG+ezBeTdUxiKs5NcPicO9diP38nk96QBAbIIGeFs=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/whosonfirst/algnhsa v0.1.0 h1:8wVksKaz1qor2Y6+fiLEPEPxsEEvUQgSdtSTQLMrnlA=

repo/indexer.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 	"github.com/whosonfirst/go-cache"
 	"github.com/whosonfirst/go-ioutil"
 	"github.com/whosonfirst/go-whosonfirst-findingaid"
@@ -18,8 +20,9 @@ import (
 // Indexer is a struct that implements the findingaid.Indexer interface for information about Who's On First repositories.
 type Indexer struct {
 	findingaid.Indexer
-	cache        cache.Cache
-	iterator_uri string
+	cache         cache.Cache
+	iterator_uri  string
+	repo_property string
 }
 
 func init() {
@@ -71,9 +74,22 @@ func NewIndexer(ctx context.Context, uri string) (findingaid.Indexer, error) {
 		return nil, fmt.Errorf("Invalid ?iterator= parameter, %w", err)
 	}
 
+	// This is necessary for some repos like sfomuseum-data/sfomuseum-data-whosonfirst
+	// where the relevant repo name is stored in a properties.sfomuseum:repo key. This
+	// logic is handled below in IndexReader
+
+	repo_property := WOF_REPO_PROPERTY
+
+	custom_repo := q.Get("repo-property")
+
+	if custom_repo != "" {
+		repo_property = custom_repo
+	}
+
 	fa := &Indexer{
-		cache:        c,
-		iterator_uri: iterator_uri,
+		cache:         c,
+		iterator_uri:  iterator_uri,
+		repo_property: repo_property,
 	}
 
 	return fa, nil
@@ -106,7 +122,37 @@ func (fa *Indexer) IndexURIs(ctx context.Context, sources ...string) error {
 // IndexReader will index an individual Who's On First record in the finding aid.
 func (fa *Indexer) IndexReader(ctx context.Context, fh io.Reader) error {
 
-	rsp, err := FindingAidResponseFromReader(ctx, fh)
+	body, err := io.ReadAll(fh)
+
+	if err != nil {
+		return fmt.Errorf("Failed to read feature, %v", err)
+	}
+
+	// This is necessary for some repos like sfomuseum-data/sfomuseum-data-whosonfirst
+	// where the relevant repo name is stored in a properties.sfomuseum:repo key
+
+	if fa.repo_property != WOF_REPO_PROPERTY {
+
+		path_wof_repo := fmt.Sprintf("properties.%s", WOF_REPO_PROPERTY)
+		path_custom_repo := fmt.Sprintf("properties.%s", fa.repo_property)
+
+		custom_rsp := gjson.GetBytes(body, path_custom_repo)
+
+		// If custom repo property is present then use its value to update wof:repo
+
+		if custom_rsp.Exists() {
+
+			custom_repo := custom_rsp.String()
+
+			body, err = sjson.SetBytes(body, path_wof_repo, custom_repo)
+
+			if err != nil {
+				return fmt.Errorf("Failed to assign custom repo (%s) to %s, %v", custom_repo, WOF_REPO_PROPERTY, err)
+			}
+		}
+	}
+
+	rsp, err := FindingAidResponseFromBytes(ctx, body)
 
 	if err != nil {
 		return err

repo/repo.go

@@ -2,12 +2,16 @@ package repo
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"github.com/tidwall/gjson"
 	"github.com/whosonfirst/go-whosonfirst-uri"
 	"io"
+	_ "log"
 )
 
+const WOF_ID_PROPERTY string = "wof:id"
+const WOF_REPO_PROPERTY string = "wof:repo"
+
 // FindingAidResonse is a struct that contains Who's On First repository information for Who's On First records.
 type FindingAidResponse struct {
 	// The unique Who's On First ID.
@@ -26,18 +30,26 @@ func FindingAidResponseFromReader(ctx context.Context, fh io.Reader) (*FindingAi
 		return nil, err
 	}
 
+	return FindingAidResponseFromBytes(ctx, body)
+}
+
+func FindingAidResponseFromBytes(ctx context.Context, body []byte) (*FindingAidResponse, error) {
+
 	// TO DO: SUPPORT ALT FILES
 
-	id_rsp := gjson.GetBytes(body, "properties.wof:id")
+	path_id := fmt.Sprintf("properties.%s", WOF_ID_PROPERTY)
+	path_repo := fmt.Sprintf("properties.%s", WOF_REPO_PROPERTY)
+
+	id_rsp := gjson.GetBytes(body, path_id)
 
 	if !id_rsp.Exists() {
-		return nil, errors.New("Missing wof:id")
+		return nil, fmt.Errorf("Missing '%s' property", path_id)
 	}
 
-	repo_rsp := gjson.GetBytes(body, "properties.wof:repo")
+	repo_rsp := gjson.GetBytes(body, path_repo)
 
 	if !repo_rsp.Exists() {
-		return nil, errors.New("Missing wof:repo")
+		return nil, fmt.Errorf("Missing '%s' property", path_repo)
 	}
 
 	wof_id := id_rsp.Int()