Merge branch 'main' of github.com:whosonfirst/go-whosonfirst-findinga…

…id into main

application/catalog/application.go

@@ -12,7 +12,7 @@ import (
 )
 
 var cache_uri string
-var indexer_uri string
+var iterator_uri string
 var findingaid_uri string
 
 type CatalogApplication struct {
@@ -29,9 +29,9 @@ func (app *CatalogApplication) DefaultFlagSet(ctx context.Context) (*flag.FlagSe
 	fs := flagset.NewFlagSet("catalog")
 
 	fs.StringVar(&cache_uri, "cache-uri", "file:///tmp/", "A valid whosonfirst/go-cache URI string.")
-	fs.StringVar(&indexer_uri, "indexer-uri", "repo://", "A valid whosonfirst/go-whosonfirst-iterate/v2 URI string.")
+	fs.StringVar(&iterator_uri, "iterator-uri", "repo://", "A valid whosonfirst/go-whosonfirst-iterate/v2 URI string.")
 
-	fs.StringVar(&findingaid_uri, "findingaid-uri", "repo://?cache={cache_uri}&indexer={indexer_uri}", "A valid whosonfirst/go-whosonfirst-findingaid URI string.")
+	fs.StringVar(&findingaid_uri, "findingaid-uri", "repo://?cache={cache_uri}&iterator={iterator_uri}", "A valid whosonfirst/go-whosonfirst-findingaid URI string.")
 
 	return fs, nil
 }
@@ -69,12 +69,12 @@ func (app *CatalogApplication) RunWithFlagSet(ctx context.Context, fs *flag.Flag
 		fa_q["cache"] = []string{cache_uri}
 	}
 
-	if fa_q.Get("indexer") == "{indexer_uri}" {
-		fa_q["indexer"] = []string{indexer_uri}
+	if fa_q.Get("iterator") == "{iterator_uri}" {
+		fa_q["iterator"] = []string{iterator_uri}
 	}
 
-	if fa_q.Get("indexer") == "" {
-		return fmt.Errorf("Missing '-indexer-uri' flag.")
+	if fa_q.Get("iterator") == "" {
+		return fmt.Errorf("Missing '-iterator-uri' flag.")
 	}
 
 	fa_uri.RawQuery = fa_q.Encode()

go.mod

@@ -5,16 +5,16 @@ go 1.16
 require (
 	github.com/aaronland/go-http-server v0.0.7
 	github.com/aaronland/go-roster v0.0.2
-	github.com/aaronland/gocloud-blob-s3 v0.1.0
+	github.com/aaronland/gocloud-blob-s3 v0.1.3
 	github.com/jtacoma/uritemplates v1.0.0
 	github.com/rs/cors v1.7.0
 	github.com/sfomuseum/go-flags v0.8.2
-	github.com/tidwall/gjson v1.9.3
+	github.com/tidwall/gjson v1.10.2
+	github.com/tidwall/sjson v1.2.3
 	github.com/whosonfirst/go-cache v0.5.0
 	github.com/whosonfirst/go-cache-blob v0.2.0
 	github.com/whosonfirst/go-ioutil v1.0.0
-	github.com/whosonfirst/go-whosonfirst-iterate v1.2.0 // indirect
 	github.com/whosonfirst/go-whosonfirst-iterate/v2 v2.0.1
 	github.com/whosonfirst/go-whosonfirst-uri v1.0.1
-	gocloud.dev v0.23.0
+	gocloud.dev v0.24.0
 )

repo/indexer.go

@@ -6,6 +6,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 	"github.com/whosonfirst/go-cache"
 	"github.com/whosonfirst/go-ioutil"
 	"github.com/whosonfirst/go-whosonfirst-findingaid"
@@ -18,8 +20,9 @@ import (
 // Indexer is a struct that implements the findingaid.Indexer interface for information about Who's On First repositories.
 type Indexer struct {
 	findingaid.Indexer
-	cache        cache.Cache
-	iterator_uri string
+	cache         cache.Cache
+	iterator_uri  string
+	repo_property string
 }
 
 func init() {
@@ -71,9 +74,22 @@ func NewIndexer(ctx context.Context, uri string) (findingaid.Indexer, error) {
 		return nil, fmt.Errorf("Invalid ?iterator= parameter, %w", err)
 	}
 
+	// This is necessary for some repos like sfomuseum-data/sfomuseum-data-whosonfirst
+	// where the relevant repo name is stored in a properties.sfomuseum:repo key. This
+	// logic is handled below in IndexReader
+
+	repo_property := WOF_REPO_PROPERTY
+
+	custom_repo := q.Get("repo-property")
+
+	if custom_repo != "" {
+		repo_property = custom_repo
+	}
+
 	fa := &Indexer{
-		cache:        c,
-		iterator_uri: iterator_uri,
+		cache:         c,
+		iterator_uri:  iterator_uri,
+		repo_property: repo_property,
 	}
 
 	return fa, nil
@@ -112,7 +128,37 @@ func (fa *Indexer) IndexURIs(ctx context.Context, sources ...string) error {
 // IndexReader will index an individual Who's On First record in the finding aid.
 func (fa *Indexer) IndexReader(ctx context.Context, fh io.Reader) error {
 
-	rsp, err := FindingAidResponseFromReader(ctx, fh)
+	body, err := io.ReadAll(fh)
+
+	if err != nil {
+		return fmt.Errorf("Failed to read feature, %v", err)
+	}
+
+	// This is necessary for some repos like sfomuseum-data/sfomuseum-data-whosonfirst
+	// where the relevant repo name is stored in a properties.sfomuseum:repo key
+
+	if fa.repo_property != WOF_REPO_PROPERTY {
+
+		path_wof_repo := fmt.Sprintf("properties.%s", WOF_REPO_PROPERTY)
+		path_custom_repo := fmt.Sprintf("properties.%s", fa.repo_property)
+
+		custom_rsp := gjson.GetBytes(body, path_custom_repo)
+
+		// If custom repo property is present then use its value to update wof:repo
+
+		if custom_rsp.Exists() {
+
+			custom_repo := custom_rsp.String()
+
+			body, err = sjson.SetBytes(body, path_wof_repo, custom_repo)
+
+			if err != nil {
+				return fmt.Errorf("Failed to assign custom repo (%s) to %s, %v", custom_repo, WOF_REPO_PROPERTY, err)
+			}
+		}
+	}
+
+	rsp, err := FindingAidResponseFromBytes(ctx, body)
 
 	if err != nil {
 		return err

repo/repo.go

@@ -2,12 +2,16 @@ package repo
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"github.com/tidwall/gjson"
 	"github.com/whosonfirst/go-whosonfirst-uri"
 	"io"
+	_ "log"
 )
 
+const WOF_ID_PROPERTY string = "wof:id"
+const WOF_REPO_PROPERTY string = "wof:repo"
+
 // FindingAidResonse is a struct that contains Who's On First repository information for Who's On First records.
 type FindingAidResponse struct {
 	// The unique Who's On First ID.
@@ -26,18 +30,26 @@ func FindingAidResponseFromReader(ctx context.Context, fh io.Reader) (*FindingAi
 		return nil, err
 	}
 
+	return FindingAidResponseFromBytes(ctx, body)
+}
+
+func FindingAidResponseFromBytes(ctx context.Context, body []byte) (*FindingAidResponse, error) {
+
 	// TO DO: SUPPORT ALT FILES
 
-	id_rsp := gjson.GetBytes(body, "properties.wof:id")
+	path_id := fmt.Sprintf("properties.%s", WOF_ID_PROPERTY)
+	path_repo := fmt.Sprintf("properties.%s", WOF_REPO_PROPERTY)
+
+	id_rsp := gjson.GetBytes(body, path_id)
 
 	if !id_rsp.Exists() {
-		return nil, errors.New("Missing wof:id")
+		return nil, fmt.Errorf("Missing '%s' property", path_id)
 	}
 
-	repo_rsp := gjson.GetBytes(body, "properties.wof:repo")
+	repo_rsp := gjson.GetBytes(body, path_repo)
 
 	if !repo_rsp.Exists() {
-		return nil, errors.New("Missing wof:repo")
+		return nil, fmt.Errorf("Missing '%s' property", path_repo)
 	}
 
 	wof_id := id_rsp.Int()