Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add upstream service handling #35

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Add upstream service handling #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7460844
add config array
cookel2 Dec 13, 2024
43ed530
work in progress - need to commit to move branches
cookel2 Dec 13, 2024
f481c72
add correct configs, create multiple upstream search clients, put res…
cookel2 Dec 23, 2024
ae07789
remove vulnerability, format code
cookel2 Dec 24, 2024
80a00fd
map resource items to search data import objects
cookel2 Jan 7, 2025
3d91145
tidy up and add unit test
cookel2 Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 40 additions & 36 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,26 @@ import (

// Config represents service configuration for dp-search-reindex-batch
type Config struct {
AwsRegion string `envconfig:"AWS_REGION"`
AwsSecSkipVerify bool `envconfig:"AWS_SEC_SKIP_VERIFY"`
DatasetAPIURL string `envconfig:"DATASET_API_URL"`
ElasticSearchURL string `envconfig:"ELASTIC_SEARCH_URL"`
MaxDocumentExtractions int `envconfig:"MAX_DOCUMENT_EXTRACTIONS"`
MaxDocumentTransforms int `envconfig:"MAX_DOCUMENT_TRANSFORMS"`
MaxDatasetExtractions int `envconfig:"MAX_DATASET_EXTRACTIONS"`
MaxDatasetTransforms int `envconfig:"MAX_DATASET_TRANSFORMS"`
PaginationLimit int `envconfig:"DATASET_PAGINATION_LIMIT"`
ServiceAuthToken string `envconfig:"SERVICE_AUTH_TOKEN" json:"-"`
SignESRequests bool `envconfig:"SIGN_ELASTICSEARCH_REQUESTS"`
TopicAPIURL string `envconfig:"TOPIC_API_URL"`
TopicTaggingEnabled bool `envconfig:"ENABLE_TOPIC_TAGGING"`
TrackerInterval time.Duration `envconfig:"TRACKER_INTERVAL"`
ZebedeeURL string `envconfig:"ZEBEDEE_URL"`
ZebedeeTimeout time.Duration `envconfig:"ZEBEDEE_TIMEOUT"`
EnableDatasetAPIReindex bool `envconfig:"ENABLE_DATASET_API_REINDEX"`
EnableZebedeeReindex bool `envconfig:"ENABLE_ZEBEDEE_REINDEX"`
AwsRegion string `envconfig:"AWS_REGION"`
AwsSecSkipVerify bool `envconfig:"AWS_SEC_SKIP_VERIFY"`
DatasetAPIURL string `envconfig:"DATASET_API_URL"`
ElasticSearchURL string `envconfig:"ELASTIC_SEARCH_URL"`
MaxDocumentExtractions int `envconfig:"MAX_DOCUMENT_EXTRACTIONS"`
MaxDocumentTransforms int `envconfig:"MAX_DOCUMENT_TRANSFORMS"`
MaxDatasetExtractions int `envconfig:"MAX_DATASET_EXTRACTIONS"`
MaxDatasetTransforms int `envconfig:"MAX_DATASET_TRANSFORMS"`
PaginationLimit int `envconfig:"DATASET_PAGINATION_LIMIT"`
ServiceAuthToken string `envconfig:"SERVICE_AUTH_TOKEN" json:"-"`
SignESRequests bool `envconfig:"SIGN_ELASTICSEARCH_REQUESTS"`
TopicAPIURL string `envconfig:"TOPIC_API_URL"`
TopicTaggingEnabled bool `envconfig:"ENABLE_TOPIC_TAGGING"`
TrackerInterval time.Duration `envconfig:"TRACKER_INTERVAL"`
ZebedeeURL string `envconfig:"ZEBEDEE_URL"`
ZebedeeTimeout time.Duration `envconfig:"ZEBEDEE_TIMEOUT"`
EnableDatasetAPIReindex bool `envconfig:"ENABLE_DATASET_API_REINDEX"`
EnableZebedeeReindex bool `envconfig:"ENABLE_ZEBEDEE_REINDEX"`
EnableOtherServicesReindex bool `envconfig:"ENABLE_OTHER_SERVICES_REINDEX"`
OtherUpstreamServices [][]string `envconfig:"OTHER_UPSTREAM_SERVICES"`
}

var cfg *Config
Expand All @@ -38,24 +40,26 @@ func Get() (*Config, error) {
}

cfg = &Config{
ZebedeeURL: "http://localhost:8082",
ZebedeeTimeout: 2 * time.Minute,
ElasticSearchURL: "http://localhost:11200",
SignESRequests: false,
AwsRegion: "eu-west-2",
AwsSecSkipVerify: false,
DatasetAPIURL: "http://localhost:22000",
ServiceAuthToken: "",
PaginationLimit: 500,
MaxDocumentExtractions: 100,
MaxDocumentTransforms: 20,
MaxDatasetExtractions: 20,
MaxDatasetTransforms: 10,
TrackerInterval: 5 * time.Second,
TopicAPIURL: "http://localhost:25300",
TopicTaggingEnabled: false,
EnableDatasetAPIReindex: true,
EnableZebedeeReindex: true,
ZebedeeURL: "http://localhost:8082",
ZebedeeTimeout: 2 * time.Minute,
ElasticSearchURL: "http://localhost:11200",
SignESRequests: false,
AwsRegion: "eu-west-2",
AwsSecSkipVerify: false,
DatasetAPIURL: "http://localhost:22000",
ServiceAuthToken: "",
PaginationLimit: 500,
MaxDocumentExtractions: 100,
MaxDocumentTransforms: 20,
MaxDatasetExtractions: 20,
MaxDatasetTransforms: 10,
TrackerInterval: 5 * time.Second,
TopicAPIURL: "http://localhost:25300",
TopicTaggingEnabled: false,
EnableDatasetAPIReindex: false,
EnableZebedeeReindex: false,
EnableOtherServicesReindex: false,
OtherUpstreamServices: [][]string{{"http://localhost:29600", "/resources"}},
}

return cfg, envconfig.Process("", cfg)
Expand Down
5 changes: 3 additions & 2 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ func TestConfig(t *testing.T) {
TopicAPIURL: "http://localhost:25300",
TopicTaggingEnabled: false,
ZebedeeTimeout: 2 * time.Minute,
EnableDatasetAPIReindex: true,
EnableZebedeeReindex: true,
EnableDatasetAPIReindex: false,
EnableZebedeeReindex: false,
OtherUpstreamServices: [][]string{{"http://localhost:29600", "/resources"}},
},
)
})
Expand Down
27 changes: 16 additions & 11 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
module github.com/ONSdigital/dp-search-reindex-batch

go 1.22
go 1.23

replace github.com/ONSdigital/dis-search-upstream-stub => /Users/cookel/EllasFiles/DpCodeRepos/dis-search-upstream-stub

require (
github.com/ONSdigital/dp-api-clients-go/v2 v2.260.1
github.com/ONSdigital/dp-api-clients-go/v2 v2.261.0
github.com/ONSdigital/dp-elasticsearch/v3 v3.0.1-alpha.4.0.20230308115225-bb7559a89d0c
github.com/ONSdigital/dp-net/v2 v2.11.2
github.com/ONSdigital/dp-net/v2 v2.12.0
github.com/ONSdigital/dp-search-api v1.45.0
github.com/ONSdigital/dp-search-data-extractor v1.3.1
github.com/ONSdigital/dp-search-data-importer v1.1.0
github.com/ONSdigital/dp-search-data-importer v1.2.0
github.com/ONSdigital/dp-topic-api v0.23.0
github.com/ONSdigital/log.go/v2 v2.4.3
github.com/elastic/go-elasticsearch/v7 v7.10.0
Expand All @@ -17,23 +19,26 @@ require (
github.com/smartystreets/goconvey v1.8.1
)

require github.com/ONSdigital/dis-search-upstream-stub v1.2.1

require (
github.com/ONSdigital/dp-healthcheck v1.6.3 // indirect
github.com/aws/aws-sdk-go v1.54.19 // indirect
github.com/fatih/color v1.17.0 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/justinas/alice v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/smarty/assertions v1.16.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.22.0 // indirect
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sys v0.28.0 // indirect
)
40 changes: 20 additions & 20 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
github.com/ONSdigital/dp-api-clients-go/v2 v2.260.1 h1:gVRu8rKdlV50HPtBISyGjMtBrZ+meaIebHpoKykMC18=
github.com/ONSdigital/dp-api-clients-go/v2 v2.260.1/go.mod h1:+4jW6BFCJwldSIwNVcclrTVrFVlx0D3e3C3CzUMvsuA=
github.com/ONSdigital/dp-api-clients-go/v2 v2.261.0 h1:fRiQosE+hGGh32n1lrT3pHKbnXvCnYADuNt6epCTKDM=
github.com/ONSdigital/dp-api-clients-go/v2 v2.261.0/go.mod h1:+4jW6BFCJwldSIwNVcclrTVrFVlx0D3e3C3CzUMvsuA=
github.com/ONSdigital/dp-elasticsearch/v3 v3.0.1-alpha.4.0.20230308115225-bb7559a89d0c h1:eSAGAcjij4KnrdtxZPYZphBYls5aJLHRwNhm8EtlK3Q=
github.com/ONSdigital/dp-elasticsearch/v3 v3.0.1-alpha.4.0.20230308115225-bb7559a89d0c/go.mod h1:w/aKn5mdX0o8sDaJPU3jas49Ed0fgrXYTeGU5DhMnHk=
github.com/ONSdigital/dp-healthcheck v1.6.3 h1:EekpiLjiXQtetNDmworUhZZMDvcG70b2cZYhVhEuUSs=
github.com/ONSdigital/dp-healthcheck v1.6.3/go.mod h1:qZXdjvZoSbMW/YLmzMZobnvbP5onaVwKhj2yDi6JXdY=
github.com/ONSdigital/dp-mocking v0.10.1 h1:yEEglJ458kUztHlnGxhMiKhIr13gMYWYOBKFbCbp89M=
github.com/ONSdigital/dp-mocking v0.10.1/go.mod h1:LVFMmSpUTgalQoWbFOXTNUXrA+W+H1Lzbv+yrhmtPEY=
github.com/ONSdigital/dp-net/v2 v2.11.2 h1:S5WfcXHva1V8ZkVLS7ZhsvQjf3PPCkCM++NdGifV/uM=
github.com/ONSdigital/dp-net/v2 v2.11.2/go.mod h1:yZ0lIzM4WfIr6Ujl1lpkCsPHay0n/VQfZJUZjlYB8MY=
github.com/ONSdigital/dp-net/v2 v2.12.0 h1:eV/kVVHXRZP0Ad9r42a9YcTce/zGezixWKFz8qmjrUs=
github.com/ONSdigital/dp-net/v2 v2.12.0/go.mod h1:QKnUJ0ih9Xii2MEPM25a01btDl3J+UTOOHbkSj93Sko=
github.com/ONSdigital/dp-search-api v1.45.0 h1:crFrAMt31OgHoDMdi5MKFiRW5hdr7IOJN16NUMMRWYY=
github.com/ONSdigital/dp-search-api v1.45.0/go.mod h1:Vt+DsN5/O2u8Z85uMd/mpcvTj6pVnwZZbhNjirUqUss=
github.com/ONSdigital/dp-search-data-extractor v1.3.1 h1:Nxu/6XBWgrkh36WzYKQ+jz0BZnEPUYdp2k7LrT0nrLQ=
github.com/ONSdigital/dp-search-data-extractor v1.3.1/go.mod h1:Z1F8IOHf5NWsq6te0/fdd495BOSijgQc/gq+4cbaJRk=
github.com/ONSdigital/dp-search-data-importer v1.1.0 h1:evHYVO8tvIavtrjzNbGGEG5aHX/kq62svn8G7OX00iU=
github.com/ONSdigital/dp-search-data-importer v1.1.0/go.mod h1:BhEk1cabd02ofuLEQRPjYTufoFho1w+HB40F3d7fLkg=
github.com/ONSdigital/dp-search-data-importer v1.2.0 h1:NATpa7YRiY8V+JxnQeVEA7ny8EaDMVvpeouabRAlNWw=
github.com/ONSdigital/dp-search-data-importer v1.2.0/go.mod h1:C8jUAecK4vPz5riXBC4qee9eMkEHgqph5YPg4KF71Es=
github.com/ONSdigital/dp-topic-api v0.23.0 h1:v/E1UacSYpMHpzmmWu5J5BPEJsItdaOcKcWGYQx7nn4=
github.com/ONSdigital/dp-topic-api v0.23.0/go.mod h1:ZYSXkk4ZeM2TK1y10PWpSzaAho2FNdBdvp1KgTFkIio=
github.com/ONSdigital/log.go/v2 v2.4.3 h1:zTW5ZV3+ytqypS7opcDkjBP+k45I+XoTuP/IPlm5oUg=
github.com/ONSdigital/log.go/v2 v2.4.3/go.mod h1:2TiXCcEsIlDBH9f+4D0NybZPecobd++dphJv2GqVDb0=
github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elastic/go-elasticsearch/v7 v7.10.0 h1:vYRwqgFM46ZUHFMRdvKr+y1WA4ehJO6WqAGV9Btbl2o=
github.com/elastic/go-elasticsearch/v7 v7.10.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
Expand Down Expand Up @@ -67,18 +67,18 @@ github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
Expand Down
54 changes: 54 additions & 0 deletions task/mapper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package task

import (
"context"

"github.com/ONSdigital/log.go/v2/log"

upstreamModels "github.com/ONSdigital/dis-search-upstream-stub/models"
extractorModels "github.com/ONSdigital/dp-search-data-extractor/models"
)

const (
ReleaseDataType = "release"
)

// MapResourceToSearchDataImport Performs default mapping of a Resource item to a SearchDataImport struct.
func MapResourceToSearchDataImport(resourceItem upstreamModels.Resource) extractorModels.SearchDataImport {
searchData := extractorModels.SearchDataImport{
UID: resourceItem.URI,
URI: resourceItem.URI,
DataType: resourceItem.ContentType,
CDID: resourceItem.CDID,
DatasetID: resourceItem.DatasetID,
MetaDescription: resourceItem.MetaDescription,
Summary: resourceItem.Summary,
ReleaseDate: resourceItem.ReleaseDate,
Title: resourceItem.Title,
Topics: resourceItem.Topics,
}
if resourceItem.Edition != "" {
searchData.Edition = resourceItem.Edition
}
if resourceItem.ContentType == ReleaseDataType {
logData := log.Data{
"resourceRCData": resourceItem,
}
log.Info(context.Background(), "resource release calender data", logData)
for _, data := range resourceItem.DateChanges {
searchData.DateChanges = append(searchData.DateChanges, extractorModels.ReleaseDateDetails{
ChangeNotice: data.ChangeNotice,
Date: data.PreviousDate,
})
}
searchData.Published = resourceItem.Published
searchData.Cancelled = resourceItem.Cancelled
searchData.Finalised = resourceItem.Finalised
searchData.ProvisionalDate = resourceItem.ProvisionalDate
searchData.Survey = resourceItem.Survey
searchData.Language = resourceItem.Language
searchData.CanonicalTopic = resourceItem.CanonicalTopic
}

return searchData
}
Loading