Merge pull request #2762 from wellcomecollection/Features-vector-changes

Switch to using full 4096 dimensional feature vector #5823
wellcomecollection · Nov 25, 2024 · cb3dd19 · cb3dd19
2 parents 47ecb52 + 22d5c4f
commit cb3dd19
Show file tree

Hide file tree

Showing 77 changed files with 255,004 additions and 310,719 deletions.
diff --git a/common/internal_model/src/main/scala/weco/catalogue/internal_model/image/Image.scala b/common/internal_model/src/main/scala/weco/catalogue/internal_model/image/Image.scala
@@ -92,21 +92,15 @@ object ImageFsm {
 }
 
 case class InferredData(
-  // We split the feature vector so that it can fit into
-  // ES's dense vector type (max length 2048)
-  features1: List[Float],
-  features2: List[Float],
-  reducedFeatures: List[Float],
+  features: List[Float],
   paletteEmbedding: List[Float],
   averageColorHex: Option[String],
   aspectRatio: Option[Float]
 )
 
 object InferredData {
   def empty: InferredData = InferredData(
-    features1 = Nil,
-    features2 = Nil,
-    reducedFeatures = Nil,
+    features = Nil,
     paletteEmbedding = Nil,
     averageColorHex = None,
     aspectRatio = None

diff --git a/...del/src/test/scala/weco/catalogue/internal_model/fixtures/index/ImagesIndexFixtures.scala b/...del/src/test/scala/weco/catalogue/internal_model/fixtures/index/ImagesIndexFixtures.scala
@@ -12,7 +12,7 @@ trait ImagesIndexFixtures extends IndexFixturesBase {
   def withLocalImagesIndex[R]: Fixture[Index, R] = {
     withLocalElasticSearchIndex[R](config =
       getConfig(
-        mappings = "mappings.images_indexed.2024-11-06.json",
+        mappings = "mappings.images_indexed.2024-11-14.json",
         analysis = "analysis.works_indexed.2024-11-06.json"
       )
     )

diff --git a/...ernal_model/src/test/scala/weco/catalogue/internal_model/generators/ImageGenerators.scala b/...ernal_model/src/test/scala/weco/catalogue/internal_model/generators/ImageGenerators.scala
@@ -171,13 +171,9 @@ trait ImageGenerators
 
   def createInferredData: InferredData = {
     val features = randomUnitLengthVector(4096)
-    val (features1, features2) = features.splitAt(features.size / 2)
-    val reducedFeatures = randomUnitLengthVector(1024)
     val paletteEmbedding = randomUnitLengthVector(1000)
     InferredData(
-      features1 = features1.toList,
-      features2 = features2.toList,
-      reducedFeatures = reducedFeatures.toList,
+      features = features.toList,
       paletteEmbedding = paletteEmbedding.toList,
       averageColorHex = Some(randomHexString),
       aspectRatio = inferredDataAspectRatio