Merge pull request #1901 from cloudsufi/xmlReader-cloudbuildfix

XmlReader e2e fix on CDF
cdapio · Dec 12, 2024 · ff19ec1 · ff19ec1
2 parents 9bf6896 + 714dc54
commit ff19ec1
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 1 deletion.
diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java
@@ -130,7 +130,7 @@ public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedO
                                          PluginPropertyUtils.pluginProp(expectedOutputFilePath), bucketName);
     } else {
       String outputFileSinkBucketName = PluginPropertyUtils.pluginProp(fileSinkBucketPath);
-      ValidationHelper.listBucketObjects(outputFileSinkBucketName.substring(5),
+      ValidationHelper.listBucketObjectsForXml(outputFileSinkBucketName.substring(5),
                                          PluginPropertyUtils.pluginProp(expectedOutputFilePath));
     }
   }

diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java
@@ -185,4 +185,28 @@ private static void validateBothFilesDataForXml(List<String> outputFileData, Str
       counter
     );
   }
+
+  public static void listBucketObjectsForXml(String bucketName, String expectedOutputFilePath) {
+    List<String> bucketObjectNames = new ArrayList<>();
+    List<String> fileData = new ArrayList<>();
+    String sourceBucketName = TestSetupHooks.fileSourceBucket;
+    Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
+    Page<Blob> blobs = storage.list(bucketName);
+
+    // Adding all the Objects which have data in a list.
+    List<Blob> bucketObjects = StreamSupport.stream(blobs.iterateAll().spliterator(), true)
+      .filter(blob -> blob.getSize() != 0)
+      .collect(Collectors.toList());
+
+    Stream<String> objectNamesWithData = bucketObjects.stream().map(blob -> blob.getName());
+    objectNamesWithData.forEach(objectName -> bucketObjectNames.add(objectName));
+
+    // Fetching the data for all the part-r Files
+    for (String objectName : bucketObjectNames) {
+      if (objectName.contains("part-r")) {
+        Collections.addAll(fileData, fetchObjectData(projectId, bucketName, objectName));
+      }
+    }
+    validateBothFilesDataForXml(fileData, expectedOutputFilePath, sourceBucketName);
+  }
 }