Merge pull request #9 from hkuich/002-fixes

- improved logging for mistakes in configuration files (better point…
typedb-osi · Jan 5, 2021 · 3adb359 · 3adb359
2 parents f98b2b4 + 71875cd
commit 3adb359
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 7 deletions.
diff --git a/build.gradle b/build.gradle
@@ -5,7 +5,7 @@ plugins {
 }
 
 group 'com.github.bayer-science-for-a-better-life'
-version '0.0.1'
+version '0.0.2'
 
 repositories {
     mavenCentral()

diff --git a/src/main/java/configuration/ProcessorConfigEntry.java b/src/main/java/configuration/ProcessorConfigEntry.java
@@ -1,9 +1,14 @@
 package configuration;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import java.util.HashMap;
 import java.util.Map;
 
 public class ProcessorConfigEntry {
 
+    private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami");
+
     private String processor;
     private String processorType;
     private String schemaType;
@@ -30,6 +35,9 @@ public HashMap<String, HashMap<String, ConceptGenerator>> getConceptGenerators()
     }
 
     public ConceptGenerator getAttributeGenerator(String key) {
+        if(getConceptGenerators().get("attributes") == null) {
+            throw new RuntimeException("You have specified the attribute [" + key + "] in your dataConfig file - but there are no attributeGenerators specified in the corresponding processor.");
+        }
         for (Map.Entry<String, ConceptGenerator> entry : getConceptGenerators().get("attributes").entrySet()) {
             if (entry.getKey().equals(key)) {
                 return entry.getValue();

diff --git a/src/main/java/generator/RelationInsertGenerator.java b/src/main/java/generator/RelationInsertGenerator.java
@@ -118,7 +118,7 @@ private StatementInstance playersInsert(ArrayList<Statement> matchStatements, in
             boolean insert = false;
             for (Statement st :matchStatements) {
                 //need to have player in match statement or cannot insert as player in relation
-                if (st.toString().contains(playerGenerator.getPlayerType())) {
+                if (st.toString().contains(playerGenerator.getUniquePlayerId())) {
                     insert = true;
                 }
             }
@@ -140,6 +140,9 @@ private Collection<? extends Statement> playersMatch(String[] tokens, String[] h
         for (DataConfigEntry.GeneratorSpecification playerDataConfigEntry : dce.getPlayers()) {
             ProcessorConfigEntry.ConceptGenerator playerGenerator = gce.getPlayerGenerator(playerDataConfigEntry.getGenerator());
             int playerDataIndex = idxOf(headerTokens, playerDataConfigEntry);
+            if(playerDataIndex == -1) {
+                appLogger.error("The column header in your dataconfig mapping to the uniquePlayerId [" + playerGenerator.getUniquePlayerId() + "] cannot be found in the file you specified.");
+            }
             if (tokens.length > playerDataIndex &&
                     !cleanToken(tokens[playerDataIndex]).isEmpty()) {
                 StatementInstance ms = Graql

diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java
@@ -70,6 +70,7 @@ public void migrate(boolean migrateEntities, boolean migrateRelations) throws IO
 
         session.close();
         client.close();
+        appLogger.info("GraMi is finished!");
     }
 
     private void migrateThingsInOrder(GraknClient.Session session, boolean migrateEntities, boolean migrateRelations) throws IOException {
@@ -91,11 +92,11 @@ private void getStatusAndMigrate(GraknClient.Session session, String conceptType
             String currentProcessor = dce.getProcessor();
             if(isOfConceptType(currentProcessor, conceptType)){
                 appLogger.info("migrating [" + dcEntryKey + "]...");
-                if (migrationStatus != null && migrationStatus.get(dcEntryKey) != null) {
+                if (migrationStatus != null && migrationStatus.get(dce.getDataPath()) != null) {
                     appLogger.info("previous migration status found for entity type: [" + dcEntryKey + "]");
-                    if (!migrationStatus.get(dcEntryKey).isCompleted()) {
-                        appLogger.info(dcEntryKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(dcEntryKey).getMigratedRows());
-                        getGeneratorAndInsert(session, dce, migrationStatus.get(dcEntryKey).getMigratedRows());
+                    if (!migrationStatus.get(dce.getDataPath()).isCompleted()) {
+                        appLogger.info(dcEntryKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(dce.getDataPath()).getMigratedRows());
+                        getGeneratorAndInsert(session, dce, migrationStatus.get(dce.getDataPath()).getMigratedRows());
                     } else {
                         appLogger.info(dcEntryKey + " is already completely migrated - moving on...");
                     }
@@ -166,8 +167,8 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, GraknCl
                 //insert the rest when loop exits with less than batch size
                 if (!rows.isEmpty()) {
                     writeThing(dce, gen, session, rows, batchSizeCounter, header);
+                    appLogger.info("final # rows processed: " + totalRecordCounter);
                 }
-                appLogger.info("final # rows processed: " + totalRecordCounter);
             } catch (IOException e) {
                 e.printStackTrace();
             }
@@ -250,6 +251,7 @@ private void updateMigrationStatusIsCompleted(DataConfigEntry dce) throws IOExce
         try {
             Gson gson = new Gson();
             Type MigrationStatusMapType = new TypeToken<HashMap<String, MigrationStatus>>(){}.getType();
+            System.out.println("data path: " + dce.getDataPath() + " and thing: " + dce.getProcessor());
             migrationStatus.get(dce.getDataPath()).setCompleted(true);
             FileWriter fw = new FileWriter(migrationStatePath);
             gson.toJson(migrationStatus, MigrationStatusMapType, fw);