From 7628d4308a77caf5e7b09052afc825447724a978 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 14:24:17 +0200 Subject: [PATCH 1/8] - removed insert counter - no longer needed for appropriate inserting of entities --- .../java/generator/EntityInsertGenerator.java | 13 ++- .../generator/EntityInsertGeneratorTest.java | 84 +++++++++---------- 2 files changed, 47 insertions(+), 50 deletions(-) diff --git a/src/main/java/generator/EntityInsertGenerator.java b/src/main/java/generator/EntityInsertGenerator.java index 55f9dc3..fa6dc3a 100644 --- a/src/main/java/generator/EntityInsertGenerator.java +++ b/src/main/java/generator/EntityInsertGenerator.java @@ -33,14 +33,12 @@ public EntityInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEnt public ArrayList> graknEntityInsert(ArrayList rows, String header) throws IllegalArgumentException { ArrayList> patterns = new ArrayList<>(); - int insertCounter = 0; for (String row : rows) { try { - ThingVariable temp = graknEntityQueryFromRow(row, header, insertCounter); + ThingVariable temp = graknEntityQueryFromRow(row, header); if (temp != null) { patterns.add(temp); } - insertCounter++; } catch (Exception e) { e.printStackTrace(); } @@ -49,15 +47,14 @@ public ArrayList> graknEntityInsert(ArrayList rows, } public ThingVariable graknEntityQueryFromRow(String row, - String header, - int insertCounter) throws Exception { + String header) throws Exception { String fileSeparator = dce.getSeparator(); String[] rowTokens = row.split(fileSeparator); String[] columnNames = header.split(fileSeparator); appLogger.debug("processing tokenized row: " + Arrays.toString(rowTokens)); malformedRow(row, rowTokens, columnNames.length); - Thing entityInsertStatement = addEntityToStatement(insertCounter); + Thing entityInsertStatement = addEntityToStatement(); for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) { entityInsertStatement = addAttribute(rowTokens, entityInsertStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); @@ -72,9 +69,9 @@ public ThingVariable graknEntityQueryFromRow(String row, } } - private Thing addEntityToStatement(int insertCounter) { + private Thing addEntityToStatement() { if (pce.getSchemaType() != null) { - return Graql.var("e-" + insertCounter).isa(pce.getSchemaType()); + return Graql.var("e").isa(pce.getSchemaType()); } else { throw new IllegalArgumentException("Required field not set in processor " + pce.getProcessor()); } diff --git a/src/test/java/generator/EntityInsertGeneratorTest.java b/src/test/java/generator/EntityInsertGeneratorTest.java index a264eb1..2ab5298 100644 --- a/src/test/java/generator/EntityInsertGeneratorTest.java +++ b/src/test/java/generator/EntityInsertGeneratorTest.java @@ -35,64 +35,64 @@ public void graknEntityQueryFromRowTest() { ArrayList > result = testEntityInsertGenerator.graknEntityInsert(rows, header); - String tc0 = "$e-0 isa entity1, has entity1-id \"entity1id0\", has entity1-name \"entity1name0\", has entity1-exp \"entity1id0exp0\""; + String tc0 = "$e isa entity1, has entity1-id \"entity1id0\", has entity1-name \"entity1name0\", has entity1-exp \"entity1id0exp0\""; Assert.assertEquals(tc0, result.get(0).toString()); - String tc1 = "$e-1 isa entity1, has entity1-id \"entity1id1\", has entity1-name \"entity1name1\", has entity1-exp \"entity1id1exp11\", has entity1-exp \"entity1id1exp12\""; + String tc1 = "$e isa entity1, has entity1-id \"entity1id1\", has entity1-name \"entity1name1\", has entity1-exp \"entity1id1exp11\", has entity1-exp \"entity1id1exp12\""; Assert.assertEquals(tc1, result.get(1).toString()); - String tc2 = "$e-2 isa entity1, has entity1-id \"entity1id2\", has entity1-name \"entity1name2\", has entity1-exp \"entity1id2exp21\", has entity1-exp \"entity1id2exp22\", has entity1-exp \"entity1id2exp23\""; + String tc2 = "$e isa entity1, has entity1-id \"entity1id2\", has entity1-name \"entity1name2\", has entity1-exp \"entity1id2exp21\", has entity1-exp \"entity1id2exp22\", has entity1-exp \"entity1id2exp23\""; Assert.assertEquals(tc2, result.get(2).toString()); - String tc3 = "$e-3 isa entity1, has entity1-id \"entity1id3\", has entity1-name \"entity1name3\""; + String tc3 = "$e isa entity1, has entity1-id \"entity1id3\", has entity1-name \"entity1name3\""; Assert.assertEquals(tc3, result.get(3).toString()); - String tc4 = "$e-4 isa entity1, has entity1-id \"entity1id4\", has entity1-name \"entity1name4\""; + String tc4 = "$e isa entity1, has entity1-id \"entity1id4\", has entity1-name \"entity1name4\""; Assert.assertEquals(tc4, result.get(4).toString()); - String tc5 = "$e-5 isa entity1, has entity1-id \"entity1id5\", has entity1-name \"entity1name5\""; + String tc5 = "$e isa entity1, has entity1-id \"entity1id5\", has entity1-name \"entity1name5\""; Assert.assertEquals(tc5, result.get(5).toString()); - String tc6 = "$e-6 isa entity1, has entity1-id \"entity1id6\", has entity1-name \"entity1name6\""; + String tc6 = "$e isa entity1, has entity1-id \"entity1id6\", has entity1-name \"entity1name6\""; Assert.assertEquals(tc6, result.get(6).toString()); - String tc7 = "$e-7 isa entity1, has entity1-id \"entity1id7\", has entity1-name \"entity1name7\""; + String tc7 = "$e isa entity1, has entity1-id \"entity1id7\", has entity1-name \"entity1name7\""; Assert.assertEquals(tc7, result.get(7).toString()); - String tc8 = "$e-8 isa entity1, has entity1-id \"entity1id8\", has entity1-name \"entity1name8\""; + String tc8 = "$e isa entity1, has entity1-id \"entity1id8\", has entity1-name \"entity1name8\""; Assert.assertEquals(tc8, result.get(8).toString()); - String tc9 = "$e-9 isa entity1, has entity1-id \"entity1id9\", has entity1-name \"entity1name9\""; + String tc9 = "$e isa entity1, has entity1-id \"entity1id9\", has entity1-name \"entity1name9\""; Assert.assertEquals(tc9, result.get(9).toString()); - String tc10 = "$e-10 isa entity1, has entity1-id \"entity1id10\", has entity1-name \"entity1name10\""; + String tc10 = "$e isa entity1, has entity1-id \"entity1id10\", has entity1-name \"entity1name10\""; Assert.assertEquals(tc10, result.get(10).toString()); - String tc11 = "$e-11 isa entity1, has entity1-id \"entity1id11\", has entity1-name \"entity1name11\""; + String tc11 = "$e isa entity1, has entity1-id \"entity1id11\", has entity1-name \"entity1name11\""; Assert.assertEquals(tc11, result.get(11).toString()); - String tc12 = "$e-12 isa entity1, has entity1-id \"entity1id12\", has entity1-name \"entity1name12\""; + String tc12 = "$e isa entity1, has entity1-id \"entity1id12\", has entity1-name \"entity1name12\""; Assert.assertEquals(tc12, result.get(12).toString()); - String tc13 = "$e-13 isa entity1, has entity1-id \"entity1id13\", has entity1-name \"entity1name13\""; + String tc13 = "$e isa entity1, has entity1-id \"entity1id13\", has entity1-name \"entity1name13\""; Assert.assertEquals(tc13, result.get(13).toString()); - String tc14 = "$e-14 isa entity1, has entity1-id \"entity1id14\", has entity1-name \"entity1name14\""; + String tc14 = "$e isa entity1, has entity1-id \"entity1id14\", has entity1-name \"entity1name14\""; Assert.assertEquals(tc14, result.get(14).toString()); - String tc15 = "$e-15 isa entity1, has entity1-id \"entity1id15\", has entity1-name \"entity1name15\""; + String tc15 = "$e isa entity1, has entity1-id \"entity1id15\", has entity1-name \"entity1name15\""; Assert.assertEquals(tc15, result.get(15).toString()); - String tc16 = "$e-16 isa entity1, has entity1-id \"entity1id16\", has entity1-name \"entity1name16\", has entity1-name \"entity1name16-2\""; + String tc16 = "$e isa entity1, has entity1-id \"entity1id16\", has entity1-name \"entity1name16\", has entity1-name \"entity1name16-2\""; Assert.assertEquals(tc16, result.get(16).toString()); - String tc17 = "$e-17 isa entity1, has entity1-id \"entity1id17\", has entity1-name \"entity1name17\""; + String tc17 = "$e isa entity1, has entity1-id \"entity1id17\", has entity1-name \"entity1name17\""; Assert.assertEquals(tc17, result.get(17).toString()); - String tc18 = "$e-18 isa entity1, has entity1-id \"entity1id18\", has entity1-name \"entity1name18\""; + String tc18 = "$e isa entity1, has entity1-id \"entity1id18\", has entity1-name \"entity1name18\""; Assert.assertEquals(tc18, result.get(18).toString()); - String tc19 = "$e-19 isa entity1, has entity1-id \"entity1id19\", has entity1-name \"entity1name19\""; + String tc19 = "$e isa entity1, has entity1-id \"entity1id19\", has entity1-name \"entity1name19\""; Assert.assertEquals(tc19, result.get(19).toString()); Assert.assertEquals(20, result.size()); @@ -110,37 +110,37 @@ public void graknEntityQueryFromRowWithBoolAndDoubleTest() { ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header); - String tc0 = "$e-0 isa entity2, has entity2-id \"entity2id0\", has entity2-bool true, has entity2-double 0.0"; + String tc0 = "$e isa entity2, has entity2-id \"entity2id0\", has entity2-bool true, has entity2-double 0.0"; Assert.assertEquals(tc0, result.get(0).toString()); - String tc1 = "$e-1 isa entity2, has entity2-id \"entity2id1\", has entity2-bool false, has entity2-double 1.1, has entity2-double 11.11"; + String tc1 = "$e isa entity2, has entity2-id \"entity2id1\", has entity2-bool false, has entity2-double 1.1, has entity2-double 11.11"; Assert.assertEquals(tc1, result.get(1).toString()); - String tc2 = "$e-2 isa entity2, has entity2-id \"entity2id2\", has entity2-bool true, has entity2-double 2.2"; + String tc2 = "$e isa entity2, has entity2-id \"entity2id2\", has entity2-bool true, has entity2-double 2.2"; Assert.assertEquals(tc2, result.get(2).toString()); - String tc3 = "$e-3 isa entity2, has entity2-id \"entity2id3\", has entity2-bool false, has entity2-double -3.3"; + String tc3 = "$e isa entity2, has entity2-id \"entity2id3\", has entity2-bool false, has entity2-double -3.3"; Assert.assertEquals(tc3, result.get(3).toString()); - String tc4 = "$e-4 isa entity2, has entity2-id \"entity2id4\", has entity2-double 4.0"; + String tc4 = "$e isa entity2, has entity2-id \"entity2id4\", has entity2-double 4.0"; Assert.assertEquals(tc4, result.get(4).toString()); - String tc5 = "$e-5 isa entity2, has entity2-id \"entity2id5\""; + String tc5 = "$e isa entity2, has entity2-id \"entity2id5\""; Assert.assertEquals(tc5, result.get(5).toString()); - String tc6 = "$e-6 isa entity2, has entity2-id \"entity2id6\""; + String tc6 = "$e isa entity2, has entity2-id \"entity2id6\""; Assert.assertEquals(tc6, result.get(6).toString()); - String tc7 = "$e-7 isa entity2, has entity2-id \"entity2id7\""; + String tc7 = "$e isa entity2, has entity2-id \"entity2id7\""; Assert.assertEquals(tc7, result.get(7).toString()); - String tc8 = "$e-8 isa entity2, has entity2-id \"entity2id8\""; + String tc8 = "$e isa entity2, has entity2-id \"entity2id8\""; Assert.assertEquals(tc8, result.get(8).toString()); - String tc9 = "$e-9 isa entity2, has entity2-id \"entity2id9\""; + String tc9 = "$e isa entity2, has entity2-id \"entity2id9\""; Assert.assertEquals(tc9, result.get(9).toString()); - String tc10 = "$e-10 isa entity2, has entity2-id \"entity2id10\""; + String tc10 = "$e isa entity2, has entity2-id \"entity2id10\""; Assert.assertEquals(tc10, result.get(10).toString()); Assert.assertEquals(11, result.size()); @@ -157,37 +157,37 @@ public void graknEntityQueryFromRowWithLongTest() { ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header); - String tc0 = "$e-0 isa entity3, has entity3-id \"entity3id0\", has entity3-int 0"; + String tc0 = "$e isa entity3, has entity3-id \"entity3id0\", has entity3-int 0"; Assert.assertEquals(tc0, result.get(0).toString()); - String tc1 = "$e-1 isa entity3, has entity3-id \"entity3id1\", has entity3-int 1, has entity3-int 11"; + String tc1 = "$e isa entity3, has entity3-id \"entity3id1\", has entity3-int 1, has entity3-int 11"; Assert.assertEquals(tc1, result.get(1).toString()); - String tc2 = "$e-2 isa entity3, has entity3-id \"entity3id2\", has entity3-int 2"; + String tc2 = "$e isa entity3, has entity3-id \"entity3id2\", has entity3-int 2"; Assert.assertEquals(tc2, result.get(2).toString()); - String tc3 = "$e-3 isa entity3, has entity3-id \"entity3id3\", has entity3-int -3"; + String tc3 = "$e isa entity3, has entity3-id \"entity3id3\", has entity3-int -3"; Assert.assertEquals(tc3, result.get(3).toString()); - String tc4 = "$e-4 isa entity3, has entity3-id \"entity3id4\""; + String tc4 = "$e isa entity3, has entity3-id \"entity3id4\""; Assert.assertEquals(tc4, result.get(4).toString()); - String tc5 = "$e-5 isa entity3, has entity3-id \"entity3id5\""; + String tc5 = "$e isa entity3, has entity3-id \"entity3id5\""; Assert.assertEquals(tc5, result.get(5).toString()); - String tc6 = "$e-6 isa entity3, has entity3-id \"entity3id6\""; + String tc6 = "$e isa entity3, has entity3-id \"entity3id6\""; Assert.assertEquals(tc6, result.get(6).toString()); - String tc7 = "$e-7 isa entity3, has entity3-id \"entity3id7\""; + String tc7 = "$e isa entity3, has entity3-id \"entity3id7\""; Assert.assertEquals(tc7, result.get(7).toString()); - String tc8 = "$e-8 isa entity3, has entity3-id \"entity3id8\""; + String tc8 = "$e isa entity3, has entity3-id \"entity3id8\""; Assert.assertEquals(tc8, result.get(8).toString()); - String tc9 = "$e-9 isa entity3, has entity3-id \"entity3id9\""; + String tc9 = "$e isa entity3, has entity3-id \"entity3id9\""; Assert.assertEquals(tc9, result.get(9).toString()); - String tc10 = "$e-10 isa entity3, has entity3-id \"entity3id10\""; + String tc10 = "$e isa entity3, has entity3-id \"entity3id10\""; Assert.assertEquals(tc10, result.get(10).toString()); Assert.assertEquals(11, result.size()); From b80a451d359bd374210730af90e220e0eec49855 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 15:55:44 +0200 Subject: [PATCH 2/8] - fixed logging --- src/main/java/generator/RelationInsertGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/generator/RelationInsertGenerator.java b/src/main/java/generator/RelationInsertGenerator.java index b795e52..f504763 100644 --- a/src/main/java/generator/RelationInsertGenerator.java +++ b/src/main/java/generator/RelationInsertGenerator.java @@ -86,7 +86,7 @@ public ArrayList>> graknRelationshipQueryFromRow(Stri assembledStatements.add(insertStatements); if (isValid(assembledStatements)) { -// System.out.println("valid query: <" + assembleQuery(assembledStatements) + ">"); + appLogger.debug("valid query: <" + assembleQuery(assembledStatements) + ">"); return assembledStatements; } else { dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper statement or is missing required players or attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); From 3dd815c923cf51976a3d64bfaa35cf59c7bb7268 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 15:56:10 +0200 Subject: [PATCH 3/8] - fixed CLI versioning --- src/main/java/cli/GramiCLI.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cli/GramiCLI.java b/src/main/java/cli/GramiCLI.java index 3f5895b..07854ef 100644 --- a/src/main/java/cli/GramiCLI.java +++ b/src/main/java/cli/GramiCLI.java @@ -8,7 +8,7 @@ import java.io.IOException; -@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0-alpha-9", mixinStandardHelpOptions = true) +@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0-alpha-12", mixinStandardHelpOptions = true) public class GramiCLI { public static void main(String[] args) { From bfec7909dae8b2d7f556fc18b4c1ba03e5bfea9f Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 15:57:23 +0200 Subject: [PATCH 4/8] - added functionality to insert floating attributes, or just attributes independent of ownership, including tests --- src/main/java/generator/GeneratorUtil.java | 96 ++++++++++++++++++- src/main/java/generator/InsertGenerator.java | 1 + src/main/java/migrator/GraknMigrator.java | 27 +++++- src/test/java/migrator/MigrationTest.java | 17 ++++ .../resources/phone-calls/dataConfig.json | 28 +++++- .../phone-calls/migrationStatus.json | 2 +- .../phone-calls/processorConfig.json | 27 ++++++ src/test/resources/phone-calls/schema.gql | 20 +++- 8 files changed, 212 insertions(+), 6 deletions(-) diff --git a/src/main/java/generator/GeneratorUtil.java b/src/main/java/generator/GeneratorUtil.java index 71cdf88..3a74932 100644 --- a/src/main/java/generator/GeneratorUtil.java +++ b/src/main/java/generator/GeneratorUtil.java @@ -2,10 +2,10 @@ import configuration.DataConfigEntry; import configuration.ProcessorConfigEntry; -import graql.lang.pattern.variable.ThingVariable; import graql.lang.pattern.variable.UnboundVariable; import graql.lang.pattern.variable.ThingVariable.Thing; import graql.lang.pattern.variable.ThingVariable.Relation; +import graql.lang.pattern.variable.ThingVariable.Attribute; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import preprocessor.RegexPreprocessor; @@ -20,6 +20,7 @@ public class GeneratorUtil { private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); + private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); public static String cleanToken(String token) { //TODO - expand cleaning of other strange characters at some point @@ -53,6 +54,33 @@ public static int[] indicesOf(String[] headerTokens, return indices; } + public static Attribute addValue(String[] tokens, + UnboundVariable statement, + String[] columnNames, + DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, + ProcessorConfigEntry pce, + DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { + String attributeGeneratorKey = generatorMappingForAttribute.getGenerator(); + ProcessorConfigEntry.ConceptGenerator attributeGenerator = pce.getAttributeGenerator(attributeGeneratorKey); + String columnName = generatorMappingForAttribute.getColumnName(); + int columnNameIndex = idxOf(columnNames, columnName); + Attribute att = null; + + if (columnNameIndex == -1) { + dataLogger.error("Column name: <" + columnName + "> was not found in file being processed"); + } else { + if ( columnNameIndex < tokens.length && + tokens[columnNameIndex] != null && + !cleanToken(tokens[columnNameIndex]).isEmpty()) { + String attributeType = attributeGenerator.getAttributeType(); + String attributeValueType = attributeGenerator.getValueType(); + String cleanedToken = cleanToken(tokens[columnNameIndex]); + att = addAttributeValueOfType(statement, attributeType, attributeValueType, cleanedToken, preprocessorConfig); + } + } + return att; + } + public static Thing addAttribute(String[] tokens, Thing statement, String[] columnNames, @@ -113,7 +141,6 @@ public static Thing addAttribute(String[] tokens, DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, ProcessorConfigEntry pce, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { - String attributeGeneratorKey = generatorMappingForAttribute.getGenerator(); ProcessorConfigEntry.ConceptGenerator attributeGenerator = pce.getAttributeGenerator(attributeGeneratorKey); String columnListSeparator = generatorMappingForAttribute.getListSeparator(); @@ -333,6 +360,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (preprocessorConfig != null) { cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); + appLogger.debug("processor processed cleaned value: " + cleanedValue); } Thing returnThing = null; @@ -390,6 +418,70 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, return returnThing; } + public static Attribute addAttributeValueOfType(UnboundVariable statement, + String conceptType, + String valueType, + String cleanedValue, + DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { + if (preprocessorConfig != null) { + cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); + } + Attribute att = null; + + switch (valueType) { + case "string": + att = statement.eq(cleanedValue); + break; + case "long": + try { + att = statement.eq(Integer.parseInt(cleanedValue)); + } catch (NumberFormatException numberFormatException) { + dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(numberFormatException.getMessage()); + } + break; + case "double": + try { + att = statement.eq(Double.parseDouble(cleanedValue)); + } catch (NumberFormatException numberFormatException) { + dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(numberFormatException.getMessage()); + } + break; + case "boolean": + if (cleanedValue.toLowerCase().equals("true")) { + att = statement.eq( true); + } else if (cleanedValue.toLowerCase().equals("false")) { + att = statement.eq( false); + } else { + dataLogger.warn("current row has column of type with non- value - skipping column"); + } + break; + case "datetime": + try { + DateTimeFormatter isoDateFormatter = DateTimeFormatter.ISO_DATE; + String[] dt = cleanedValue.split("T"); + LocalDate date = LocalDate.parse(dt[0], isoDateFormatter); + if (dt.length > 1) { + LocalTime time = LocalTime.parse(dt[1], DateTimeFormatter.ISO_TIME); + LocalDateTime dateTime = date.atTime(time); + att = statement.eq(dateTime); + } else { + LocalDateTime dateTime = date.atStartOfDay(); + att = statement.eq(dateTime); + } + } catch (DateTimeException dateTimeException) { + dataLogger.warn("current row has column of type with non- datetime value: "); + dataLogger.warn(dateTimeException.getMessage()); + } + break; + default: + dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); + break; + } + return att; + } + private static String applyPreprocessor(String cleanedValue, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig.PreprocessorParams params = preprocessorConfig.getParams(); diff --git a/src/main/java/generator/InsertGenerator.java b/src/main/java/generator/InsertGenerator.java index 2d51576..19a1ece 100644 --- a/src/main/java/generator/InsertGenerator.java +++ b/src/main/java/generator/InsertGenerator.java @@ -7,6 +7,7 @@ public abstract class InsertGenerator { public ArrayList> graknEntityInsert(ArrayList rows, String header) { return null; }; + public ArrayList> graknAttributeInsert(ArrayList rows, String header) { return null; }; public HashMap>>> graknRelationInsert(ArrayList rows, String header) throws Exception { return null; }; public HashMap>>> graknAppendAttributeInsert(ArrayList rows, String header) throws Exception { return null; }; } diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java index 7863fa4..d5dbfa3 100644 --- a/src/main/java/migrator/GraknMigrator.java +++ b/src/main/java/migrator/GraknMigrator.java @@ -72,6 +72,11 @@ public void migrate(boolean migrateEntities, boolean migrateRelations, boolean m } private void migrateThingsInOrder(Session session, boolean migrateEntities, boolean migrateRelations, boolean migrateRelationRelations, boolean migrateAppendAttributes) throws IOException { + + appLogger.info("migrating floating attributes..."); + getStatusAndMigrate(session, "attribute"); + appLogger.info("migration of floating attributes completed"); + if (migrateEntities) { appLogger.info("migrating entities..."); getStatusAndMigrate(session, "entity"); @@ -92,6 +97,12 @@ private void migrateThingsInOrder(Session session, boolean migrateEntities, bool getStatusAndMigrate(session, "append-attribute"); appLogger.info("migration of append-attribute completed"); } + boolean migrate_attribute_relations = true; + if (migrate_attribute_relations) { + appLogger.info("migrating attribute-attribute relations..."); + getStatusAndMigrate(session, "attribute-relation"); + appLogger.info("migration of attribute-attribute relations completed"); + } } private void getStatusAndMigrate(Session session, String processorType) throws IOException { @@ -101,7 +112,7 @@ private void getStatusAndMigrate(Session session, String processorType) throws I if(isOfProcessorType(currentProcessor, processorType)){ appLogger.info("migrating [" + dcEntryKey + "]..."); if (migrationStatus != null && migrationStatus.get(dce.getDataPath()) != null) { - appLogger.info("previous migration status found for schema type: [" + dcEntryKey + "]"); + appLogger.info("previous migration status found for: [" + dcEntryKey + "]"); if (!migrationStatus.get(dce.getDataPath()).isCompleted()) { appLogger.info(dcEntryKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(dce.getDataPath()).getMigratedRows()); getGeneratorAndInsert(session, dce, migrationStatus.get(dce.getDataPath()).getMigratedRows()); @@ -212,6 +223,14 @@ private void writeThing(DataConfigEntry dce, InsertGenerator gen, Session sessio HashMap>>> statements = gen.graknAppendAttributeInsert(rows, header); appLogger.trace("number of generated insert Statements: " + statements.get("match").size()); graknInserter.matchInsertThreadedInserting(statements, session, threads, dce.getBatchSize()); + } else if (isOfProcessorType(dce.getProcessor(), "attribute")) { + ArrayList> statements = gen.graknAttributeInsert(rows, header); + appLogger.trace("number of generated insert Statements: " + statements.size()); + graknInserter.insertThreadedInserting(statements, session, threads, dce.getBatchSize()); +// } else if (isOfProcessorType(dce.getProcessor(), "attribute-relation")) { +// HashMap>>> statements = gen.graknAttributeRelationInsert(rows, header); +// appLogger.trace("number of generated insert Statements: " + statements.get("match").size()); +// graknInserter.matchInsertThreadedInserting(statements, session, threads, dce.getBatchSize()); } else { throw new IllegalArgumentException("the processor <" + dce.getProcessor() + "> is not known"); } @@ -291,6 +310,12 @@ private InsertGenerator getProcessor(DataConfigEntry dce) { } else if (gce != null && gce.getProcessorType().equals("append-attribute")) { appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); return new AppendAttributeGenerator(dce, gce); + } else if (gce != null && gce.getProcessorType().equals("attribute")) { + appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); + return new AttributeInsertGenerator(dce, gce); +// } else if (gce != null && gce.getProcessorType().equals("attribute-relation")) { +// appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); +// return new AttributeRelationInsertGenerator(dce, gce); } else { throw new IllegalArgumentException(String.format("Invalid/No generator provided for: %s", dce.getProcessor())); } diff --git a/src/test/java/migrator/MigrationTest.java b/src/test/java/migrator/MigrationTest.java index bcb078e..b2d1304 100644 --- a/src/test/java/migrator/MigrationTest.java +++ b/src/test/java/migrator/MigrationTest.java @@ -58,6 +58,7 @@ public void migratePhoneCallsTest() throws IOException { testRelations(session); testRelationWithRelations(session); testAppendAttribute(session); + testAttributes(session); session.close(); client.close(); } @@ -245,6 +246,22 @@ public void testAppendAttribute(Session session) { read.close(); } + public void testAttributes(Session session) { + + Transaction read = session.transaction(Transaction.Type.READ); + GraqlMatch getQuery = Graql.match(var("a").isa("is-in-use")).get("a"); + Assert.assertEquals(2, read.query().match(getQuery).count()); + + read = session.transaction(Transaction.Type.READ); + getQuery = Graql.match(var("a").eq("in-use").isa("is-in-use")).get("a"); + Assert.assertEquals(1, read.query().match(getQuery).count()); + + read = session.transaction(Transaction.Type.READ); + getQuery = Graql.match(var("a").eq("not-in-use").isa("is-in-use")).get("a"); + Assert.assertEquals(1, read.query().match(getQuery).count()); + read.close(); + } + @Test public void issue10Test() throws IOException { diff --git a/src/test/resources/phone-calls/dataConfig.json b/src/test/resources/phone-calls/dataConfig.json index 77492cd..b66edc3 100644 --- a/src/test/resources/phone-calls/dataConfig.json +++ b/src/test/resources/phone-calls/dataConfig.json @@ -3,7 +3,7 @@ "dataPath": "src/test/resources/phone-calls/person.csv", "separator": ",", "processor": "person", - "batchSize": 2000, + "batchSize": 50, "threads": 4, "attributes": [ { @@ -200,5 +200,31 @@ ], "batchSize": 100, "threads": 4 + }, + "is-in-use": { + "dataPath": "src/test/resources/phone-calls/is-in-use.csv", + "separator": ",", + "processor": "is-in-use", + "batchSize": 50, + "threads": 4, + "attributes": [ + { + "columnName": "is_in_use", + "generator": "is-in-use" + } + ] + }, + "weather": { + "dataPath": "src/test/resources/phone-calls/weather.csv", + "separator": ",", + "processor": "weather", + "batchSize": 50, + "threads": 4, + "attributes": [ + { + "columnName": "weather_condition", + "generator": "perceived-as" + } + ] } } \ No newline at end of file diff --git a/src/test/resources/phone-calls/migrationStatus.json b/src/test/resources/phone-calls/migrationStatus.json index d5b0c6b..37bfb30 100644 --- a/src/test/resources/phone-calls/migrationStatus.json +++ b/src/test/resources/phone-calls/migrationStatus.json @@ -1 +1 @@ -{"src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12}} \ No newline at end of file +{"src/test/resources/phone-calls/is-in-use.csv":{"conceptName":"is-in-use","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"src/test/resources/phone-calls/weather.csv":{"conceptName":"weather","isCompleted":true,"migratedRows":3},"src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12}} \ No newline at end of file diff --git a/src/test/resources/phone-calls/processorConfig.json b/src/test/resources/phone-calls/processorConfig.json index 14ace25..1c4bc31 100644 --- a/src/test/resources/phone-calls/processorConfig.json +++ b/src/test/resources/phone-calls/processorConfig.json @@ -209,6 +209,33 @@ } } } + }, + { + "processor": "is-in-use", + "processorType": "attribute", + "schemaType": "is-in-use", + "conceptGenerators": { + "attributes": { + "is-in-use": { + "attributeType": "is-in-use", + "valueType": "string" + } + } + } + }, + { + "processor": "weather", + "processorType": "entity", + "schemaType": "weather", + "conceptGenerators": { + "attributes": { + "perceived-as": { + "attributeType": "perceived-as", + "valueType": "string", + "required": true + } + } + } } ] } \ No newline at end of file diff --git a/src/test/resources/phone-calls/schema.gql b/src/test/resources/phone-calls/schema.gql index 23d2f9d..11bdd76 100644 --- a/src/test/resources/phone-calls/schema.gql +++ b/src/test/resources/phone-calls/schema.gql @@ -19,7 +19,8 @@ define nick-name sub attribute, value string; twitter-username sub attribute, - value string; + value string, + plays in-use:account; fakebook-link sub attribute, value string; call-rating sub attribute, @@ -35,6 +36,7 @@ define owns started-at, owns duration, owns call-rating, + plays circumstance:call-of-interest, plays communication-channel:past-call; communication-channel sub relation, @@ -58,3 +60,19 @@ define owns twitter-username, owns fakebook-link, plays communication-channel:peer; + + weather sub entity, + owns perceived-as, + plays circumstance:weather-perception; + perceived-as sub attribute, value string; + + circumstance sub relation, + relates weather-perception, + relates call-of-interest; + + is-in-use sub attribute, value string, + plays in-use:status; + + in-use sub relation, + relates status, + relates account; From 75ae46de41ebc8a1a70c3984e94d3e7ca8db7f05 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 19:59:57 +0200 Subject: [PATCH 5/8] - intelliJ failed to add new files... --- .../generator/AttributeInsertGenerator.java | 91 +++++++++++++++++++ .../AttributeInsertGeneratorTest.java | 42 +++++++++ src/test/resources/phone-calls/is-in-use.csv | 3 + 3 files changed, 136 insertions(+) create mode 100644 src/main/java/generator/AttributeInsertGenerator.java create mode 100644 src/test/java/generator/AttributeInsertGeneratorTest.java create mode 100644 src/test/resources/phone-calls/is-in-use.csv diff --git a/src/main/java/generator/AttributeInsertGenerator.java b/src/main/java/generator/AttributeInsertGenerator.java new file mode 100644 index 0000000..3b8d649 --- /dev/null +++ b/src/main/java/generator/AttributeInsertGenerator.java @@ -0,0 +1,91 @@ +package generator; + +import configuration.DataConfigEntry; +import configuration.ProcessorConfigEntry; +import graql.lang.Graql; +import graql.lang.pattern.Pattern; +import graql.lang.pattern.variable.ThingVariable; +import graql.lang.pattern.variable.ThingVariable.Attribute; +import graql.lang.pattern.variable.UnboundVariable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Arrays; + +import static generator.GeneratorUtil.addValue; +import static generator.GeneratorUtil.malformedRow; + +public class AttributeInsertGenerator extends InsertGenerator { + + private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); + private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); + private final DataConfigEntry dce; + private final ProcessorConfigEntry pce; + + public AttributeInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) { + super(); + this.dce = dataConfigEntry; + this.pce = processorConfigEntry; + appLogger.debug("Creating AttributeInsertGenerator for processor " + processorConfigEntry.getProcessor() + " of type " + processorConfigEntry.getProcessorType()); + } + + public ArrayList> graknAttributeInsert(ArrayList rows, + String header) throws IllegalArgumentException { + ArrayList> patterns = new ArrayList<>(); + for (String row : rows) { + try { + ThingVariable temp = graknAttributeQueryFromRow(row, header); + if (temp != null) { + patterns.add(temp); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + return patterns; + } + + public ThingVariable graknAttributeQueryFromRow(String row, + String header) throws Exception { + String fileSeparator = dce.getSeparator(); + String[] rowTokens = row.split(fileSeparator); + String[] columnNames = header.split(fileSeparator); + appLogger.debug("processing tokenized row: " + Arrays.toString(rowTokens)); + malformedRow(row, rowTokens, columnNames.length); + + UnboundVariable attributeInitialStatement = addAttributeToStatement(); + Attribute attributeInsertStatement = null; + + for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) { + attributeInsertStatement = addValue(rowTokens, attributeInitialStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); + } + + if (attributeInsertStatement != null) { + attributeInsertStatement = attributeInsertStatement.isa(pce.getSchemaType()); + + if (isValid(attributeInsertStatement)) { + appLogger.debug("valid query: "); + return attributeInsertStatement; + } else { + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); + return null; + } + } else { + return null; + } + } + + private UnboundVariable addAttributeToStatement() { + if (pce.getSchemaType() != null) { + return Graql.var("a"); + } else { + throw new IllegalArgumentException("Required field not set in processor " + pce.getProcessor()); + } + } + + private boolean isValid(Pattern pa) { + String patternAsString = pa.toString(); + return patternAsString.contains("isa " + pce.getSchemaType()); + } +} diff --git a/src/test/java/generator/AttributeInsertGeneratorTest.java b/src/test/java/generator/AttributeInsertGeneratorTest.java new file mode 100644 index 0000000..a5444da --- /dev/null +++ b/src/test/java/generator/AttributeInsertGeneratorTest.java @@ -0,0 +1,42 @@ +package generator; + +import configuration.MigrationConfig; +import configuration.ProcessorConfigEntry; +import graql.lang.pattern.variable.ThingVariable; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; + +import static test.TestUtil.getData; +import static util.Util.getAbsPath; + +public class AttributeInsertGeneratorTest { + + private final String adcp = getAbsPath("src/test/resources/phone-calls/dataConfig.json"); + private final String gcp = getAbsPath("src/test/resources/phone-calls/processorConfig.json"); + private final String file = getAbsPath("src/test/resources/phone-calls/is-in-use.csv"); + private final MigrationConfig migrationConfig = new MigrationConfig("localhost:1729", "null", "null", adcp, gcp); + private final HashMap> genConf = migrationConfig.getProcessorConfig(); + + @Test + public void graknAttributeQueryFromRowTest() { + + AttributeInsertGenerator testAttributeInsertGenerator = new AttributeInsertGenerator(migrationConfig.getDataConfig().get("is-in-use"), genConf.get("processors").get(8)); + + ArrayList rows = getData(file); + String header = rows.get(0); + rows = new ArrayList<>(rows.subList(1, rows.size())); + + ArrayList> result = testAttributeInsertGenerator.graknAttributeInsert(rows, header); + + String tc0 = "$a \"in_use\" isa is-in-use"; + Assert.assertEquals(tc0, result.get(0).toString()); + + String tc1 = "$a \"not_in_use\" isa is-in-use"; + Assert.assertEquals(tc1, result.get(1).toString()); + + Assert.assertEquals(2, result.size()); + } +} diff --git a/src/test/resources/phone-calls/is-in-use.csv b/src/test/resources/phone-calls/is-in-use.csv new file mode 100644 index 0000000..47803f5 --- /dev/null +++ b/src/test/resources/phone-calls/is-in-use.csv @@ -0,0 +1,3 @@ +is_in_use +in-use +not-in-use From a292c11fbfc58fc0bef77939296a8d6dae56fa18 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 20:47:32 +0200 Subject: [PATCH 6/8] - can now add attributes into relations, having extended the RelationInsertGenerator --- src/main/java/generator/GeneratorUtil.java | 20 +++++----- .../generator/RelationInsertGenerator.java | 40 ++++++++++++------- src/main/java/migrator/GraknMigrator.java | 7 ---- .../RelationInsertGeneratorTest.java | 4 +- src/test/java/migrator/MigrationTest.java | 14 ++++++- .../resources/phone-calls/dataConfig.json | 18 +++++---- src/test/resources/phone-calls/in-use.csv | 9 +++++ src/test/resources/phone-calls/is-in-use.csv | 6 +-- .../phone-calls/migrationStatus.json | 2 +- .../phone-calls/processorConfig.json | 23 +++++++---- src/test/resources/phone-calls/schema.gql | 16 ++------ 11 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 src/test/resources/phone-calls/in-use.csv diff --git a/src/main/java/generator/GeneratorUtil.java b/src/main/java/generator/GeneratorUtil.java index 3a74932..99c5c10 100644 --- a/src/main/java/generator/GeneratorUtil.java +++ b/src/main/java/generator/GeneratorUtil.java @@ -72,10 +72,9 @@ public static Attribute addValue(String[] tokens, if ( columnNameIndex < tokens.length && tokens[columnNameIndex] != null && !cleanToken(tokens[columnNameIndex]).isEmpty()) { - String attributeType = attributeGenerator.getAttributeType(); String attributeValueType = attributeGenerator.getValueType(); String cleanedToken = cleanToken(tokens[columnNameIndex]); - att = addAttributeValueOfType(statement, attributeType, attributeValueType, cleanedToken, preprocessorConfig); + att = addAttributeValueOfType(statement, attributeValueType, cleanedToken, preprocessorConfig); } } return att; @@ -418,8 +417,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, return returnThing; } - public static Attribute addAttributeValueOfType(UnboundVariable statement, - String conceptType, + public static Attribute addAttributeValueOfType(UnboundVariable unboundVar, String valueType, String cleanedValue, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { @@ -430,11 +428,11 @@ public static Attribute addAttributeValueOfType(UnboundVariable statement, switch (valueType) { case "string": - att = statement.eq(cleanedValue); + att = unboundVar.eq(cleanedValue); break; case "long": try { - att = statement.eq(Integer.parseInt(cleanedValue)); + att = unboundVar.eq(Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { dataLogger.warn("current row has column of type with non- value - skipping column"); dataLogger.warn(numberFormatException.getMessage()); @@ -442,7 +440,7 @@ public static Attribute addAttributeValueOfType(UnboundVariable statement, break; case "double": try { - att = statement.eq(Double.parseDouble(cleanedValue)); + att = unboundVar.eq(Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { dataLogger.warn("current row has column of type with non- value - skipping column"); dataLogger.warn(numberFormatException.getMessage()); @@ -450,9 +448,9 @@ public static Attribute addAttributeValueOfType(UnboundVariable statement, break; case "boolean": if (cleanedValue.toLowerCase().equals("true")) { - att = statement.eq( true); + att = unboundVar.eq( true); } else if (cleanedValue.toLowerCase().equals("false")) { - att = statement.eq( false); + att = unboundVar.eq( false); } else { dataLogger.warn("current row has column of type with non- value - skipping column"); } @@ -465,10 +463,10 @@ public static Attribute addAttributeValueOfType(UnboundVariable statement, if (dt.length > 1) { LocalTime time = LocalTime.parse(dt[1], DateTimeFormatter.ISO_TIME); LocalDateTime dateTime = date.atTime(time); - att = statement.eq(dateTime); + att = unboundVar.eq(dateTime); } else { LocalDateTime dateTime = date.atStartOfDay(); - att = statement.eq(dateTime); + att = unboundVar.eq(dateTime); } } catch (DateTimeException dateTimeException) { dataLogger.warn("current row has column of type with non- datetime value: "); diff --git a/src/main/java/generator/RelationInsertGenerator.java b/src/main/java/generator/RelationInsertGenerator.java index f504763..76777fd 100644 --- a/src/main/java/generator/RelationInsertGenerator.java +++ b/src/main/java/generator/RelationInsertGenerator.java @@ -1,10 +1,6 @@ package generator; -import static generator.GeneratorUtil.idxOf; -import static generator.GeneratorUtil.indicesOf; -import static generator.GeneratorUtil.cleanToken; -import static generator.GeneratorUtil.addAttribute; -import static generator.GeneratorUtil.addAttributeOfColumnType; +import static generator.GeneratorUtil.*; import configuration.DataConfigEntry; import configuration.ProcessorConfigEntry; @@ -13,6 +9,7 @@ import graql.lang.pattern.variable.ThingVariable; import graql.lang.pattern.variable.ThingVariable.Thing; import graql.lang.pattern.variable.ThingVariable.Relation; +import graql.lang.pattern.variable.ThingVariable.Attribute; import graql.lang.pattern.variable.UnboundVariable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -115,7 +112,6 @@ private String assembleQuery(ArrayList>> queries) { private Collection> createPlayerMatchAndInsert(String[] rowTokens, String[] columnNames, int insertCounter) { ArrayList> players = new ArrayList<>(); -// UnboundVariable relVariable = Graql.var("rel-" + insertCounter); UnboundVariable relVariable = Graql.var("rel"); ArrayList> relationStrings = new ArrayList<>(); int playerCounter = 0; @@ -139,10 +135,13 @@ private Collection> createPlayerMatchAndInsert(String for (String exploded : currentCleanedToken.split(columnListSeparator)) { if (!cleanToken(exploded).isEmpty()) { String currentExplodedCleanedToken = cleanToken(exploded); -// String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter + "-" + insertCounter; String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); - players.add(createPlayerMatchStatement(currentExplodedCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + if (playerGenerator.getUniquePlayerId().contains("_attribute_player_")) { + players.add(createAttributePlayerMatchStatement(currentExplodedCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + } else { + players.add(createEntityPlayerMatchStatement(currentExplodedCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + } ArrayList rel = new ArrayList<>(); rel.add(playerRole); rel.add(playerVariable); @@ -151,10 +150,13 @@ private Collection> createPlayerMatchAndInsert(String } } } else { // single player, no columnListSeparator -// String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter + "-" + insertCounter; String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); - players.add(createPlayerMatchStatement(currentCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + if (playerGenerator.getUniquePlayerId().contains("_attribute_player_")) { + players.add(createAttributePlayerMatchStatement(currentCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + } else { + players.add(createEntityPlayerMatchStatement(currentCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + } ArrayList rel = new ArrayList<>(); rel.add(playerRole); rel.add(playerVariable); @@ -248,10 +250,10 @@ private Collection> createPlayerMatchAndInsert(String return players; } - private ThingVariable createPlayerMatchStatement(String cleanedToken, - ProcessorConfigEntry.ConceptGenerator playerGenerator, - String playerVariable, - DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { + private ThingVariable createEntityPlayerMatchStatement(String cleanedToken, + ProcessorConfigEntry.ConceptGenerator playerGenerator, + String playerVariable, + DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { Thing ms = Graql .var(playerVariable) .isa(playerGenerator.getPlayerType()); @@ -261,6 +263,16 @@ private ThingVariable createPlayerMatchStatement(String cleanedToken, return ms; } + private ThingVariable createAttributePlayerMatchStatement(String cleanedToken, + ProcessorConfigEntry.ConceptGenerator playerGenerator, + String playerVariable, + DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { + UnboundVariable uv = Graql.var(playerVariable); + Attribute ms = addAttributeValueOfType(uv, playerGenerator.getIdValueType(), cleanedToken, preprocessorConfig); + ms = ms.isa(playerGenerator.getPlayerType()); + return ms; + } + private ThingVariable createRelationPlayerMatchStatementByAttribute(String cleanedToken, ProcessorConfigEntry.ConceptGenerator playerGenerator, DataConfigEntry.DataConfigGeneratorMapping dcm, diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java index d5dbfa3..6b9762a 100644 --- a/src/main/java/migrator/GraknMigrator.java +++ b/src/main/java/migrator/GraknMigrator.java @@ -227,10 +227,6 @@ private void writeThing(DataConfigEntry dce, InsertGenerator gen, Session sessio ArrayList> statements = gen.graknAttributeInsert(rows, header); appLogger.trace("number of generated insert Statements: " + statements.size()); graknInserter.insertThreadedInserting(statements, session, threads, dce.getBatchSize()); -// } else if (isOfProcessorType(dce.getProcessor(), "attribute-relation")) { -// HashMap>>> statements = gen.graknAttributeRelationInsert(rows, header); -// appLogger.trace("number of generated insert Statements: " + statements.get("match").size()); -// graknInserter.matchInsertThreadedInserting(statements, session, threads, dce.getBatchSize()); } else { throw new IllegalArgumentException("the processor <" + dce.getProcessor() + "> is not known"); } @@ -313,9 +309,6 @@ private InsertGenerator getProcessor(DataConfigEntry dce) { } else if (gce != null && gce.getProcessorType().equals("attribute")) { appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); return new AttributeInsertGenerator(dce, gce); -// } else if (gce != null && gce.getProcessorType().equals("attribute-relation")) { -// appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); -// return new AttributeRelationInsertGenerator(dce, gce); } else { throw new IllegalArgumentException(String.format("Invalid/No generator provided for: %s", dce.getProcessor())); } diff --git a/src/test/java/generator/RelationInsertGeneratorTest.java b/src/test/java/generator/RelationInsertGeneratorTest.java index a3102d1..b5dbcfa 100644 --- a/src/test/java/generator/RelationInsertGeneratorTest.java +++ b/src/test/java/generator/RelationInsertGeneratorTest.java @@ -16,12 +16,10 @@ public class RelationInsertGeneratorTest { - private final String keyspaceName = "grakn_migrator_test"; - private final String asp = getAbsPath("src/test/resources/genericTests/schema-test.gql"); private final String adcp = getAbsPath("src/test/resources/genericTests/dataConfig-test.json"); private final String gcp = getAbsPath("src/test/resources/genericTests/processorConfig-test.json"); private final String rel1dp = getAbsPath("src/test/resources/genericTests/rel1-test-data.tsv"); - private final MigrationConfig migrationConfig = new MigrationConfig("localhost:48555",keyspaceName, asp, adcp, gcp); + private final MigrationConfig migrationConfig = new MigrationConfig(null,null, null, adcp, gcp); private final HashMap> gc = migrationConfig.getProcessorConfig(); @Test diff --git a/src/test/java/migrator/MigrationTest.java b/src/test/java/migrator/MigrationTest.java index b2d1304..134fb6c 100644 --- a/src/test/java/migrator/MigrationTest.java +++ b/src/test/java/migrator/MigrationTest.java @@ -59,6 +59,7 @@ public void migratePhoneCallsTest() throws IOException { testRelationWithRelations(session); testAppendAttribute(session); testAttributes(session); + testAttributeRelation(session); session.close(); client.close(); } @@ -253,15 +254,24 @@ public void testAttributes(Session session) { Assert.assertEquals(2, read.query().match(getQuery).count()); read = session.transaction(Transaction.Type.READ); - getQuery = Graql.match(var("a").eq("in-use").isa("is-in-use")).get("a"); + getQuery = Graql.match(var("a").eq("yes").isa("is-in-use")).get("a"); Assert.assertEquals(1, read.query().match(getQuery).count()); read = session.transaction(Transaction.Type.READ); - getQuery = Graql.match(var("a").eq("not-in-use").isa("is-in-use")).get("a"); + getQuery = Graql.match(var("a").eq("no").isa("is-in-use")).get("a"); Assert.assertEquals(1, read.query().match(getQuery).count()); read.close(); } + public void testAttributeRelation(Session session) { + + Transaction read = session.transaction(Transaction.Type.READ); + GraqlMatch getQuery = Graql.match(var("a").isa("in-use")).get("a"); + Assert.assertEquals(7, read.query().match(getQuery).count()); + + read.close(); + } + @Test public void issue10Test() throws IOException { diff --git a/src/test/resources/phone-calls/dataConfig.json b/src/test/resources/phone-calls/dataConfig.json index b66edc3..777ac10 100644 --- a/src/test/resources/phone-calls/dataConfig.json +++ b/src/test/resources/phone-calls/dataConfig.json @@ -209,21 +209,25 @@ "threads": 4, "attributes": [ { - "columnName": "is_in_use", + "columnName": "values", "generator": "is-in-use" } ] }, - "weather": { - "dataPath": "src/test/resources/phone-calls/weather.csv", + "in-use": { + "dataPath": "src/test/resources/phone-calls/in-use.csv", "separator": ",", - "processor": "weather", + "processor": "in-use", "batchSize": 50, "threads": 4, - "attributes": [ + "players": [ + { + "columnName": "pn", + "generator": "account" + }, { - "columnName": "weather_condition", - "generator": "perceived-as" + "columnName": "in_use", + "generator": "status" } ] } diff --git a/src/test/resources/phone-calls/in-use.csv b/src/test/resources/phone-calls/in-use.csv new file mode 100644 index 0000000..c5e54c5 --- /dev/null +++ b/src/test/resources/phone-calls/in-use.csv @@ -0,0 +1,9 @@ +pn,in_use ++7 171 898 0853,yes ++370 351 224 5176,no ++81 308 988 7153,yes ++54 398 559 0423,yes ++263 498 495 0617,yes ++81 746 154 2598,no ++63 815 962 6097,yes ++62 107 530 7500, diff --git a/src/test/resources/phone-calls/is-in-use.csv b/src/test/resources/phone-calls/is-in-use.csv index 47803f5..6bf7004 100644 --- a/src/test/resources/phone-calls/is-in-use.csv +++ b/src/test/resources/phone-calls/is-in-use.csv @@ -1,3 +1,3 @@ -is_in_use -in-use -not-in-use +values +yes +no diff --git a/src/test/resources/phone-calls/migrationStatus.json b/src/test/resources/phone-calls/migrationStatus.json index 37bfb30..d649d61 100644 --- a/src/test/resources/phone-calls/migrationStatus.json +++ b/src/test/resources/phone-calls/migrationStatus.json @@ -1 +1 @@ -{"src/test/resources/phone-calls/is-in-use.csv":{"conceptName":"is-in-use","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"src/test/resources/phone-calls/weather.csv":{"conceptName":"weather","isCompleted":true,"migratedRows":3},"src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12}} \ No newline at end of file +{"src/test/resources/phone-calls/is-in-use.csv":{"conceptName":"is-in-use","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"src/test/resources/phone-calls/in-use.csv":{"conceptName":"in-use","isCompleted":true,"migratedRows":8},"src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12}} \ No newline at end of file diff --git a/src/test/resources/phone-calls/processorConfig.json b/src/test/resources/phone-calls/processorConfig.json index 1c4bc31..997656a 100644 --- a/src/test/resources/phone-calls/processorConfig.json +++ b/src/test/resources/phone-calls/processorConfig.json @@ -224,14 +224,23 @@ } }, { - "processor": "weather", - "processorType": "entity", - "schemaType": "weather", + "processor": "in-use", + "processorType": "relation", + "schemaType": "in-use", "conceptGenerators": { - "attributes": { - "perceived-as": { - "attributeType": "perceived-as", - "valueType": "string", + "players": { + "account": { + "playerType": "phone-number", + "uniquePlayerId": "_attribute_player_", + "idValueType": "string", + "roleType": "account", + "required": true + }, + "status": { + "playerType": "is-in-use", + "uniquePlayerId": "_attribute_player_", + "idValueType": "string", + "roleType": "status", "required": true } } diff --git a/src/test/resources/phone-calls/schema.gql b/src/test/resources/phone-calls/schema.gql index 11bdd76..863dad6 100644 --- a/src/test/resources/phone-calls/schema.gql +++ b/src/test/resources/phone-calls/schema.gql @@ -11,7 +11,8 @@ define last-name sub attribute, value string; phone-number sub attribute, - value string; + value string, + plays in-use:account; city sub attribute, value string; age sub attribute, @@ -19,8 +20,7 @@ define nick-name sub attribute, value string; twitter-username sub attribute, - value string, - plays in-use:account; + value string; fakebook-link sub attribute, value string; call-rating sub attribute, @@ -36,7 +36,6 @@ define owns started-at, owns duration, owns call-rating, - plays circumstance:call-of-interest, plays communication-channel:past-call; communication-channel sub relation, @@ -61,15 +60,6 @@ define owns fakebook-link, plays communication-channel:peer; - weather sub entity, - owns perceived-as, - plays circumstance:weather-perception; - perceived-as sub attribute, value string; - - circumstance sub relation, - relates weather-perception, - relates call-of-interest; - is-in-use sub attribute, value string, plays in-use:status; From f262cc16017568c080f82b7670b03755581f0dd7 Mon Sep 17 00:00:00 2001 From: hkuich Date: Tue, 30 Mar 2021 21:32:55 +0200 Subject: [PATCH 7/8] - cleaned migrator --- src/main/java/migrator/GraknMigrator.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java index 6b9762a..d714ce1 100644 --- a/src/main/java/migrator/GraknMigrator.java +++ b/src/main/java/migrator/GraknMigrator.java @@ -97,12 +97,6 @@ private void migrateThingsInOrder(Session session, boolean migrateEntities, bool getStatusAndMigrate(session, "append-attribute"); appLogger.info("migration of append-attribute completed"); } - boolean migrate_attribute_relations = true; - if (migrate_attribute_relations) { - appLogger.info("migrating attribute-attribute relations..."); - getStatusAndMigrate(session, "attribute-relation"); - appLogger.info("migration of attribute-attribute relations completed"); - } } private void getStatusAndMigrate(Session session, String processorType) throws IOException { From 8478839a05fbd3ebc24b213dccdfee9573d044d8 Mon Sep 17 00:00:00 2001 From: hkuich Date: Thu, 8 Apr 2021 08:24:13 +0200 Subject: [PATCH 8/8] - updated to 2.0.1 and client 2.0.0 --- build.gradle | 4 +- src/main/java/generator/GeneratorUtil.java | 24 +++---- src/main/java/insert/GraknInserter.java | 31 ++++----- src/main/java/migrator/GraknMigrator.java | 65 ++++++++++--------- src/main/java/migrator/SchemaUpdater.java | 2 +- .../AttributeInsertGeneratorTest.java | 4 +- src/test/java/insert/GraknInserterTest.java | 24 +++---- src/test/java/migrator/MigrationTest.java | 60 ++++++++--------- src/test/java/migrator/SchemaUpdaterTest.java | 16 ++--- .../genericTests/migrationStatus-test.json | 2 +- src/test/resources/phone-calls/call.csv | 22 +++---- .../phone-calls/migrationStatus.json | 2 +- 12 files changed, 129 insertions(+), 127 deletions(-) diff --git a/build.gradle b/build.gradle index e37b550..9db4902 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group 'com.github.bayer-science-for-a-better-life' -version '0.1.0-alpha-12' +version '0.1.0' repositories { mavenCentral() @@ -15,7 +15,7 @@ repositories { } dependencies { - compile group: 'io.grakn.client', name: 'grakn-client', version: '2.0.0-alpha-12' + compile group: 'io.grakn.client', name: 'grakn-client', version: '2.0.0' testCompile group: 'junit', name: 'junit', version: '4.12' compile 'com.google.code.gson:gson:2.8.6' compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' diff --git a/src/main/java/generator/GeneratorUtil.java b/src/main/java/generator/GeneratorUtil.java index 99c5c10..ad1cf80 100644 --- a/src/main/java/generator/GeneratorUtil.java +++ b/src/main/java/generator/GeneratorUtil.java @@ -243,7 +243,7 @@ public static Thing addAttributeOfColumnType(Thing statement, try { statement = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -251,7 +251,7 @@ public static Thing addAttributeOfColumnType(Thing statement, try { statement = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -261,7 +261,7 @@ public static Thing addAttributeOfColumnType(Thing statement, } else if (cleanedValue.toLowerCase().equals("false")) { statement = statement.has(conceptType, false); } else { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); } break; case "datetime": @@ -278,7 +278,7 @@ public static Thing addAttributeOfColumnType(Thing statement, statement = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn("current row has column of type with non- datetime value: "); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); dataLogger.warn(dateTimeException.getMessage()); } break; @@ -306,7 +306,7 @@ public static Relation addAttributeOfColumnType(Relation statement, try { statement = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -314,7 +314,7 @@ public static Relation addAttributeOfColumnType(Relation statement, try { statement = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -324,7 +324,7 @@ public static Relation addAttributeOfColumnType(Relation statement, } else if (cleanedValue.toLowerCase().equals("false")) { statement = statement.has(conceptType, false); } else { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); } break; case "datetime": @@ -341,7 +341,7 @@ public static Relation addAttributeOfColumnType(Relation statement, statement = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn("current row has column of type with non- datetime value: "); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); dataLogger.warn(dateTimeException.getMessage()); } break; @@ -371,7 +371,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, try { returnThing = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -379,7 +379,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, try { returnThing = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); dataLogger.warn(numberFormatException.getMessage()); } break; @@ -389,7 +389,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, } else if (cleanedValue.toLowerCase().equals("false")) { returnThing = statement.has(conceptType, false); } else { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); } break; case "datetime": @@ -406,7 +406,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, returnThing = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn("current row has column of type with non- datetime value: "); + dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); dataLogger.warn(dateTimeException.getMessage()); } break; diff --git a/src/main/java/insert/GraknInserter.java b/src/main/java/insert/GraknInserter.java index 8ef491f..eaa5d31 100644 --- a/src/main/java/insert/GraknInserter.java +++ b/src/main/java/insert/GraknInserter.java @@ -1,8 +1,9 @@ package insert; -import grakn.client.GraknClient; -import grakn.client.GraknClient.Transaction; -import grakn.client.GraknClient.Session; +import grakn.client.Grakn; +import grakn.client.api.GraknClient; +import grakn.client.api.GraknSession; +import grakn.client.api.GraknTransaction; import graql.lang.Graql; import graql.lang.pattern.variable.ThingVariable; import graql.lang.query.GraqlDefine; @@ -18,10 +19,10 @@ public class GraknInserter { + private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); private final String schemaPath; private final String databaseName; private final String graknURI; - private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); public GraknInserter(String graknURI, String port, String schemaPath, String databaseName) { this.schemaPath = schemaPath; @@ -42,11 +43,11 @@ private void createDatabase(GraknClient client) { } private void defineToGrakn(String schemaAsString, GraknClient client) { - Session schemaSession = getSchemaSession(client); + GraknSession schemaSession = getSchemaSession(client); GraqlDefine q = Graql.parseQuery(schemaAsString); - Transaction writeTransaction = schemaSession.transaction(Transaction.Type.WRITE); + GraknTransaction writeTransaction = schemaSession.transaction(GraknTransaction.Type.WRITE); writeTransaction.query().define(q); writeTransaction.commit(); writeTransaction.close(); @@ -55,7 +56,7 @@ private void defineToGrakn(String schemaAsString, GraknClient client) { appLogger.info("Defined schema to database <" + databaseName + ">"); } - public void matchInsertThreadedInserting(HashMap>>> statements, Session session, int threads, int batchSize) throws InterruptedException { + public void matchInsertThreadedInserting(HashMap>>> statements, GraknSession session, int threads, int batchSize) throws InterruptedException { AtomicInteger queryIndex = new AtomicInteger(0); Thread[] ts = new Thread[threads]; @@ -66,7 +67,7 @@ public void matchInsertThreadedInserting(HashMap>> insertStatements = statements.get("insert"); while (queryIndex.get() < matchStatements.size()) { - try (Transaction tx = session.transaction(Transaction.Type.WRITE)) { + try (GraknTransaction tx = session.transaction(GraknTransaction.Type.WRITE)) { int q; for (int i = 0; i < batchSize && (q = queryIndex.getAndIncrement()) < matchStatements.size(); i++) { ArrayList> rowMatchStatements = matchStatements.get(q); @@ -90,7 +91,7 @@ public void matchInsertThreadedInserting(HashMap> statements, Session session, int threads, int batchSize) throws InterruptedException { + public void insertThreadedInserting(ArrayList> statements, GraknSession session, int threads, int batchSize) throws InterruptedException { AtomicInteger queryIndex = new AtomicInteger(0); Thread[] ts = new Thread[threads]; @@ -98,7 +99,7 @@ public void insertThreadedInserting(ArrayList> statements, Sess Runnable insertThread = () -> { while (queryIndex.get() < statements.size()) { - try (Transaction tx = session.transaction(Transaction.Type.WRITE)) { + try (GraknTransaction tx = session.transaction(GraknTransaction.Type.WRITE)) { int q; for (int i = 0; i < batchSize && (q = queryIndex.getAndIncrement()) < statements.size(); i++) { GraqlInsert query = Graql.insert(statements.get(q)); @@ -120,16 +121,16 @@ public void insertThreadedInserting(ArrayList> statements, Sess } // Utility functions - public Session getDataSession(GraknClient client) { - return client.session(databaseName, Session.Type.DATA); + public GraknSession getDataSession(GraknClient client) { + return client.session(databaseName, GraknSession.Type.DATA); } - public Session getSchemaSession(GraknClient client) { - return client.session(databaseName, Session.Type.SCHEMA); + public GraknSession getSchemaSession(GraknClient client) { + return client.session(databaseName, GraknSession.Type.SCHEMA); } public GraknClient getClient() { - return GraknClient.core(graknURI); + return Grakn.coreClient(graknURI); } private void deleteDatabaseIfExists(GraknClient client) { diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java index d714ce1..03015da 100644 --- a/src/main/java/migrator/GraknMigrator.java +++ b/src/main/java/migrator/GraknMigrator.java @@ -4,8 +4,8 @@ import com.google.gson.reflect.TypeToken; import configuration.*; import generator.*; -import grakn.client.GraknClient; -import grakn.client.GraknClient.Session; +import grakn.client.api.GraknClient; +import grakn.client.api.GraknSession; import graql.lang.pattern.variable.ThingVariable; import loader.DataLoader; import insert.GraknInserter; @@ -63,7 +63,7 @@ public void migrate(boolean migrateEntities, boolean migrateRelations, boolean m appLogger.info("using existing DB and schema to continue previous migration..."); } - GraknClient.Session dataSession = graknInserter.getDataSession(client); + GraknSession dataSession = graknInserter.getDataSession(client); migrateThingsInOrder(dataSession, migrateEntities, migrateRelations, migrateRelationRelations, migrateAppendAttributes); dataSession.close(); @@ -71,11 +71,11 @@ public void migrate(boolean migrateEntities, boolean migrateRelations, boolean m appLogger.info("GraMi is finished migrating your stuff!"); } - private void migrateThingsInOrder(Session session, boolean migrateEntities, boolean migrateRelations, boolean migrateRelationRelations, boolean migrateAppendAttributes) throws IOException { + private void migrateThingsInOrder(GraknSession session, boolean migrateEntities, boolean migrateRelations, boolean migrateRelationRelations, boolean migrateAppendAttributes) throws IOException { - appLogger.info("migrating floating attributes..."); + appLogger.info("migrating independent attributes..."); getStatusAndMigrate(session, "attribute"); - appLogger.info("migration of floating attributes completed"); + appLogger.info("migration of independent attributes completed"); if (migrateEntities) { appLogger.info("migrating entities..."); @@ -96,26 +96,27 @@ private void migrateThingsInOrder(Session session, boolean migrateEntities, bool appLogger.info("migrating append-attribute..."); getStatusAndMigrate(session, "append-attribute"); appLogger.info("migration of append-attribute completed"); - } + } //TODO: relations with attributes have to come at end } - private void getStatusAndMigrate(Session session, String processorType) throws IOException { + private void getStatusAndMigrate(GraknSession session, String processorType) throws IOException { for (String dcEntryKey : dataConfig.keySet()) { DataConfigEntry dce = dataConfig.get(dcEntryKey); String currentProcessor = dce.getProcessor(); if(isOfProcessorType(currentProcessor, processorType)){ appLogger.info("migrating [" + dcEntryKey + "]..."); - if (migrationStatus != null && migrationStatus.get(dce.getDataPath()) != null) { - appLogger.info("previous migration status found for: [" + dcEntryKey + "]"); - if (!migrationStatus.get(dce.getDataPath()).isCompleted()) { - appLogger.info(dcEntryKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(dce.getDataPath()).getMigratedRows()); - getGeneratorAndInsert(session, dce, migrationStatus.get(dce.getDataPath()).getMigratedRows()); + String migrationStatusKey = dcEntryKey + "-" + dce.getDataPath(); + if (migrationStatus != null && migrationStatus.get(migrationStatusKey) != null) { + appLogger.info("previous migration status found for: [" + migrationStatusKey + "]"); + if (!migrationStatus.get(migrationStatusKey).isCompleted()) { + appLogger.info(migrationStatusKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(migrationStatusKey).getMigratedRows()); + getGeneratorAndInsert(session, dce, migrationStatus.get(migrationStatusKey).getMigratedRows(), migrationStatusKey); } else { - appLogger.info(dcEntryKey + " is already completely migrated - moving on..."); + appLogger.info(migrationStatusKey + " is already completely migrated - moving on..."); } } else { - appLogger.info("nothing previously migrated for [" + dcEntryKey + "] - starting with row 0"); - getGeneratorAndInsert(session, dce, 0); + appLogger.info("nothing previously migrated for [" + migrationStatusKey + "] - starting with row 0"); + getGeneratorAndInsert(session, dce, 0, migrationStatusKey); } } } @@ -130,15 +131,15 @@ private boolean isOfProcessorType(String key, String conceptType) { return false; } - private void getGeneratorAndInsert(Session session, DataConfigEntry dce, int skipRows) throws IOException { + private void getGeneratorAndInsert(GraknSession session, DataConfigEntry dce, int skipRows, String migrationStatusKey) throws IOException { // choose insert generator InsertGenerator gen = getProcessor(dce); - writeThingToGrakn(dce, gen, session, skipRows); - updateMigrationStatusIsCompleted(dce); + writeThingToGrakn(dce, gen, session, skipRows, migrationStatusKey); + updateMigrationStatusIsCompleted(migrationStatusKey); } - private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, Session session, int skipLines) { + private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, GraknSession session, int skipLines, String migrationStatusKey) { appLogger.info("inserting using " + dce.getThreads() + " threads" + " with thread commit size of " + dce.getBatchSize() + " rows"); @@ -171,7 +172,7 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, Session if (batchSizeCounter == dce.getBatchSize() * dce.getThreads()) { System.out.print("+"); System.out.flush(); - writeThing(dce, gen, session, rows, batchSizeCounter, header); + writeThing(dce, gen, session, rows, batchSizeCounter, header, migrationStatusKey); batchSizeCounter = 0; rows.clear(); } @@ -183,7 +184,7 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, Session } //insert the rest when loop exits with less than batch size if (!rows.isEmpty()) { - writeThing(dce, gen, session, rows, batchSizeCounter, header); + writeThing(dce, gen, session, rows, batchSizeCounter, header, migrationStatusKey); if (totalRecordCounter % 50000 != 0) { System.out.println(); } @@ -201,7 +202,7 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, Session - private void writeThing(DataConfigEntry dce, InsertGenerator gen, Session session, ArrayList rows, int lineCounter, String header) throws IOException { + private void writeThing(DataConfigEntry dce, InsertGenerator gen, GraknSession session, ArrayList rows, int lineCounter, String header, String migrationStatusKey) throws IOException { int threads = dce.getThreads(); try { if (isOfProcessorType(dce.getProcessor(), "entity")) { @@ -224,7 +225,7 @@ private void writeThing(DataConfigEntry dce, InsertGenerator gen, Session sessio } else { throw new IllegalArgumentException("the processor <" + dce.getProcessor() + "> is not known"); } - updateMigrationStatusMigratedRows(dce, lineCounter); + updateMigrationStatusMigratedRows(dce, lineCounter, migrationStatusKey); } catch (Exception ee) { ee.printStackTrace(); } @@ -245,22 +246,22 @@ private void initializeMigrationStatus() { } } - private void updateMigrationStatusMigratedRows(DataConfigEntry dce, int lineCounter) throws IOException { + private void updateMigrationStatusMigratedRows(DataConfigEntry dce, int lineCounter, String migrationStatusKey) throws IOException { try { ProcessorConfigEntry pce = getGenFromGenConfig(dce.getProcessor(), migrationConfig.getProcessorConfig()); Gson gson = new Gson(); Type MigrationStatusMapType = new TypeToken>(){}.getType(); if (migrationStatus != null) { - if (migrationStatus.get(dce.getDataPath()) != null) { //updating an existing entry - int updatedMigratedRows = migrationStatus.get(dce.getDataPath()).getMigratedRows() + lineCounter; - migrationStatus.get(dce.getDataPath()).setMigratedRows(updatedMigratedRows); + if (migrationStatus.get(migrationStatusKey) != null) { //updating an existing entry + int updatedMigratedRows = migrationStatus.get(migrationStatusKey).getMigratedRows() + lineCounter; + migrationStatus.get(migrationStatusKey).setMigratedRows(updatedMigratedRows); } else { // writing new entry - migrationStatus.put(dce.getDataPath(), new MigrationStatus(pce.getSchemaType(), false, lineCounter)); + migrationStatus.put(migrationStatusKey, new MigrationStatus(pce.getSchemaType(), false, lineCounter)); } } else { //writing very first entry (i.e. file was empty) migrationStatus = new HashMap<>(); - migrationStatus.put(dce.getDataPath(), new MigrationStatus(pce.getSchemaType(), false, lineCounter)); + migrationStatus.put(migrationStatusKey, new MigrationStatus(pce.getSchemaType(), false, lineCounter)); } // update file @@ -272,11 +273,11 @@ private void updateMigrationStatusMigratedRows(DataConfigEntry dce, int lineCoun } } - private void updateMigrationStatusIsCompleted(DataConfigEntry dce) throws IOException { + private void updateMigrationStatusIsCompleted(String migrationStatusKey) throws IOException { try { Gson gson = new Gson(); Type MigrationStatusMapType = new TypeToken>(){}.getType(); - migrationStatus.get(dce.getDataPath()).setCompleted(true); + migrationStatus.get(migrationStatusKey).setCompleted(true); FileWriter fw = new FileWriter(migrationStatePath); gson.toJson(migrationStatus, MigrationStatusMapType, fw); fw.flush(); diff --git a/src/main/java/migrator/SchemaUpdater.java b/src/main/java/migrator/SchemaUpdater.java index 31c8cc3..3630edd 100644 --- a/src/main/java/migrator/SchemaUpdater.java +++ b/src/main/java/migrator/SchemaUpdater.java @@ -1,7 +1,7 @@ package migrator; import configuration.SchemaUpdateConfig; -import grakn.client.GraknClient; +import grakn.client.api.GraknClient; import insert.GraknInserter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/test/java/generator/AttributeInsertGeneratorTest.java b/src/test/java/generator/AttributeInsertGeneratorTest.java index a5444da..4dfc5c7 100644 --- a/src/test/java/generator/AttributeInsertGeneratorTest.java +++ b/src/test/java/generator/AttributeInsertGeneratorTest.java @@ -31,10 +31,10 @@ public void graknAttributeQueryFromRowTest() { ArrayList> result = testAttributeInsertGenerator.graknAttributeInsert(rows, header); - String tc0 = "$a \"in_use\" isa is-in-use"; + String tc0 = "$a \"yes\" isa is-in-use"; Assert.assertEquals(tc0, result.get(0).toString()); - String tc1 = "$a \"not_in_use\" isa is-in-use"; + String tc1 = "$a \"no\" isa is-in-use"; Assert.assertEquals(tc1, result.get(1).toString()); Assert.assertEquals(2, result.size()); diff --git a/src/test/java/insert/GraknInserterTest.java b/src/test/java/insert/GraknInserterTest.java index 2575f3c..2dd9f8b 100644 --- a/src/test/java/insert/GraknInserterTest.java +++ b/src/test/java/insert/GraknInserterTest.java @@ -1,8 +1,8 @@ package insert; -import grakn.client.GraknClient; -import grakn.client.GraknClient.Session; -import grakn.client.GraknClient.Transaction; +import grakn.client.api.GraknClient; +import grakn.client.api.GraknSession; +import grakn.client.api.GraknTransaction; import graql.lang.Graql; import graql.lang.query.GraqlDelete; import graql.lang.query.GraqlInsert; @@ -32,13 +32,13 @@ public void reloadKeyspaceTest() { Assert.assertTrue(client.databases().contains(databaseName)); //ensure Keyspace contains schema - Session dataSession = gi.getDataSession(client); - Transaction read = dataSession.transaction(Transaction.Type.READ); + GraknSession dataSession = gi.getDataSession(client); + GraknTransaction read = dataSession.transaction(GraknTransaction.Type.READ); GraqlMatch.Limited mq = Graql.match(var("e").sub("entity")).get("e").limit(100); Assert.assertEquals(4, read.query().match(mq).count()); read.close(); - read = dataSession.transaction(Transaction.Type.READ); + read = dataSession.transaction(GraknTransaction.Type.READ); mq = Graql.match(var("e").type("entity1")).get("e").limit(100); Assert.assertEquals(1, read.query().match(mq).count()); read.close(); @@ -62,26 +62,26 @@ public void insertToGraknTest() { gi.cleanAndDefineSchemaToDatabase(client); //perform data entry - Session dataSession = gi.getDataSession(client); - Transaction write = dataSession.transaction(Transaction.Type.WRITE); + GraknSession dataSession = gi.getDataSession(client); + GraknTransaction write = dataSession.transaction(GraknTransaction.Type.WRITE); GraqlInsert insertQuery = Graql.insert(var("e").isa("entity1").has("entity1-id", "ide1")); write.query().insert(insertQuery); write.commit(); write.close(); //ensure graph contains our insert - Transaction read = dataSession.transaction(Transaction.Type.READ); + GraknTransaction read = dataSession.transaction(GraknTransaction.Type.READ); GraqlMatch.Limited getQuery = Graql.match(var("e").isa("entity1").has("entity1-id", "ide1")).get("e").limit(100); Assert.assertEquals(1, read.query().match(getQuery).count()); read.close(); - read = dataSession.transaction(Transaction.Type.READ); + read = dataSession.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("e").isa("entity1").has("entity1-id", "ide2")).limit(100); Assert.assertEquals(0, read.query().match(getQuery).count()); read.close(); //another test for our insert - read = dataSession.transaction(Transaction.Type.READ); + read = dataSession.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("e").isa("entity1").has("entity1-id", "ide1")).get("e").limit(100); read.query().match(getQuery).forEach( answers -> { answers.concepts().stream().forEach( entry -> { @@ -91,7 +91,7 @@ public void insertToGraknTest() { read.close(); //clean up: - Transaction delete = dataSession.transaction(Transaction.Type.WRITE); + GraknTransaction delete = dataSession.transaction(GraknTransaction.Type.WRITE); GraqlDelete delQuery = Graql.match( var("e").isa("entity1").has("entity1-id", "ide1") ).delete(var("e").isa("entity1")); diff --git a/src/test/java/migrator/MigrationTest.java b/src/test/java/migrator/MigrationTest.java index 134fb6c..5048961 100644 --- a/src/test/java/migrator/MigrationTest.java +++ b/src/test/java/migrator/MigrationTest.java @@ -1,9 +1,9 @@ package migrator; import configuration.MigrationConfig; -import grakn.client.GraknClient; -import grakn.client.GraknClient.Session; -import grakn.client.GraknClient.Transaction; +import grakn.client.api.GraknClient; +import grakn.client.api.GraknSession; +import grakn.client.api.GraknTransaction; import graql.lang.Graql; import graql.lang.pattern.variable.ThingVariable; import graql.lang.pattern.variable.ThingVariable.Thing; @@ -53,7 +53,7 @@ public void migratePhoneCallsTest() throws IOException { GraknInserter gi = new GraknInserter(graknURI.split(":")[0], graknURI.split(":")[1], asp, databaseName); GraknClient client = gi.getClient(); - Session session = gi.getDataSession(client); + GraknSession session = gi.getDataSession(client); testEntities(session); testRelations(session); testRelationWithRelations(session); @@ -64,45 +64,45 @@ public void migratePhoneCallsTest() throws IOException { client.close(); } - public void testEntities(Session session) { + public void testEntities(GraknSession session) { // query person by phone-number - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch getQuery = Graql.match(var("p").isa("person").has("phone-number", "+261 860 539 4754")).get("p").limit(1000); Assert.assertEquals(1, read.query().match(getQuery).count()); // query person by last name - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("p").isa("person").has("last-name", "Smith")).get("p").limit(1000); Assert.assertEquals(2, read.query().match(getQuery).count()); // query all entities of type person - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("c").isa("person")).get("c").limit(1000); Assert.assertEquals(32, read.query().match(getQuery).count()); // query all entites of type company - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("e").isa("company")).get("e").limit(1000); Assert.assertEquals(2, read.query().match(getQuery).count()); read.close(); } - public void testRelations(Session session) { + public void testRelations(GraknSession session) { // query call by duration - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch getQuery = Graql.match(var("c").isa("call").has("duration", 2851)).get("c").limit(1000); Assert.assertEquals(1, read.query().match(getQuery).count()); // query call by date - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("c").isa("call").has("started-at", getDT("2018-09-17T18:43:42"))).get("c").limit(1000); Assert.assertEquals(1, read.query().match(getQuery).count()); // query call by caller - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); Thing player = Graql.var("p").isa("person").has("phone-number", "+7 171 898 0853"); Relation relation = Graql.var("c").isa("call").toUnbound().rel("caller", "p"); ArrayList statements = new ArrayList<>(); @@ -113,7 +113,7 @@ public void testRelations(Session session) { Assert.assertEquals(14, read.query().match(getQuery).count()); // query call by callee - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); player = Graql.var("p").isa("person").has("phone-number", "+7 171 898 0853"); relation = Graql.var("c").isa("call").toUnbound().rel("callee", "p"); statements = new ArrayList<>(); @@ -123,7 +123,7 @@ public void testRelations(Session session) { Assert.assertEquals(4, read.query().match(getQuery).count()); // query call by caller & callee - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); Thing playerOne = Graql.var("p1").isa("person").has("phone-number", "+7 171 898 0853"); Thing playerTwo = Graql.var("p2").isa("person").has("phone-number", "+57 629 420 5680"); relation = Graql.var("c").isa("call").toUnbound().rel("caller", "p1").rel("callee", "p2"); @@ -137,10 +137,10 @@ public void testRelations(Session session) { read.close(); } - public void testRelationWithRelations(Session session) { + public void testRelationWithRelations(GraknSession session) { // query specific communication-channel and count the number of past calls (single past-call): - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); Thing playerOne = Graql.var("p1").isa("person").has("phone-number", "+54 398 559 0423"); Thing playerTwo = Graql.var("p2").isa("person").has("phone-number", "+48 195 624 2025"); Relation relation = Graql.var("c").rel("peer", "p1").rel("peer", "p2").rel("past-call","x").isa("communication-channel"); @@ -154,7 +154,7 @@ public void testRelationWithRelations(Session session) { Assert.assertEquals(1, read.query().match(getQuery).count()); // query specific communication-channel and count the number of past calls (listSeparated past-calls: - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); playerOne = Graql.var("p1").isa("person").has("phone-number", "+263 498 495 0617"); playerTwo = Graql.var("p2").isa("person").has("phone-number", "+33 614 339 0298"); relation = Graql.var("c").rel("peer", "p1").rel("peer", "p2").rel("past-call", "x").isa("communication-channel"); @@ -168,7 +168,7 @@ public void testRelationWithRelations(Session session) { Assert.assertEquals(6, read.query().match(getQuery).count()); // make sure that this doesn't get inserted: - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); playerOne = Graql.var("p1").isa("person").has("phone-number", "+7 690 597 4443"); playerTwo = Graql.var("p2").isa("person").has("phone-number", "+54 398 559 9999"); relation = Graql.var("c").rel("peer", "p1").rel("peer", "p2").rel("past-call", "x").isa("communication-channel"); @@ -182,7 +182,7 @@ public void testRelationWithRelations(Session session) { Assert.assertEquals(0, read.query().match(getQuery).count()); // these are added by doing player matching for past calls: - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); playerOne = Graql.var("p1").isa("person").has("phone-number", "+81 308 988 7153"); playerTwo = Graql.var("p2").isa("person").has("phone-number", "+351 515 605 7915"); relation = Graql.var("c").rel("peer", "p1").rel("peer", "p2").rel("past-call", "x").isa("communication-channel"); @@ -195,7 +195,7 @@ public void testRelationWithRelations(Session session) { getQuery = Graql.match(statements).get("x").limit(1000); Assert.assertEquals(5, read.query().match(getQuery).count()); - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); playerOne = Graql.var("p1").isa("person").has("phone-number", "+7 171 898 0853"); playerTwo = Graql.var("p2").isa("person").has("phone-number", "+57 629 420 5680"); relation = Graql.var("c").rel("peer", "p1").rel("peer", "p2").rel("past-call", "x").isa("communication-channel"); @@ -209,7 +209,7 @@ public void testRelationWithRelations(Session session) { Assert.assertEquals(4, read.query().match(getQuery).count()); // these must not be found (come from player-matched past-call): - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); playerOne = Graql.var("p1").isa("person").has("phone-number", "+261 860 539 4754"); relation = Graql.var("c").rel("peer", "p1").rel("past-call", "x").isa("communication-channel"); statements = new ArrayList<>(); @@ -223,10 +223,10 @@ public void testRelationWithRelations(Session session) { read.close(); } - public void testAppendAttribute(Session session) { + public void testAppendAttribute(GraknSession session) { // Count number of total inserts - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch.Limited getQuery = Graql.match(var("p").isa("person").has("twitter-username", var("x"))).get("p").limit(1000); Assert.assertEquals(6, read.query().match(getQuery).count()); @@ -247,25 +247,25 @@ public void testAppendAttribute(Session session) { read.close(); } - public void testAttributes(Session session) { + public void testAttributes(GraknSession session) { - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch getQuery = Graql.match(var("a").isa("is-in-use")).get("a"); Assert.assertEquals(2, read.query().match(getQuery).count()); - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("a").eq("yes").isa("is-in-use")).get("a"); Assert.assertEquals(1, read.query().match(getQuery).count()); - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery = Graql.match(var("a").eq("no").isa("is-in-use")).get("a"); Assert.assertEquals(1, read.query().match(getQuery).count()); read.close(); } - public void testAttributeRelation(Session session) { + public void testAttributeRelation(GraknSession session) { - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch getQuery = Graql.match(var("a").isa("in-use")).get("a"); Assert.assertEquals(7, read.query().match(getQuery).count()); diff --git a/src/test/java/migrator/SchemaUpdaterTest.java b/src/test/java/migrator/SchemaUpdaterTest.java index cb0aaeb..23f3a57 100644 --- a/src/test/java/migrator/SchemaUpdaterTest.java +++ b/src/test/java/migrator/SchemaUpdaterTest.java @@ -2,9 +2,9 @@ import configuration.MigrationConfig; import configuration.SchemaUpdateConfig; -import grakn.client.GraknClient; -import grakn.client.GraknClient.Session; -import grakn.client.GraknClient.Transaction; +import grakn.client.api.GraknClient; +import grakn.client.api.GraknSession; +import grakn.client.api.GraknTransaction; import graql.lang.Graql; import graql.lang.query.GraqlMatch; import insert.GraknInserter; @@ -46,28 +46,28 @@ public void updateSchemaTest() throws IOException { private void postUpdateSchemaTests(GraknInserter gi) { GraknClient client = gi.getClient(); - Session session = gi.getDataSession(client); + GraknSession session = gi.getDataSession(client); // query attribute type - Transaction read = session.transaction(Transaction.Type.READ); + GraknTransaction read = session.transaction(GraknTransaction.Type.READ); GraqlMatch.Filtered getQuery = Graql.match(var("a").type("added-attribute")).get("a"); Assert.assertEquals(1, read.query().match(getQuery).count()); read.close(); // query entity type - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); GraqlMatch.Limited getQuery2 = Graql.match(var("a").type("added-entity")).get("a").limit(1000); Assert.assertEquals(1, read.query().match(getQuery2).count()); read.close(); // query relation type - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery2 = Graql.match(var("a").type("added-relation")).get("a").limit(1000); Assert.assertEquals(1, read.query().match(getQuery2).count()); read.close(); // query role type - read = session.transaction(Transaction.Type.READ); + read = session.transaction(GraknTransaction.Type.READ); getQuery2 = Graql.match(type("added-relation").relates(var("r"))).limit(1000); Assert.assertEquals(1, read.query().match(getQuery2).count()); read.close(); diff --git a/src/test/resources/genericTests/migrationStatus-test.json b/src/test/resources/genericTests/migrationStatus-test.json index 421f67b..21f8d94 100644 --- a/src/test/resources/genericTests/migrationStatus-test.json +++ b/src/test/resources/genericTests/migrationStatus-test.json @@ -1 +1 @@ -{"src/test/resources/genericTests/entity1-test-data.tsv":{"conceptName":"entity1","isCompleted":true,"migratedRows":32},"src/test/resources/genericTests/entity2-test-data.tsv":{"conceptName":"entity2","isCompleted":true,"migratedRows":12},"src/test/resources/genericTests/entity3-test-data.tsv":{"conceptName":"entity3","isCompleted":true,"migratedRows":12},"src/test/resources/genericTests/rel1-test-data.tsv":{"conceptName":"rel1","isCompleted":true,"migratedRows":44}} \ No newline at end of file +{"entity3-src/test/resources/genericTests/entity3-test-data.tsv":{"conceptName":"entity3","isCompleted":true,"migratedRows":12},"entity2-src/test/resources/genericTests/entity2-test-data.tsv":{"conceptName":"entity2","isCompleted":true,"migratedRows":12},"rel1-src/test/resources/genericTests/rel1-test-data.tsv":{"conceptName":"rel1","isCompleted":true,"migratedRows":44},"entity1-src/test/resources/genericTests/entity1-test-data.tsv":{"conceptName":"entity1","isCompleted":true,"migratedRows":32}} \ No newline at end of file diff --git a/src/test/resources/phone-calls/call.csv b/src/test/resources/phone-calls/call.csv index d287842..fe86398 100644 --- a/src/test/resources/phone-calls/call.csv +++ b/src/test/resources/phone-calls/call.csv @@ -1,21 +1,21 @@ caller_id,callee_id,started_at,duration +54 398 559 0423,+48 195 624 2025,2018-09-16T22:24:19,122 -+54 398 559 0423,+48 195 624 2025,2018-09-17T22:24:19,122 -+54 398 559 0423,+48 195 624 2025,2018-09-18T22:24:19,122 -+54 398 559 0423,+48 195 624 2025,2018-09-19T22:24:19,122 -+54 398 559 0423,+48 195 624 2025,2018-09-20T22:24:19,122 ++54 398 559 0423,+48 195 624 2025,2018-09-17T22:24:19,123 ++54 398 559 0423,+48 195 624 2025,2018-09-18T22:24:19,124 ++54 398 559 0423,+48 195 624 2025,2018-09-19T22:24:19,125 ++54 398 559 0423,+48 195 624 2025,2018-09-20T22:24:19,126 +263 498 495 0617,+48 195 624 2025,2018-09-18T01:34:48,514 +81 308 988 7153,+33 614 339 0298,2018-09-21T20:21:17,120 +263 498 495 0617,+33 614 339 0298,2018-09-11T22:10:34,144 -+263 498 495 0617,+33 614 339 0298,2018-09-12T22:10:34,144 -+263 498 495 0617,+33 614 339 0298,2018-09-13T22:10:34,144 -+263 498 495 0617,+33 614 339 0298,2018-09-14T22:10:34,144 -+263 498 495 0617,+33 614 339 0298,2018-09-15T22:10:34,144 -+263 498 495 0617,+33 614 339 0298,2018-09-16T22:10:34,144 ++263 498 495 0617,+33 614 339 0298,2018-09-12T22:10:34,145 ++263 498 495 0617,+33 614 339 0298,2018-09-13T22:10:34,146 ++263 498 495 0617,+33 614 339 0298,2018-09-14T22:10:34,147 ++263 498 495 0617,+33 614 339 0298,2018-09-15T22:10:34,148 ++263 498 495 0617,+33 614 339 0298,2018-09-16T22:10:34,149 +54 398 559 0423,+7 552 196 4096,2018-09-25T20:24:59,556 +81 308 988 7153,+351 515 605 7915,2018-09-23T22:23:25,336 -+81 308 988 7153,+351 515 605 7915,2018-09-24T22:23:25,336 -+81 308 988 7153,+351 515 605 7915,2018-09-25T22:23:25,336 ++81 308 988 7153,+351 515 605 7915,2018-09-24T22:23:25,332 ++81 308 988 7153,+351 515 605 7915,2018-09-25T22:23:25,331 +261 860 539 4754,+351 272 414 6570,2018-09-26T05:34:19,405 +62 107 530 7500,+263 498 495 0617,2018-09-25T22:58:02,5665 +54 398 559 0423,+86 892 682 0628,2018-09-23T08:55:18,822 diff --git a/src/test/resources/phone-calls/migrationStatus.json b/src/test/resources/phone-calls/migrationStatus.json index d649d61..c05578e 100644 --- a/src/test/resources/phone-calls/migrationStatus.json +++ b/src/test/resources/phone-calls/migrationStatus.json @@ -1 +1 @@ -{"src/test/resources/phone-calls/is-in-use.csv":{"conceptName":"is-in-use","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"src/test/resources/phone-calls/in-use.csv":{"conceptName":"in-use","isCompleted":true,"migratedRows":8},"src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12}} \ No newline at end of file +{"append-pp-fakebook-src/test/resources/phone-calls/append-fb-preprocessed.csv":{"conceptName":"person","isCompleted":true,"migratedRows":5},"person-src/test/resources/phone-calls/person.csv":{"conceptName":"person","isCompleted":true,"migratedRows":35},"in-use-src/test/resources/phone-calls/in-use.csv":{"conceptName":"in-use","isCompleted":true,"migratedRows":8},"company-src/test/resources/phone-calls/company.csv":{"conceptName":"company","isCompleted":true,"migratedRows":2},"calls-src/test/resources/phone-calls/call.csv":{"conceptName":"call","isCompleted":true,"migratedRows":218},"contract-src/test/resources/phone-calls/contract.csv":{"conceptName":"contract","isCompleted":true,"migratedRows":12},"is-in-use-src/test/resources/phone-calls/is-in-use.csv":{"conceptName":"is-in-use","isCompleted":true,"migratedRows":2},"communication-channel-pm-src/test/resources/phone-calls/communication-channel-pm.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":6},"append-twitter-src/test/resources/phone-calls/append-twitter.csv":{"conceptName":"person","isCompleted":true,"migratedRows":10},"communication-channel-src/test/resources/phone-calls/communication-channel.csv":{"conceptName":"communication-channel","isCompleted":true,"migratedRows":7},"append-call-rating-src/test/resources/phone-calls/append-call-rating.csv":{"conceptName":"call","isCompleted":true,"migratedRows":7}} \ No newline at end of file