From cec0d95e654923ed715fead6c8f108c66b89154f Mon Sep 17 00:00:00 2001 From: cboucheIGN Date: Wed, 21 Sep 2022 10:22:28 +0200 Subject: [PATCH 1/2] validator-core: Support required constraint in ForeignKeyValidator --- .../database/internal/ForeignKeyFinder.java | 44 ++++++------- .../database/ForeignKeyValidator.java | 2 +- .../constraint/ForeignKeyConstraintTest.java | 8 +++ .../database/ForeignKeyValidatorTest.java | 64 +++++++++++++++++++ 4 files changed, 95 insertions(+), 23 deletions(-) diff --git a/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java b/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java index 81c3ae55..b631a23b 100644 --- a/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java +++ b/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java @@ -6,8 +6,11 @@ import java.util.Arrays; import java.util.List; + import fr.ign.validator.database.Database; import fr.ign.validator.database.RowIterator; +import fr.ign.validator.model.AttributeType; +import fr.ign.validator.model.TableModel; import fr.ign.validator.model.constraint.ForeignKeyConstraint; /** @@ -34,7 +37,7 @@ public ForeignKeyMismatch(String id, String file, String values) { * Retreive all Foreign Key Mismatch by perfoming an SQL query to database * * @param database - * @param tableName + * @param tableModel * @param foreignKey * @return * @throws SQLException @@ -42,7 +45,7 @@ public ForeignKeyMismatch(String id, String file, String values) { */ public List foreignKeyNotFound( Database database, - String tableName, + TableModel tableModel, ForeignKeyConstraint foreignKey) throws SQLException, IOException { // TODO validate condition to avoid SQL injection and crashes @@ -54,29 +57,24 @@ public List foreignKeyNotFound( + " LIKE " + " target." + foreignKey.getTargetColumnNames().get(i); if (i == 0) { - conditions.add(" WHERE " + condition); + conditions.add("WHERE " + condition); } else { - conditions.add(" AND " + condition); + conditions.add("AND " + condition); } } - // Query exemple - // sub request retrieve all foreign key match - // SELECT r.__id, r.__file, TYPEPSC, STYPEPSC - // FROM PRESCRIPTION_SURF AS r WHERE r.__id NOT IN ( - // SELECT a.__id FROM PRESCRIPTION_SURF AS a - // JOIN PrescriptionUrbaType AS b - // ON (a.TYPEPSC LIKE b.TYPEPSC AND a.STYPEPSC LIKE b.STYPEPSC) ) + List nullableClause = new ArrayList(); + for (int i = 0; i < foreignKey.getSourceColumnNames().size(); i++) { + String name = foreignKey.getSourceColumnNames().get(i); + AttributeType attribute = tableModel.getFeatureType().getAttribute(name); + if (attribute == null || attribute.getConstraints().isRequired()) { + continue; + } + String condition = "AND " + name + " IS NOT NULL"; + nullableClause.add(condition); + } -// String query = "SELECT r.__id, r.__file, " -// + String.join(", ", foreignKey.getSourceColumnNames()) -// + " FROM " + tableName + " AS r" -// + " WHERE r.__id NOT IN (" -// + " SELECT a.__id " -// + " FROM " + tableName + " AS a" -// + " JOIN " + foreignKey.getTargetTableName() + " AS b" -// + " ON (" + String.join(" AND ", conditions) + ")" -// + " )"; + String tableName = tableModel.getName(); String query = "SELECT src.__id, src.__file, " + String.join(", ", foreignKey.getSourceColumnNames()) @@ -84,8 +82,10 @@ public List foreignKeyNotFound( + " WHERE NOT EXISTS ( " + " SELECT true " + " FROM " + foreignKey.getTargetTableName() + " AS target " - + String.join("", conditions) - + ")"; + + String.join(" ", conditions) + + ") " + + String.join(" ", nullableClause); + RowIterator it = database.query(query); List result = new ArrayList(); diff --git a/validator-core/src/main/java/fr/ign/validator/validation/database/ForeignKeyValidator.java b/validator-core/src/main/java/fr/ign/validator/validation/database/ForeignKeyValidator.java index af1ef8fb..39ae3470 100644 --- a/validator-core/src/main/java/fr/ign/validator/validation/database/ForeignKeyValidator.java +++ b/validator-core/src/main/java/fr/ign/validator/validation/database/ForeignKeyValidator.java @@ -68,7 +68,7 @@ private void doValidate(Context context, Database database) throws SQLException, List mismatchs = foreignKeyFinder.foreignKeyNotFound( database, - tableModel.getName(), + tableModel, foreignKey ); diff --git a/validator-core/src/test/java/fr/ign/validator/model/constraint/ForeignKeyConstraintTest.java b/validator-core/src/test/java/fr/ign/validator/model/constraint/ForeignKeyConstraintTest.java index 860fa516..a6bb262e 100644 --- a/validator-core/src/test/java/fr/ign/validator/model/constraint/ForeignKeyConstraintTest.java +++ b/validator-core/src/test/java/fr/ign/validator/model/constraint/ForeignKeyConstraintTest.java @@ -15,6 +15,14 @@ public void testParseOk() { Assert.assertEquals("(VALUE,SUB_VALUE) REFERENCES MY_REFERENCE(TYPE,SUB_TYPE)", constraint.toString()); } + @Test + public void testParseGPU() { + String constraintString = "(STYPEP) REFERENCES ListeStypepCC(STYPEP)"; + ForeignKeyConstraint constraint = ForeignKeyConstraint.parseForeignKey(constraintString); + + Assert.assertEquals("(STYPEP) REFERENCES ListeStypepCC(STYPEP)", constraint.toString()); + } + @Test public void testParseFormat2() { String constraintString = " ( VALUE , SUB_VALUE ) REFERENCES MY_REFERENCE ( TYPE, SUB_TYPE ) "; diff --git a/validator-core/src/test/java/fr/ign/validator/validation/database/ForeignKeyValidatorTest.java b/validator-core/src/test/java/fr/ign/validator/validation/database/ForeignKeyValidatorTest.java index e8bd7e88..7f9959bd 100644 --- a/validator-core/src/test/java/fr/ign/validator/validation/database/ForeignKeyValidatorTest.java +++ b/validator-core/src/test/java/fr/ign/validator/validation/database/ForeignKeyValidatorTest.java @@ -51,7 +51,9 @@ public void setUp() throws NoSuchAuthorityCodeException, FactoryException { // creates attributes "TYPE", "SUB_TYPE" AttributeType attribute = new StringType(); attribute.setName("TYPE"); + attribute.getConstraints().setRequired(true); AttributeType attribute2 = new StringType(); + attribute2.getConstraints().setRequired(true); attribute2.setName("SUB_TYPE"); List> attributes = new ArrayList<>(); @@ -74,8 +76,10 @@ public void setUp() throws NoSuchAuthorityCodeException, FactoryException { attribute.setName("ID"); AttributeType attribute2 = new StringType(); attribute2.setName("VALUE"); + attribute2.getConstraints().setRequired(true); AttributeType attribute3 = new StringType(); attribute3.setName("SUB_VALUE"); + attribute3.getConstraints().setRequired(false); List> attributes = new ArrayList<>(); attributes.add(attribute); @@ -187,4 +191,64 @@ public void testNotValid() throws Exception { } + @Test + public void testNullValueNotValid() throws Exception { + // creates an empty database + File path = new File(folder.getRoot(), "document_database.db"); + Database database = new Database(path); + + // add the table TEST into the database + database.query("CREATE TABLE MY_TABLE(__id TEXT, __file TEXT, id TEXT, value TEXT, sub_value TEXT);"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id1', 'type1', 'sub_type1');"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id2', 'type1', 'sub_type2');"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id3', 'type2', 'sub_type1');"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id4', 'type2', null);"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id4', 'type2', 'sub_type5');"); + database.query("INSERT INTO MY_TABLE(id, value, sub_value) VALUES ('id4', null, 'sub_type5');"); + + // add the table RELATION into the database + database.query("CREATE TABLE MY_REFERENCE(__id TEXT, __file TEXT, type TEXT, sub_type TEXT);"); + database.query("INSERT INTO MY_REFERENCE(type, sub_type) VALUES ('type1', 'sub_type1');"); + database.query("INSERT INTO MY_REFERENCE(type, sub_type) VALUES ('type1', 'sub_type2');"); + database.query("INSERT INTO MY_REFERENCE(type, sub_type) VALUES ('type2', 'sub_type5');"); + + // check that the validator doesn't send any error + ForeignKeyValidator validator = new ForeignKeyValidator(); + validator.validate(context, database); + + Assert.assertEquals(2, reportBuilder.getErrorsByCode(CoreErrorCodes.TABLE_FOREIGN_KEY_NOT_FOUND).size()); + + List errors = reportBuilder.getErrorsByCode(CoreErrorCodes.TABLE_FOREIGN_KEY_NOT_FOUND); + int index = 0; + // check first error + { + ValidatorError error = errors.get(index++); + assertEquals("--", error.getAttribute()); + assertEquals(null, error.getId()); + assertEquals("", error.getFeatureId()); + assertEquals("MY_TABLE", error.getFileModel()); + assertEquals(ErrorScope.FEATURE, error.getScope()); + assertEquals( + "La correspondance (VALUE, SUB_VALUE) = (type2, sub_type1) n'est pas autorisée, car non présente dans la liste de référence MY_REFERENCE.", + error.getMessage() + ); + assertEquals("SAMPLE_MODEL", error.getDocumentModel()); + } + + { + ValidatorError error = errors.get(index++); + assertEquals("--", error.getAttribute()); + assertEquals(null, error.getId()); + assertEquals("", error.getFeatureId()); + assertEquals("MY_TABLE", error.getFileModel()); + assertEquals(ErrorScope.FEATURE, error.getScope()); + assertEquals( + "La correspondance (VALUE, SUB_VALUE) = (null, sub_type5) n'est pas autorisée, car non présente dans la liste de référence MY_REFERENCE.", + error.getMessage() + ); + assertEquals("SAMPLE_MODEL", error.getDocumentModel()); + } + + } + } From 228f46bc2290a176860a9e7f3b5815725a3cc5a5 Mon Sep 17 00:00:00 2001 From: cboucheIGN Date: Wed, 21 Sep 2022 10:22:59 +0200 Subject: [PATCH 2/2] mvn formatter:format --- .../database/internal/ForeignKeyFinder.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java b/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java index b631a23b..d9816425 100644 --- a/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java +++ b/validator-core/src/main/java/fr/ign/validator/database/internal/ForeignKeyFinder.java @@ -6,7 +6,6 @@ import java.util.Arrays; import java.util.List; - import fr.ign.validator.database.Database; import fr.ign.validator.database.RowIterator; import fr.ign.validator.model.AttributeType; @@ -65,13 +64,13 @@ public List foreignKeyNotFound( List nullableClause = new ArrayList(); for (int i = 0; i < foreignKey.getSourceColumnNames().size(); i++) { - String name = foreignKey.getSourceColumnNames().get(i); - AttributeType attribute = tableModel.getFeatureType().getAttribute(name); - if (attribute == null || attribute.getConstraints().isRequired()) { - continue; - } + String name = foreignKey.getSourceColumnNames().get(i); + AttributeType attribute = tableModel.getFeatureType().getAttribute(name); + if (attribute == null || attribute.getConstraints().isRequired()) { + continue; + } String condition = "AND " + name + " IS NOT NULL"; - nullableClause.add(condition); + nullableClause.add(condition); } String tableName = tableModel.getName();