From f08a606ce8aa180fc4a54a3ab4d817271f0dd222 Mon Sep 17 00:00:00 2001 From: datomo Date: Tue, 7 Nov 2023 22:03:17 +0100 Subject: [PATCH] fixing parts of the query routing, adding version specific folders, new modes --- build.gradle | 15 ++-- .../db/util/PolyphenyHomeDirManager.java | 19 ++++- .../org/polypheny/db/util/PolyphenyMode.java | 24 ++++++ .../polypheny/db/util/VersionCollector.java | 69 +++++++++++++++++ .../db/algebra/enumerable/EnumerableScan.java | 2 +- .../org/polypheny/db/catalog/Catalog.java | 8 +- .../polypheny/db/docker/DockerContainer.java | 2 +- .../polypheny/db/docker/DockerManager.java | 2 +- .../db/routing/ProposedRoutingPlan.java | 4 +- .../org/polypheny/db/routing/RoutingPlan.java | 4 +- .../polypheny/db/tools/RoutedAlgBuilder.java | 17 +---- dbms/build.gradle | 42 ++++++++++- .../java/org/polypheny/db/PolyphenyDb.java | 6 +- .../db/cli/PolyphenyModesConverter.java | 36 ++++++--- .../org/polypheny/db/ddl/DdlManagerImpl.java | 6 +- .../db/processing/AbstractQueryProcessor.java | 75 +++++++++---------- .../db/routing/UiRoutingPageUtil.java | 5 +- .../dto/CachedProposedRoutingPlan.java | 4 +- .../routing/dto/ProposedRoutingPlanImpl.java | 12 +-- .../db/routing/routers/CachedPlanRouter.java | 5 +- .../routers/FullPlacementQueryRouter.java | 13 +--- .../db/routing/routers/IcarusRouter.java | 5 +- .../db/routing/routers/SimpleRouter.java | 3 + .../java/org/polypheny/db/TestHelper.java | 2 +- .../db/sql/util/PlannerImplMock.java | 2 +- .../polypheny/db/webui/crud/LanguageCrud.java | 2 +- 26 files changed, 259 insertions(+), 125 deletions(-) create mode 100644 config/src/main/java/org/polypheny/db/util/PolyphenyMode.java create mode 100644 config/src/main/java/org/polypheny/db/util/VersionCollector.java diff --git a/build.gradle b/build.gradle index 39f49daf9d..93f86a7c60 100644 --- a/build.gradle +++ b/build.gradle @@ -163,14 +163,6 @@ allprojects { test.dependsOn(":plugins:assemblePlugins") } -def getCurrentBranch() { - def branch = "" - def proc = "git rev-parse --abbrev-ref HEAD".execute() - proc.in.eachLine { line -> branch = line } - proc.err.eachLine { line -> println line } - proc.waitFor() - branch -} idea { @@ -178,10 +170,13 @@ idea { settings { runConfigurations { "Polypheny-DB"(Gradle) { - taskNames = ["assemblePlugins", "run"] + taskNames = ["assemblePlugins", "runDev"] } "Polypheny-DB (reset)"(Gradle) { - taskNames = ["assemblePlugins", ":dbms:runReset"] + taskNames = ["assemblePlugins", ":dbms:runDevReset"] + } + "Polypheny-DB (production)"(Gradle) { + taskNames = ["assemblePlugins", "run"] } } copyright { diff --git a/config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java b/config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java index 8555db98ab..d81b2d2d79 100644 --- a/config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java +++ b/config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -36,6 +37,7 @@ public class PolyphenyHomeDirManager { private File root; private final List dirs = new ArrayList<>(); private final List deleteOnExit = new ArrayList<>(); + private static PolyphenyMode mode; public static PolyphenyHomeDirManager getInstance() { @@ -53,7 +55,7 @@ private PolyphenyHomeDirManager() { } else { pathVar = System.getProperty( "user.home" ); } - root = new File( pathVar, ".polypheny" ); + root = Path.of( pathVar, ".polypheny", getPrefix() ).toFile(); if ( !tryCreatingFolder( root ) ) { root = new File( "." ); @@ -72,6 +74,21 @@ private PolyphenyHomeDirManager() { } + public static PolyphenyHomeDirManager setModeAndGetInstance( PolyphenyMode mode ) { + if ( PolyphenyHomeDirManager.mode != null ) { + throw new RuntimeException( "Could not set the mode." ); + } + PolyphenyHomeDirManager.mode = mode; + return PolyphenyHomeDirManager.getInstance(); + } + + + private String getPrefix() { + VersionCollector collector = VersionCollector.INSTANCE; + return mode == PolyphenyMode.PRODUCTION ? collector.version : collector.version + "-" + collector.branch; + } + + private boolean tryCreatingFolder( File file ) { if ( file.isFile() ) { return false; diff --git a/config/src/main/java/org/polypheny/db/util/PolyphenyMode.java b/config/src/main/java/org/polypheny/db/util/PolyphenyMode.java new file mode 100644 index 0000000000..8c79512f80 --- /dev/null +++ b/config/src/main/java/org/polypheny/db/util/PolyphenyMode.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.util; + +public enum PolyphenyMode { + PRODUCTION, + DEVELOPMENT, + TEST, + BENCHMARK +} diff --git a/config/src/main/java/org/polypheny/db/util/VersionCollector.java b/config/src/main/java/org/polypheny/db/util/VersionCollector.java new file mode 100644 index 0000000000..072750563b --- /dev/null +++ b/config/src/main/java/org/polypheny/db/util/VersionCollector.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019-2023 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.util; + + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class VersionCollector { + + + public static final String VERSION = "version"; + public static final String BRANCH = "branch"; + public static VersionCollector INSTANCE = new VersionCollector(); + + private final Properties versionProperties = new Properties(); + + final String version; + + final String branch; + + public boolean inJar = false; + + + private VersionCollector() { + InputStream inputStream = getClass().getClassLoader().getResourceAsStream( "classpath:/version.properties" ); + + try { + if ( inputStream == null ) { + inJar = true; + // When running unit tests, no jar is built, so we load a copy of the file that we saved during build.gradle. + // Possibly this also is the case during debugging, therefore we save in bin/main instead of bin/test. + inputStream = new FileInputStream( "bin/main/version.properties" ); + } + + versionProperties.load( inputStream ); + } catch ( Exception e ) { + useDefaults(); + } + version = versionProperties.getProperty( VERSION ); + branch = versionProperties.getProperty( BRANCH ); + + } + + + private void useDefaults() { + log.warn( "Using default version." ); + versionProperties.put( VERSION, "default" ); + versionProperties.put( BRANCH, "default" ); + } + +} diff --git a/core/src/main/java/org/polypheny/db/algebra/enumerable/EnumerableScan.java b/core/src/main/java/org/polypheny/db/algebra/enumerable/EnumerableScan.java index 536c34c324..914ea649e1 100644 --- a/core/src/main/java/org/polypheny/db/algebra/enumerable/EnumerableScan.java +++ b/core/src/main/java/org/polypheny/db/algebra/enumerable/EnumerableScan.java @@ -38,7 +38,6 @@ import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.entity.LogicalEntity; import org.polypheny.db.catalog.entity.physical.PhysicalTable; import org.polypheny.db.interpreter.Row; @@ -53,6 +52,7 @@ import org.polypheny.db.schema.types.ScannableEntity; import org.polypheny.db.schema.types.StreamableEntity; import org.polypheny.db.util.BuiltInMethod; +import org.polypheny.db.util.PolyphenyMode; /** diff --git a/core/src/main/java/org/polypheny/db/catalog/Catalog.java b/core/src/main/java/org/polypheny/db/catalog/Catalog.java index 13258cb700..640b682d06 100644 --- a/core/src/main/java/org/polypheny/db/catalog/Catalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/Catalog.java @@ -48,6 +48,7 @@ import org.polypheny.db.catalog.logistic.NamespaceType; import org.polypheny.db.catalog.snapshot.Snapshot; import org.polypheny.db.iface.QueryInterfaceManager.QueryInterfaceTemplate; +import org.polypheny.db.util.PolyphenyMode; public abstract class Catalog implements ExtensionPoint { @@ -275,11 +276,4 @@ public static Snapshot snapshot() { public abstract void restore(); - public enum PolyphenyMode { - DEFAULT, - TEST, - BENCHMARK - } - - } diff --git a/core/src/main/java/org/polypheny/db/docker/DockerContainer.java b/core/src/main/java/org/polypheny/db/docker/DockerContainer.java index 43b4dca83c..5146869c84 100644 --- a/core/src/main/java/org/polypheny/db/docker/DockerContainer.java +++ b/core/src/main/java/org/polypheny/db/docker/DockerContainer.java @@ -39,9 +39,9 @@ import org.bouncycastle.tls.TlsFatalAlert; import org.bouncycastle.tls.TlsNoCloseNotifyException; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.config.RuntimeConfig; +import org.polypheny.db.util.PolyphenyMode; /** * The container is the main interaction instance for calling classes when interacting with Docker. diff --git a/core/src/main/java/org/polypheny/db/docker/DockerManager.java b/core/src/main/java/org/polypheny/db/docker/DockerManager.java index a05a1e81fe..44a6e6cbd7 100644 --- a/core/src/main/java/org/polypheny/db/docker/DockerManager.java +++ b/core/src/main/java/org/polypheny/db/docker/DockerManager.java @@ -27,12 +27,12 @@ import java.util.stream.Collectors; import javax.annotation.Nullable; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.config.Config.ConfigListener; import org.polypheny.db.config.ConfigDocker; import org.polypheny.db.config.ConfigManager; import org.polypheny.db.config.RuntimeConfig; +import org.polypheny.db.util.PolyphenyMode; public final class DockerManager { diff --git a/core/src/main/java/org/polypheny/db/routing/ProposedRoutingPlan.java b/core/src/main/java/org/polypheny/db/routing/ProposedRoutingPlan.java index 8043bd2198..d5d63c9d33 100644 --- a/core/src/main/java/org/polypheny/db/routing/ProposedRoutingPlan.java +++ b/core/src/main/java/org/polypheny/db/routing/ProposedRoutingPlan.java @@ -19,8 +19,8 @@ import java.util.List; import java.util.Map; import org.polypheny.db.algebra.AlgRoot; +import org.polypheny.db.catalog.entity.allocation.AllocationColumn; import org.polypheny.db.plan.AlgOptCost; -import org.polypheny.db.util.Pair; public interface ProposedRoutingPlan extends RoutingPlan { @@ -63,7 +63,7 @@ public interface ProposedRoutingPlan extends RoutingPlan { * @return The physical placements of the necessary partitions: {@code AllocationId -> List} */ @Override - Map>> getPhysicalPlacementsOfPartitions(); // PartitionId -> List + Map> getPhysicalPlacementsOfPartitions(); // PartitionId -> List /** * @return Optional pre costs. diff --git a/core/src/main/java/org/polypheny/db/routing/RoutingPlan.java b/core/src/main/java/org/polypheny/db/routing/RoutingPlan.java index 960b3f9de2..c1d98a5d3b 100644 --- a/core/src/main/java/org/polypheny/db/routing/RoutingPlan.java +++ b/core/src/main/java/org/polypheny/db/routing/RoutingPlan.java @@ -18,7 +18,7 @@ import java.util.List; import java.util.Map; -import org.polypheny.db.util.Pair; +import org.polypheny.db.catalog.entity.allocation.AllocationColumn; /** @@ -33,6 +33,6 @@ public interface RoutingPlan { Class getRouter(); // PartitionId -> List - Map>> getPhysicalPlacementsOfPartitions(); + Map> getPhysicalPlacementsOfPartitions(); } diff --git a/core/src/main/java/org/polypheny/db/tools/RoutedAlgBuilder.java b/core/src/main/java/org/polypheny/db/tools/RoutedAlgBuilder.java index 7409b9ee48..c005f64a03 100644 --- a/core/src/main/java/org/polypheny/db/tools/RoutedAlgBuilder.java +++ b/core/src/main/java/org/polypheny/db/tools/RoutedAlgBuilder.java @@ -20,8 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; import lombok.Getter; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.type.AlgDataType; @@ -34,16 +32,15 @@ import org.polypheny.db.rex.RexLiteral; import org.polypheny.db.transaction.Statement; import org.polypheny.db.type.entity.document.PolyDocument; -import org.polypheny.db.util.Pair; /** * Extension of RelBuilder for routed plans with some more information. */ +@Getter public class RoutedAlgBuilder extends AlgBuilder { - @Getter - protected Map>> physicalPlacementsOfPartitions = new HashMap<>(); // PartitionId -> List + protected Map> physicalPlacementsOfPartitions = new HashMap<>(); // PartitionId -> List public RoutedAlgBuilder( Context context, AlgOptCluster cluster, Snapshot snapshot ) { @@ -99,16 +96,10 @@ public RoutedAlgBuilder documents( List documents, AlgDataType row } - public void addPhysicalInfo( Map> physicalPlacements ) { - final Map>> map = physicalPlacements.entrySet().stream() - .collect( Collectors.toMap( Entry::getKey, entry -> map( entry.getValue() ) ) ); - physicalPlacementsOfPartitions.putAll( map ); + public void addPhysicalInfo( Map> partitionColumns ) { + physicalPlacementsOfPartitions.putAll( partitionColumns ); } - private List> map( List columns ) { - return columns.stream().map( col -> new Pair<>( col.placementId, col.columnId ) ).collect( Collectors.toList() ); - } - } diff --git a/dbms/build.gradle b/dbms/build.gradle index 3d88f9696e..724f73b11a 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -251,12 +251,20 @@ application { //applicationDefaultJvmArgs = ["-Dlog4j.configuration=config/log4j.properties"] } -tasks.register('runReset', JavaExec) { +tasks.register('runDevReset', JavaExec) { group = 'Execution' description = 'Run the main class with JavaExecTask' classpath sourceSets.main.runtimeClasspath mainClass = 'org.polypheny.db.PolyphenyDb' - args = ['-resetCatalog', '-resetPlugins'] + args = ['-resetCatalog', '-resetPlugins', '-mode', 'dev'] +} + +tasks.register('runDev', JavaExec) { + group = 'Execution' + description = 'Run the main class with JavaExecTask' + classpath sourceSets.main.runtimeClasspath + mainClass = 'org.polypheny.db.PolyphenyDb' + args = ['-mode', 'dev'] } startScripts { @@ -290,3 +298,33 @@ shadowDistTar { shadowDistZip { dependsOn(jar) } + +def getCurrentBranch() { + def branch = "" + def proc = "git rev-parse --abbrev-ref HEAD".execute() + proc.in.eachLine { line -> branch = line } + proc.err.eachLine { line -> println line } + proc.waitFor() + branch +} + +task createProperties() { + doLast { + def details = getCurrentBranch(); + System.out.println("$buildDir") + new File("$buildDir/version.properties").withWriter { w -> + Properties p = new Properties() + p['version'] = version + p['branch'] = details; + p.store w, null + } + // copy needed, otherwise the bean VersionController can't load the file at startup when running complete-app tests. + copy { + from "$buildDir/version.properties" + into "bin/main/" + } + } +} +classes { + dependsOn createProperties +} diff --git a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java index 7328c8666a..5c062ac608 100644 --- a/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java +++ b/dbms/src/main/java/org/polypheny/db/PolyphenyDb.java @@ -42,7 +42,6 @@ import org.polypheny.db.adapter.index.IndexManager; import org.polypheny.db.adapter.java.AdapterTemplate; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.entity.LogicalAdapter.AdapterType; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.catalog.impl.PolyCatalog; @@ -81,6 +80,7 @@ import org.polypheny.db.transaction.TransactionManager; import org.polypheny.db.transaction.TransactionManagerImpl; import org.polypheny.db.util.PolyphenyHomeDirManager; +import org.polypheny.db.util.PolyphenyMode; import org.polypheny.db.view.MaterializedViewManager; import org.polypheny.db.view.MaterializedViewManagerImpl; import org.polypheny.db.webui.ConfigService; @@ -114,7 +114,7 @@ public class PolyphenyDb { public boolean memoryCatalog = false; @Option(name = { "-mode" }, description = "Special system configuration for running tests", typeConverterProvider = PolyphenyModesConverter.class) - public PolyphenyMode mode = PolyphenyMode.DEFAULT; + public PolyphenyMode mode = PolyphenyMode.PRODUCTION; @Option(name = { "-gui" }, description = "Show splash screen on startup and add taskbar gui") public boolean desktopMode = false; @@ -223,7 +223,7 @@ public void runPolyphenyDb() { } // Restore content of Polypheny folder - PolyphenyHomeDirManager phdm = PolyphenyHomeDirManager.getInstance(); + PolyphenyHomeDirManager phdm = PolyphenyHomeDirManager.setModeAndGetInstance( mode ); if ( phdm.checkIfExists( "_test_backup" ) && phdm.getFileIfExists( "_test_backup" ).isDirectory() ) { File backupFolder = phdm.getFileIfExists( "_test_backup" ); // Cleanup Polypheny folder diff --git a/dbms/src/main/java/org/polypheny/db/cli/PolyphenyModesConverter.java b/dbms/src/main/java/org/polypheny/db/cli/PolyphenyModesConverter.java index 7b702621b6..7572f9750a 100644 --- a/dbms/src/main/java/org/polypheny/db/cli/PolyphenyModesConverter.java +++ b/dbms/src/main/java/org/polypheny/db/cli/PolyphenyModesConverter.java @@ -16,32 +16,50 @@ package org.polypheny.db.cli; +import com.github.rvesse.airline.model.ArgumentsMetadata; +import com.github.rvesse.airline.model.OptionMetadata; +import com.github.rvesse.airline.parser.ParseState; import com.github.rvesse.airline.types.DefaultTypeConverter; +import com.github.rvesse.airline.types.TypeConverter; import java.util.Arrays; import lombok.extern.slf4j.Slf4j; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; +import org.polypheny.db.util.PolyphenyMode; @Slf4j public class PolyphenyModesConverter extends DefaultTypeConverter { + @Override + public TypeConverter getTypeConverter( OptionMetadata option, ParseState state ) { + return this; + } + + + @Override + public TypeConverter getTypeConverter( ArgumentsMetadata arguments, ParseState state ) { + return this; + } + + @Override public Object convert( String name, Class type, String value ) { - String adjustedName = name.toUpperCase(); + String adjustedName = value.toUpperCase(); if ( Arrays.stream( PolyphenyMode.values() ).anyMatch( v -> v.name().equals( adjustedName ) ) ) { - return PolyphenyMode.valueOf( adjustedName ); + return PolyphenyMode.valueOf( adjustedName.toUpperCase() ); } - switch ( adjustedName ) { - case "T": + switch ( adjustedName.toLowerCase() ) { + case "t": return PolyphenyMode.TEST; - case "B": + case "b": + case "bench": return PolyphenyMode.BENCHMARK; - case "D": - return PolyphenyMode.DEFAULT; + case "d": + case "dev": + return PolyphenyMode.DEVELOPMENT; } log.warn( "Could not find the mode: " + adjustedName ); - return PolyphenyMode.DEFAULT; + return PolyphenyMode.PRODUCTION; } } diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 425802ca20..9d56f425ee 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -221,11 +221,11 @@ private void handleSource( DataSource adapter ) { LogicalTable logical = catalog.getLogicalRel( Catalog.defaultNamespaceId ).addTable( tableName, EntityType.SOURCE, !(adapter).isDataReadOnly() ); List columns = new ArrayList<>(); - AllocationPartitionGroup group = catalog.getAllocRel( Catalog.defaultNamespaceId ).addPartitionGroup( logical.id, null, Catalog.defaultNamespaceId, PartitionType.NONE, 1, true ); - AllocationPartition partition = catalog.getAllocRel( Catalog.defaultNamespaceId ).addPartition( logical.id, Catalog.defaultNamespaceId, group.id, UNPARTITIONED, false, PlacementType.AUTOMATIC, DataPlacementRole.UP_TO_DATE, null, PartitionType.NONE ); + + Pair partitionProperty = createSinglePartition( logical.namespaceId, logical ); AllocationPlacement placement = catalog.getAllocRel( Catalog.defaultNamespaceId ).addPlacement( logical.id, Catalog.defaultNamespaceId, adapter.adapterId ); - AllocationEntity allocation = catalog.getAllocRel( Catalog.defaultNamespaceId ).addAllocation( adapter.getAdapterId(), placement.id, partition.id, logical.id ); + AllocationEntity allocation = catalog.getAllocRel( Catalog.defaultNamespaceId ).addAllocation( adapter.getAdapterId(), placement.id, partitionProperty.left.id, logical.id ); List aColumns = new ArrayList<>(); int colPos = 1; diff --git a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java index 81fb0e3c2f..5571374822 100644 --- a/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java +++ b/dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java @@ -95,6 +95,8 @@ import org.polypheny.db.monitoring.events.MonitoringType; import org.polypheny.db.monitoring.events.QueryEvent; import org.polypheny.db.monitoring.events.StatementEvent; +import org.polypheny.db.partition.PartitionManagerFactory; +import org.polypheny.db.partition.properties.PartitionProperty; import org.polypheny.db.plan.AlgOptCost; import org.polypheny.db.plan.AlgOptUtil; import org.polypheny.db.plan.AlgTraitSet; @@ -1300,58 +1302,51 @@ private Map> getAccessedPartitionsPerScan( AlgNode alg, Map< // Get placements of this table LogicalTable catalogTable = table.unwrap( LogicalTable.class ); + PartitionProperty property = Catalog.snapshot().alloc().getPartitionProperty( catalogTable.id ).orElseThrow(); + fallback = true; // todo dl - /*if ( aggregatedPartitionValues.containsKey( scanId ) ) { - if ( aggregatedPartitionValues.get( scanId ) != null ) { - if ( !aggregatedPartitionValues.get( scanId ).isEmpty() ) { - List partitionValues = new ArrayList<>( aggregatedPartitionValues.get( scanId ) ); - - if ( log.isDebugEnabled() ) { - /*log.debug( - "TableID: {} is partitioned on column: {} - {}", - catalogTable.id, - catalogTable.partitionProperty.partitionColumnId, - Catalog.getInstance().getSnapshot().rel().getColumn( catalogTable.partitionProperty.partitionColumnId ).name ); + if ( aggregatedPartitionValues.containsKey( scanId ) && aggregatedPartitionValues.get( scanId ) != null && !aggregatedPartitionValues.get( scanId ).isEmpty() ) { + fallback = false; + List partitionValues = new ArrayList<>( aggregatedPartitionValues.get( scanId ) ); - } - List identifiedPartitions = new ArrayList<>(); - for ( String partitionValue : partitionValues ) { - if ( log.isDebugEnabled() ) { - log.debug( "Extracted PartitionValue: {}", partitionValue ); - } - long identifiedPartition = PartitionManagerFactory.getInstance() - .getPartitionManager( catalogTable.partitionProperty.partitionType ) - .getTargetPartitionId( catalogTable, partitionValue ); + if ( log.isDebugEnabled() ) { + log.debug( + "TableID: {} is partitioned on column: {} - {}", + catalogTable.id, + property.partitionColumnId, + Catalog.getInstance().getSnapshot().rel().getColumn( property.partitionColumnId ).orElseThrow().name ); - identifiedPartitions.add( identifiedPartition ); - if ( log.isDebugEnabled() ) { - log.debug( "Identified PartitionId: {} for value: {}", identifiedPartition, partitionValue ); - } - } + } + List identifiedPartitions = new ArrayList<>(); + for ( String partitionValue : partitionValues ) { + if ( log.isDebugEnabled() ) { + log.debug( "Extracted PartitionValue: {}", partitionValue ); + } + long identifiedPartition = PartitionManagerFactory.getInstance() + .getPartitionManager( property.partitionType ) + .getTargetPartitionId( catalogTable, property, partitionValue ); - accessedPartitionList.merge( - scanId, - identifiedPartitions, - ( l1, l2 ) -> Stream.concat( l1.stream(), l2.stream() ).collect( Collectors.toList() ) ); - scanPerTable.putIfAbsent( scanId, catalogTable.id ); - // Fallback all partitionIds are needed - } else { - fallback = true; + identifiedPartitions.add( identifiedPartition ); + if ( log.isDebugEnabled() ) { + log.debug( "Identified PartitionId: {} for value: {}", identifiedPartition, partitionValue ); } - } else { - fallback = true; } - } else { - fallback = true; + + accessedPartitions.merge( + scanId, + identifiedPartitions, + ( l1, l2 ) -> Stream.concat( l1.stream(), l2.stream() ).collect( Collectors.toList() ) ); + scanPerTable.putIfAbsent( scanId, catalogTable.id ); + // Fallback all partitionIds are needed } if ( fallback ) { - accessedPartitionList.merge( + accessedPartitions.merge( scanId, - catalogTable.partitionProperty.partitionIds, + property.partitionIds, ( l1, l2 ) -> Stream.concat( l1.stream(), l2.stream() ).collect( Collectors.toList() ) ); scanPerTable.putIfAbsent( scanId, catalogTable.id ); - }*/ + } } } diff --git a/dbms/src/main/java/org/polypheny/db/routing/UiRoutingPageUtil.java b/dbms/src/main/java/org/polypheny/db/routing/UiRoutingPageUtil.java index 2e85fb78f1..ed7c2b204a 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/UiRoutingPageUtil.java +++ b/dbms/src/main/java/org/polypheny/db/routing/UiRoutingPageUtil.java @@ -42,7 +42,6 @@ import org.polypheny.db.plan.AlgOptUtil; import org.polypheny.db.processing.util.Plan; import org.polypheny.db.transaction.Statement; -import org.polypheny.db.util.Pair; /** @@ -104,9 +103,9 @@ private static void addSelectedAdapterTable( InformationManager queryAnalyzer, P group, ImmutableList.of( "Entity", "Field", "Allocation Id", "Adapter" ) ); if ( proposedRoutingPlan.getPhysicalPlacementsOfPartitions() != null ) { - for ( Entry>> entry : proposedRoutingPlan.getPhysicalPlacementsOfPartitions().entrySet() ) { + for ( Entry> entry : proposedRoutingPlan.getPhysicalPlacementsOfPartitions().entrySet() ) { Long k = entry.getKey(); - List> v = entry.getValue(); + List v = entry.getValue(); AllocationEntity alloc = snapshot.alloc().getEntity( k ).orElseThrow(); LogicalEntity entity = snapshot.getLogicalEntity( alloc.logicalId ).orElseThrow(); diff --git a/dbms/src/main/java/org/polypheny/db/routing/dto/CachedProposedRoutingPlan.java b/dbms/src/main/java/org/polypheny/db/routing/dto/CachedProposedRoutingPlan.java index c2c09d278d..f40ed06056 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/dto/CachedProposedRoutingPlan.java +++ b/dbms/src/main/java/org/polypheny/db/routing/dto/CachedProposedRoutingPlan.java @@ -21,11 +21,11 @@ import java.util.Map; import lombok.Getter; import lombok.Setter; +import org.polypheny.db.catalog.entity.allocation.AllocationColumn; import org.polypheny.db.plan.AlgOptCost; import org.polypheny.db.routing.ProposedRoutingPlan; import org.polypheny.db.routing.Router; import org.polypheny.db.routing.RoutingPlan; -import org.polypheny.db.util.Pair; /** @@ -36,7 +36,7 @@ public class CachedProposedRoutingPlan implements RoutingPlan { @Getter - public Map>> physicalPlacementsOfPartitions; // PartitionId -> List> + public Map> physicalPlacementsOfPartitions; // PartitionId -> List> protected String queryClass; protected String physicalQueryClass; protected AlgOptCost preCosts; diff --git a/dbms/src/main/java/org/polypheny/db/routing/dto/ProposedRoutingPlanImpl.java b/dbms/src/main/java/org/polypheny/db/routing/dto/ProposedRoutingPlanImpl.java index 577b8ba137..9af7db8d2c 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/dto/ProposedRoutingPlanImpl.java +++ b/dbms/src/main/java/org/polypheny/db/routing/dto/ProposedRoutingPlanImpl.java @@ -25,11 +25,11 @@ import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.catalog.entity.allocation.AllocationColumn; import org.polypheny.db.plan.AlgOptCost; import org.polypheny.db.routing.ProposedRoutingPlan; import org.polypheny.db.routing.Router; import org.polypheny.db.tools.RoutedAlgBuilder; -import org.polypheny.db.util.Pair; /** @@ -43,7 +43,7 @@ public class ProposedRoutingPlanImpl implements ProposedRoutingPlan { protected String queryClass; protected String physicalQueryClass; protected Class router; - protected Map>> physicalPlacementsOfPartitions; // PartitionId -> List + protected Map> physicalPlacementsOfPartitions; // PartitionId -> List protected AlgOptCost preCosts; @@ -114,9 +114,9 @@ public boolean equals( Object obj ) { return true; } - for ( Map.Entry>> entry : this.physicalPlacementsOfPartitions.entrySet() ) { + for ( Map.Entry> entry : this.physicalPlacementsOfPartitions.entrySet() ) { final Long id = entry.getKey(); - List> values = entry.getValue(); + List values = entry.getValue(); if ( !other.physicalPlacementsOfPartitions.containsKey( id ) ) { return false; @@ -141,9 +141,9 @@ public int hashCode() { if ( this.physicalPlacementsOfPartitions != null && !this.physicalPlacementsOfPartitions.isEmpty() ) { return this.physicalPlacementsOfPartitions.values() .stream().flatMap( Collection::stream ) - .map( elem -> elem.right.hashCode() * elem.left.hashCode() ) + .map( AllocationColumn::hashCode ) .reduce( Integer::sum ) - .get(); + .orElseThrow(); } return super.hashCode(); } diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/CachedPlanRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/CachedPlanRouter.java index 277cef1456..212146e3bf 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/CachedPlanRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/CachedPlanRouter.java @@ -19,7 +19,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.algebra.AlgNode; import org.polypheny.db.algebra.AlgRoot; @@ -68,9 +67,7 @@ private RoutedAlgBuilder buildCachedSelect( AlgNode node, RoutedAlgBuilder build Map> partitions = new HashMap<>(); for ( long partition : partitionIds ) { if ( cachedPlan.physicalPlacementsOfPartitions.get( partition ) != null ) { - List colPlacements = cachedPlan.physicalPlacementsOfPartitions.get( partition ).stream() - .map( placementInfo -> catalog.getSnapshot().alloc().getColumn( placementInfo.left, placementInfo.right ).orElseThrow() ) - .collect( Collectors.toList() ); + List colPlacements = cachedPlan.physicalPlacementsOfPartitions.get( partition ); partitions.put( partition, colPlacements ); } } diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/FullPlacementQueryRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/FullPlacementQueryRouter.java index 16087ada05..f981ebad38 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/FullPlacementQueryRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/FullPlacementQueryRouter.java @@ -108,20 +108,15 @@ protected List handleNonePartitioning( log.debug( "{} is NOT partitioned - Routing will be easy", table.name ); } - //final Set> placements = selectPlacement( table, queryInformation ); - List newBuilders = new ArrayList<>(); - /*for ( List placementCombination : placements ) { - Map> currentPlacementDistribution = new HashMap<>(); - PartitionProperty property = snapshot.alloc().getPartitionProperty( catalogTable.id );*/ - //currentPlacementDistribution.put( property.partitionIds.get( 0 ), placementCombination ); - List allocationEntities = Catalog.snapshot().alloc().getPlacementsFromLogical( table.id ); - List allocs = Catalog.snapshot().alloc().getAllocsOfPlacement( allocationEntities.get( 0 ).id ); + List placements = Catalog.snapshot().alloc().getPlacementsFromLogical( table.id ); + List allocs = Catalog.snapshot().alloc().getAllocsOfPlacement( placements.get( 0 ).id ); + List columns = Catalog.snapshot().alloc().getColumns( allocs.get( 0 ).placementId ); for ( RoutedAlgBuilder builder : builders ) { RoutedAlgBuilder newBuilder = RoutedAlgBuilder.createCopy( statement, cluster, builder ); - //newBuilder.addPhysicalInfo( currentPlacementDistribution ); + newBuilder.addPhysicalInfo( Map.of( allocs.get( 0 ).partitionId, columns ) ); newBuilder.push( super.buildJoinedScan( statement, cluster, table, Map.of( allocs.get( 0 ).placementId, allocs.get( 0 ).unwrap( AllocationTable.class ).getColumns() ) ) ); newBuilders.add( newBuilder ); } diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/IcarusRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/IcarusRouter.java index 4ff728ef3b..5bc9309857 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/IcarusRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/IcarusRouter.java @@ -36,7 +36,6 @@ import org.polypheny.db.routing.LogicalQueryInformation; import org.polypheny.db.tools.RoutedAlgBuilder; import org.polypheny.db.transaction.Statement; -import org.polypheny.db.util.Pair; @Slf4j @@ -112,11 +111,11 @@ protected List handleNonePartitioning( // Find corresponding builder: final RoutedAlgBuilder builder = builders.stream().filter( b -> { - final List> listPairs = b.getPhysicalPlacementsOfPartitions().values().stream() + final List listPairs = b.getPhysicalPlacementsOfPartitions().values().stream() .flatMap( Collection::stream ) .collect( Collectors.toList() ); final Optional found = listPairs.stream() - .map( elem -> elem.left ) + .map( elem -> elem.placementId ) .filter( elem -> elem == placementId ) .findFirst(); return found.isPresent(); diff --git a/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java b/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java index 4ff5fafdc0..e09bc0513b 100644 --- a/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java +++ b/dbms/src/main/java/org/polypheny/db/routing/routers/SimpleRouter.java @@ -73,11 +73,14 @@ protected List handleNonePartitioning( AlgNode node, LogicalTa // Get placements and convert into placement distribution // final Map> placements = selectPlacement( catalogTable ); List entities = Catalog.snapshot().alloc().getFromLogical( table.id ); + List columns = Catalog.snapshot().alloc().getColumns( entities.get( 0 ).placementId ); // Only one builder available // builders.get( 0 ).addPhysicalInfo( placements ); super.handleRelScan( builders.get( 0 ), statement, entities.get( 0 ) ); + builders.get( 0 ).addPhysicalInfo( Map.of( entities.get( 0 ).partitionId, columns ) ); + return builders; } diff --git a/dbms/src/test/java/org/polypheny/db/TestHelper.java b/dbms/src/test/java/org/polypheny/db/TestHelper.java index 09faf4906c..f1c02eabaf 100644 --- a/dbms/src/test/java/org/polypheny/db/TestHelper.java +++ b/dbms/src/test/java/org/polypheny/db/TestHelper.java @@ -56,7 +56,6 @@ import org.junit.Assert; import org.polypheny.db.algebra.type.DocumentType; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.functions.Functions; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.transaction.TransactionManager; @@ -68,6 +67,7 @@ import org.polypheny.db.type.entity.PolyString; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.util.Pair; +import org.polypheny.db.util.PolyphenyMode; import org.polypheny.db.webui.HttpServer; import org.polypheny.db.webui.models.results.DocResult; import org.polypheny.db.webui.models.results.GraphResult; diff --git a/plugins/sql-language/src/test/java/org/polypheny/db/sql/util/PlannerImplMock.java b/plugins/sql-language/src/test/java/org/polypheny/db/sql/util/PlannerImplMock.java index e86fc28cac..23f15c036b 100644 --- a/plugins/sql-language/src/test/java/org/polypheny/db/sql/util/PlannerImplMock.java +++ b/plugins/sql-language/src/test/java/org/polypheny/db/sql/util/PlannerImplMock.java @@ -26,7 +26,6 @@ import org.polypheny.db.algebra.metadata.CachingAlgMetadataProvider; import org.polypheny.db.algebra.operators.OperatorTable; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.snapshot.Snapshot; import org.polypheny.db.config.PolyphenyDbConnectionConfig; import org.polypheny.db.languages.NodeParseException; @@ -62,6 +61,7 @@ import org.polypheny.db.tools.Program; import org.polypheny.db.tools.ValidationException; import org.polypheny.db.util.Conformance; +import org.polypheny.db.util.PolyphenyMode; import org.polypheny.db.util.SourceStringReader; diff --git a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java index 07aaf1b6cc..c5b7a6c6d8 100644 --- a/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java +++ b/webui/src/main/java/org/polypheny/db/webui/crud/LanguageCrud.java @@ -37,7 +37,6 @@ import org.polypheny.db.algebra.AlgRoot; import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.catalog.Catalog; -import org.polypheny.db.catalog.Catalog.PolyphenyMode; import org.polypheny.db.catalog.entity.allocation.AllocationEntity; import org.polypheny.db.catalog.entity.allocation.AllocationPlacement; import org.polypheny.db.catalog.entity.logical.LogicalCollection; @@ -60,6 +59,7 @@ import org.polypheny.db.transaction.TransactionManager; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.type.entity.graph.PolyGraph; +import org.polypheny.db.util.PolyphenyMode; import org.polypheny.db.webui.Crud; import org.polypheny.db.webui.models.IndexModel; import org.polypheny.db.webui.models.PlacementModel;