Merge pull request #1523 from OriginTrail/prerelease/testnet

OriginTrail Testnet Release v5.0.2
OriginTrail · Apr 23, 2021 · 96bc9b9 · 96bc9b9
2 parents 290de7b + b897bb8
commit 96bc9b9
Show file tree

Hide file tree

Showing 41 changed files with 971 additions and 123 deletions.
diff --git a/config/config.json b/config/config.json
@@ -217,6 +217,11 @@
       "username": "ot_node",
       "password": "origintrail",
       "root_user_password": "origintrail"
+    },
+    "dataset_pruning": {
+      "enabled": false,
+      "imported_pruning_delay_in_minutes": 1440,
+      "replicated_pruning_delay_in_minutes": 1440
     }
   },
   "testnet": {
@@ -412,6 +417,11 @@
       "username": "ot_node",
       "password": "origintrail",
       "root_user_password": "origintrail"
+    },
+    "dataset_pruning": {
+      "enabled": false,
+      "imported_pruning_delay_in_minutes": 1440,
+      "replicated_pruning_delay_in_minutes": 1440
     }
   },
   "mainnet": {
@@ -580,6 +590,11 @@
       "username": "ot_node",
       "password": "origintrail",
       "root_user_password": "origintrail"
+    },
+    "dataset_pruning": {
+      "enabled": false,
+      "imported_pruning_delay_in_minutes": 1440,
+      "replicated_pruning_delay_in_minutes": 1440
     }
   }
 }
diff --git a/migrations/202010201032128-update-events-add-blockchain-id.js b/migrations/202010201032128-update-events-add-blockchain-id.js
@@ -5,7 +5,11 @@ if (!process.env.NODE_ENV) {
     process.env.NODE_ENV = 'testnet';
 }
 
-const environmentConfig = global_config[process.env.NODE_ENV];
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202011261221123-update-offers-add-blockchain-id.js b/migrations/202011261221123-update-offers-add-blockchain-id.js
@@ -5,7 +5,11 @@ if (!process.env.NODE_ENV) {
     process.env.NODE_ENV = 'testnet';
 }
 
-const environmentConfig = global_config[process.env.NODE_ENV];
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202011261243123-update-bids-add-blockchain-id.js b/migrations/202011261243123-update-bids-add-blockchain-id.js
@@ -5,7 +5,11 @@ if (!process.env.NODE_ENV) {
     process.env.NODE_ENV = 'testnet';
 }
 
-const environmentConfig = global_config[process.env.NODE_ENV];
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202011261416123-update-data-info-extend-data-provider-wallet.js b/migrations/202011261416123-update-data-info-extend-data-provider-wallet.js
@@ -5,7 +5,11 @@ if (!process.env.NODE_ENV) {
     process.env.NODE_ENV = 'testnet';
 }
 
-const environmentConfig = global_config[process.env.NODE_ENV];
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202011301208123-update-data-sellers-add-blockchain-id.js b/migrations/202011301208123-update-data-sellers-add-blockchain-id.js
@@ -4,7 +4,12 @@ if (!process.env.NODE_ENV) {
     // Environment not set. Use the production.
     process.env.NODE_ENV = 'testnet';
 }
-const environmentConfig = global_config[process.env.NODE_ENV];
+
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202011301210123-update-data-trades-add-blockchain-id.js b/migrations/202011301210123-update-data-trades-add-blockchain-id.js
@@ -4,7 +4,12 @@ if (!process.env.NODE_ENV) {
     // Environment not set. Use the production.
     process.env.NODE_ENV = 'testnet';
 }
-const environmentConfig = global_config[process.env.NODE_ENV];
+
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202012090818123-update-purchased-data-add-blockchain-id.js b/migrations/202012090818123-update-purchased-data-add-blockchain-id.js
@@ -4,7 +4,12 @@ if (!process.env.NODE_ENV) {
     // Environment not set. Use the production.
     process.env.NODE_ENV = 'testnet';
 }
-const environmentConfig = global_config[process.env.NODE_ENV];
+
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202012151048123-update-challenges-add-blockchain-id.js b/migrations/202012151048123-update-challenges-add-blockchain-id.js
@@ -4,7 +4,12 @@ if (!process.env.NODE_ENV) {
     // Environment not set. Use the production.
     process.env.NODE_ENV = 'testnet';
 }
-const environmentConfig = global_config[process.env.NODE_ENV];
+
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202102120853043-update-commands-add-blockchain-id.js b/migrations/202102120853043-update-commands-add-blockchain-id.js
@@ -4,7 +4,12 @@ if (!process.env.NODE_ENV) {
     // Environment not set. Use the production.
     process.env.NODE_ENV = 'testnet';
 }
-const environmentConfig = global_config[process.env.NODE_ENV];
+
+const environment = process.env.NODE_ENV === 'mariner' ? 'mainnet' : process.env.NODE_ENV;
+if (['mainnet', 'testnet', 'development'].indexOf(environment) < 0) {
+    throw Error(`Unsupported node environment ${environment}`);
+}
+const environmentConfig = global_config[environment];
 const blockchain_id = environmentConfig.blockchain.implementations[0].network_id;
 
 module.exports = {

diff --git a/migrations/202104201448123-update-offers-add-replication-timestamp.js b/migrations/202104201448123-update-offers-add-replication-timestamp.js
@@ -0,0 +1,14 @@
+module.exports = {
+    up: async (queryInterface, Sequelize) => {
+        await queryInterface.addColumn(
+            'offers',
+            'replication_start_timestamp',
+            {
+                type: Sequelize.STRING,
+            },
+        );
+    },
+    down: async (queryInterface) => {
+        await queryInterface.removeColumn('offers', 'replication_start_timestamp');
+    },
+};
diff --git a/migrations/202104201448124-create-pruning-history.js b/migrations/202104201448124-create-pruning-history.js
@@ -0,0 +1,26 @@
+module.exports = {
+    up: async (queryInterface, Sequelize) => {
+        await queryInterface.createTable('pruning_history', {
+            id: {
+                allowNull: false,
+                primaryKey: true,
+                type: Sequelize.STRING,
+            },
+            data_set_id: {
+                allowNull: false,
+                type: Sequelize.STRING,
+            },
+            imported_timestamp: {
+                allowNull: false,
+                type: Sequelize.STRING,
+            },
+            pruned_timestamp: {
+                allowNull: false,
+                type: Sequelize.STRING,
+            },
+        });
+    },
+    down: async (queryInterface) => {
+        await queryInterface.dropTable('pruning_history');
+    },
+};
diff --git a/models/offers.js b/models/offers.js
@@ -28,6 +28,7 @@ module.exports = (sequelize, DataTypes) => {
         price_factor_used_for_price_calculation: DataTypes.INTEGER,
         offer_finalize_transaction_hash: DataTypes.STRING(128),
         blockchain_id: DataTypes.STRING,
+        replication_start_timestamp: DataTypes.STRING,
     }, {});
     offers.associate = (models) => {
     // associations can be defined here

diff --git a/models/pruning_history.js b/models/pruning_history.js
@@ -0,0 +1,21 @@
+const uuidv4 = require('uuid/v4');
+
+module.exports = (sequelize, DataTypes) => {
+    const pruning_history = sequelize.define('pruning_history', {
+        id: {
+            type: DataTypes.STRING,
+            defaultValue: () => uuidv4(),
+            primaryKey: true,
+        },
+        data_set_id: DataTypes.STRING,
+        imported_timestamp: DataTypes.STRING,
+        pruned_timestamp: DataTypes.STRING,
+    }, {
+        freezeTableName: true,
+        tableName: 'pruning_history',
+    });
+    pruning_history.associate = (models) => {
+        // associations can be defined here
+    };
+    return pruning_history;
+};
diff --git a/modules/Blockchain.js b/modules/Blockchain.js
@@ -561,26 +561,36 @@ class Blockchain {
         }
 
         const that = this;
+        let processingEvents = false;
+
         const handle = setInterval(async () => {
-            if (!that.appState.started) {
-                return;
-            }
+            try {
+                if (!that.appState.started) {
+                    return;
+                }
+                if (processingEvents) {
+                    return;
+                }
+                processingEvents = true;
 
-            const where = {
-                event,
-                finished: 0,
-                [Op.or]: blockStartConditions,
-            };
+                const where = {
+                    event,
+                    finished: 0,
+                    [Op.or]: blockStartConditions,
+                };
 
-            const eventData = await Models.events.findAll({ where });
-            if (eventData) {
-                eventData.forEach(async (data) => {
-                    const dataToSend = JSON.parse(data.dataValues.data);
-                    dataToSend.blockchain_id = data.dataValues.blockchain_id;
-                    this.emitter.emit(`eth-${data.event}`, dataToSend);
-                    data.finished = 1;
-                    await data.save();
-                });
+                const eventData = await Models.events.findAll({ where });
+                if (eventData) {
+                    eventData.forEach(async (data) => {
+                        const dataToSend = JSON.parse(data.dataValues.data);
+                        dataToSend.blockchain_id = data.dataValues.blockchain_id;
+                        this.emitter.emit(`eth-${data.event}`, dataToSend);
+                        data.finished = 1;
+                        await data.save();
+                    });
+                }
+            } finally {
+                processingEvents = false;
             }
         }, 2000);
 
@@ -704,25 +714,37 @@ class Blockchain {
     async handleReceivedEvents(events, contractName, blockchain_id) {
         for (let i = 0; events && i < events.length; i += 1) {
             const event = events[i];
-            const timestamp = Date.now();
+
             if (event.returnValues.DH_wallet) {
                 event.returnValues.DH_wallet = event.returnValues.DH_wallet.toLowerCase();
             }
-            /* eslint-disable-next-line */
-            await Models.events.create({
-                id: uuidv4(),
-                contract: contractName,
-                event: event.event,
-                data: JSON.stringify(event.returnValues),
-                data_set_id: Utilities.normalizeHex(event.returnValues.dataSetId),
-                block: event.blockNumber,
-                blockchain_id,
-                timestamp,
-                finished: 0,
+            const eventData = JSON.stringify(event.returnValues);
+            // eslint-disable-next-line no-await-in-loop
+            const databaseEvent = await Models.events.findOne({
+                where: {
+                    contract: contractName,
+                    event: event.event,
+                    data: eventData,
+                    block: event.blockNumber,
+                    blockchain_id,
+                },
             });
+            if (!databaseEvent) {
+                const timestamp = Date.now();
+                /* eslint-disable-next-line */
+                await Models.events.create({
+                    id: uuidv4(),
+                    contract: contractName,
+                    event: event.event,
+                    data: eventData,
+                    data_set_id: Utilities.normalizeHex(event.returnValues.dataSetId),
+                    block: event.blockNumber,
+                    blockchain_id,
+                    timestamp,
+                    finished: 0,
+                });
+            }
         }
-
-
         const twoWeeksAgo = new Date();
         twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14);
         // Delete old events