Merge pull request #674 from net-zero-project/main

Update O&G data loader

.env.SAMPLE

@@ -45,9 +45,9 @@ LEDGER_FIRST_BLOCK="0"
 #LEDGER_ETH_JSON_RPC_URL="https://goerli.infura.io/v3/<your-infura-project-id>"
 #LEDGER_ETH_WS_URL="wss://goerli.infura.io/ws/v3/<your-infura-project-id>"
 #LEDGER_ETH_NETWORK="goerli"
-#LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS="0xE8854ec567b3762046d8f773547f373e6A1A620A"
+#LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS="0x118938E52100df8FfE8793f8ec231f56d85e8E72"
 #LEDGER_FIRST_BLOCK="7385471"
-#LEDGER_CARBON_TRACKER_ADDRESS="0xDb80143bCf04B53Ed586b7f806C25ff275b5Cc5b"
+#LEDGER_CARBON_TRACKER_ADDRESS="0x039EF5c92CC8249D2cb3785B68bc4F79dC8a7625"
 
 # it should be 2048 for Avalanche testnet and could be 5000 for bsc testnet.  2048 seems to work for Goerli
 LEDGER_EVENTS_BLOCK_INTERVAL="2048"

app/api-server/controller/synchronizer.ts

@@ -305,7 +305,7 @@ const getNumOfProductTokens = async (opts: OPTS_TYPE): Promise<number> => {
         return result;
     } catch (err) {
         console.error(err)
-        throw new Error('Error in _numOfProducts: ' + err)
+        throw new Error('Error in getNumOfProductTokens: ' + err)
     }
 }
 

app/api-server/server.ts

@@ -26,7 +26,7 @@ const assertEnv = (key: string): string => {
   return process.env[key] || '';
 }
 const contract_address = assertEnv('LEDGER_EMISSION_TOKEN_CONTRACT_ADDRESS')
-const tracker_address = process.env['LEDGER_CARBON_TRACKER_ADDRESS']
+const tracker_address = assertEnv('LEDGER_CARBON_TRACKER_ADDRESS')
 const network_name = assertEnv('LEDGER_ETH_NETWORK')
 const network_rpc_url = assertEnv('LEDGER_ETH_JSON_RPC_URL')
 const network_ws_url = process.env['LEDGER_ETH_WS_URL']

app/frontend/contracts/src/addresses.ts

@@ -21,6 +21,24 @@ const networksAndAddresses = {
     }
   },
 
+  hedera_testnet: {
+    network: "Hedera Testnet",
+    tokenNetwork: {
+      address: "0x00000000000000000000000000000000003be83f", 
+    },
+    carbonTracker: {
+      address: "0x00000000000000000000000000000000003c0064",
+    },
+    dao: {
+      governor: {
+        address: "",
+      },
+      daoToken: {
+        address: "",
+      }
+    }
+  },
+
   goerli: {
     network: "Goerli Testnet",
     tokenNetwork: {
@@ -34,10 +52,10 @@ const networksAndAddresses = {
     },
     dao: {
       governor: {
-        address: "0x9F4590684d1DD950dF65Dc71D39f782bc3af317A",
+        address: "0x6875735ee7864673B211eC2ec2a534179F863DAD",
       },
       daoToken: {
-        address: "0x67E5b82809f28aA321818b90bF5c603D954f87C8",
+        address: "0x08AA8Ae1E7A087b591a5a1234fBE20e592c5c417",
       }
     }
   },
@@ -152,6 +170,6 @@ const networksAndAddresses = {
 }
 
 // change this to the network you want
-const addresses = networksAndAddresses.hardhat;
+const addresses = networksAndAddresses.hedera_testnet;
 
 export default addresses;

app/frontend/react-app/src/components/tracker-info-modal.tsx

@@ -107,12 +107,11 @@ const TrackerInfoModal:FC<TrackerInfoModalProps> = ({provider,show,tracker,onHid
             : null)}
             <tr>
               <td>From date</td>
-              {/*TO-DO infer fromDate/thruDate from Net Emission tokens*/}
-              <td><DisplayDate date={tracker.fromDate}/></td>
+              <td><DisplayDate date={Math.min.apply(Math,tracker?.tokens?.map(e=>{return e?.token?.thruDate!}) as number[])}/></td>
             </tr>
             <tr>
               <td>Thru date</td>
-              <td><DisplayDate date={tracker.thruDate}/></td>
+              <td><DisplayDate date={Math.min.apply(Math,tracker?.tokens?.map(e=>{return e?.token?.thruDate!}) as number[])}/></td>
             </tr>
             <tr>
               <td>Description</td>

app/methane/client/src/components/static-data.tsx

@@ -1,6 +1,7 @@
 import type { Field } from "@blockchain-carbon-accounting/react-app/src/components/static-data"
 export type Asset = {
   name?: string
+  operator?: string
   latitude: number
   longitude: number
   division_type?: string
@@ -77,6 +78,12 @@ export const ASSET_FIELDS: Field[] = [
     type: 'string',
     ops: ['eq']
 },
+{
+    alias: 'Operator',
+    name: 'operator',
+    type: 'string',
+    ops: ['like']
+},
 ]
 
 export const PRODUCT_FIELDS: Field[] = [

app/methane/client/src/pages/assets.tsx

@@ -182,7 +182,7 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
         <div className="mt-4">
           <h2 style={{display: 'inline'}}>
             Operator: {operator?.name}&nbsp;
-            {assetCount} Assets&nbsp;
+            {operator?.assetsCount} ({assetCount}) Assets &nbsp;
           </h2>
           &nbsp;
           <Button className="mb-3" onClick={switchQueryBuilder} variant={(showQueryBuilder) ? 'dark' : 'outline-dark'}><BsFunnel /></Button>
@@ -197,6 +197,7 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
             <thead>
               <tr>
                 <th>Name</th>
+                <th>Operator</th>
                 <th>State</th>
                 <th>Status</th>
                 <th></th>
@@ -206,7 +207,8 @@ const RegisteredOperator: ForwardRefRenderFunction<AssetsHandle, AssetProps> = (
               {!!selectedAssets &&
                 selectedAssets.map((asset,index) => (
                   <tr key={[asset?.name,index].join('_')} onClick={() => handleOpenOperatorInfoModal(asset)} onMouseOver={pointerHover}>
-                    <td> {asset.name}</td>
+                    <td>{asset.name}</td>
+                    <td>{asset?.operator}</td>
                     <td>{asset?.division_name}</td>
                     <td>{asset?.status}</td>
                     <td>

data/loadOGdata.sh

@@ -1,4 +1,6 @@
-npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"
+#npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"
+#npm run dataLoader set_asset_operators
+
 npm run dataLoader load_product_data "./oil_and_gas/files/2022-og-benchmarking-report-data.xlsx" "Company_basin" -- --format Benchmark --source "https://www.sustainability.com/globalassets/sustainability.com/thinking/pdfs/2022/2022-og-benchmarking-report-data.xlsx" --skip_rows 7
 
 npm run dataLoader load_product_data "./oil_and_gas/files/VIIRS_Global_flaring_d.7_slope_0.029353_2021_web.xlsx" "flare upstream" -- --format VIIRS --year 2021 --source "https://eogdata.mines.edu/global_flare_data/VIIRS_Global_flaring_d.7_slope_0.029353_2021_web.xlsx"

data/oil_and_gas/README.md

@@ -9,7 +9,24 @@ Or you can download the data
 sh download.sh
 ```
 
-Once you have the data, load them into postgres db (blockchain-carbon-accounting) by running the following from `/data`:
+### Oil & gas asset and operator data.
+
+The oil & gas asset data includes >1.5 million assets.
+To avoid reloading the data into postres `download.sh` returns 3 binary files that can be used to restore the following postrgres tables (after initializing the DB with `npm run pg:init`)
+
+1. oil_and_gas_asset
+2. operator
+3. asset_operator
+
+Otherwise the tables can be loaded using the following commands from `\data`.
+
+``` npm run dataLoader load_og_assets "./oil_and_gas/files/Oil_and_Natural_Gas_Wells.geojson" -- --format US_asset_data --source "https://hifld-geoplatform.opendata.arcgis.com/datasets/geoplatform::oil-and-natural-gas-wells/explore"```
+
+```npm run dataLoader set_asset_operators```
+
+### Oil & gas product & emissions data
+
+Load the product data into postgres db by running the following from `/data` directorty:
 ```
 sh loadOGdata.sh
 ```

data/oil_and_gas/download.sh

@@ -9,6 +9,11 @@ curl -L -c cookies.txt 'https://docs.google.com/uc?export=download&id='$fileId \
 curl -L -b cookies.txt -o $fileName \
      'https://docs.google.com/uc?export=download&id='$fileId'&confirm='$(<confirm.txt)
 rm -f confirm.txt cookies.txt
+echo "Downloading postgres table backup..."
+curl \
+-L -o 'asset_operator' 'https://docs.google.com/uc?export=download&id=1ME0XIz065OEMReqfvsVOl7oW5ye2BmUl' \
+-L -o 'operator' 'https://docs.google.com/uc?export=download&id=1MHKrkMvq9FZCw_RYsiIrXi1rRmfLmSD7' \
+-L -o 'oil_and_gas_asset' 'https://docs.google.com/uc?export=download&id=1VPYAKasx3SJ9l3wuW99CMMFHEGU-4Dfr'
 
 echo "Downloading oil and gas product data from google drive..."
 echo "VIIRS flaring data..."

data/src/dataLoader.ts

@@ -4,6 +4,7 @@ import {
 } from "@blockchain-carbon-accounting/data-common"
 import { 
   importOilAndGasAssets, 
+  setAssetOperators,
   importProductData,
   updateProductDates 
 } from "@blockchain-carbon-accounting/oil-and-gas-data-lib"
@@ -155,9 +156,28 @@ const progressBar = new SingleBar(
           const count = await db.getOilAndGasAssetRepo().count([])
           console.log(`=== Done, we now have ${count} OilAndGasAssets in the DB`)
           await db.close()
+
         })
       }
-
+    }
+  )
+  .command(
+    "set_asset_operators",
+    "assign operator to asset by operator name in oil_and_gas asset table",
+    // eslint-disable-next-line
+    (yargs: any) => {
+      yargs
+    },
+    // eslint-disable-next-line
+    async (argv: any) => {
+      console.log("=== Starting set_asset_operators ...")
+      const db = await init(parseCommonYargsOptions(argv))
+      await setAssetOperators(
+        argv, progressBar, 
+        db.getOperatorRepo(),
+        db.getOilAndGasAssetRepo(),
+        db.getAssetOperatorRepo(),
+        "0xf3AF07FdA6F11b55e60AB3574B3947e54DebADf7")
     }
   )
   .command(
@@ -220,7 +240,8 @@ const progressBar = new SingleBar(
       // generate the ETH wallet
       //const newAccount = ethers.Wallet.createRandom();
       const name: string | undefined = 'Operator Repository';
-      const wallet = await db.getWalletRepo().insertWallet({
+      let wallet = await db.getWalletRepo().selectWallet("0xbDA5747bFD65F08deb54cb465eB87D40e51B197E")
+      if(!wallet) {wallet = await db.getWalletRepo().insertWallet({
         email: "[email protected]",
         organization: 'Two Ravens Energy & Climate Consulting Ltd.',
         address: "0xbDA5747bFD65F08deb54cb465eB87D40e51B197E",
@@ -232,7 +253,7 @@ const progressBar = new SingleBar(
         verification_token_sent_at: new Date(),
         email_verified: true,
         name,
-      })
+      })}
       await importProductData(argv, progressBar, 
         db.getProductRepo(),
         db.getOperatorRepo(),

data/src/models/assetOperator.ts

@@ -3,6 +3,7 @@ import type {
 } from '@blockchain-carbon-accounting/oil-and-gas-data-lib';
 import {
   Column, Entity,
+  PrimaryColumn,
   PrimaryGeneratedColumn,
   ManyToOne,
   JoinColumn,
@@ -14,9 +15,9 @@ import { OilAndGasAsset } from './oilAndGasAsset';
 import { Operator } from './operator';
 
 @Entity({name: 'asset_operator'})
-@Unique(['asset', 'operator', 'from_date' ])
-@Unique(['asset', 'operator', 'thru_date' ])
-@Index(["asset"], { unique: true, where: `"thru_date" IS NULL` })
+@Unique(['assetUuid', 'operatorUuid', 'from_date' ])
+@Unique(['assetUuid', 'operatorUuid', 'thru_date' ])
+@Index(['assetUuid', 'operatorUuid'], { unique: true, where: `"thru_date" IS NULL` })
 @Check('"share" >= 0')
 @Check('"share" <= 1')
 export class AssetOperator implements AssetOperatorInterface {
@@ -27,14 +28,14 @@ export class AssetOperator implements AssetOperatorInterface {
   @Column()
   class!: string;
 
-  @Column()
+  @PrimaryColumn()
   assetUuid!: string;
 
   @ManyToOne(() => OilAndGasAsset, (asset) => asset.asset_operators)
   @JoinColumn({name: 'assetUuid'})
   asset!: OilAndGasAsset;
 
-  @Column()
+  @PrimaryColumn()
   operatorUuid!: string;
 
   @ManyToOne(() => Operator, (operator) => operator.asset_operators)

data/src/models/productToken.ts

@@ -1,6 +1,7 @@
 import {
     Entity,
     PrimaryColumn,
+    PrimaryGeneratedColumn,
     Column,
     ManyToOne,
     //ManyToMany,
@@ -14,6 +15,10 @@ import { Tracker } from './tracker';
 import { TrackedProduct } from './trackedProduct';
 @Entity()
 export class ProductToken {
+
+    //@PrimaryGeneratedColumn("uuid")
+    //uuid!: number;
+
     @PrimaryColumn()
     productId!: number;
 

data/src/repositories/assetOperator.repo.ts

@@ -17,13 +17,23 @@ export class AssetOperatorRepo implements AssetOperatorDbInterface{
     this._db = dbConnection
   }
 
+  public findByPrimaryColumns = async (assetUuid: string, operatorUuid: string): Promise<AssetOperatorInterface | null> => {
+    return await this._db.getRepository(AssetOperator)
+      .findOneBy({assetUuid,operatorUuid})
+  }
   public putAssetOperator = async (doc: AssetOperatorInterface) => {
     try{
-      const repo = this._db.getRepository(AssetOperator)  
+      const repo = this._db.getRepository(AssetOperator) 
+      const assetOperator = await this.findByPrimaryColumns(doc.assetUuid,doc.operatorUuid) 
+      //increment asset_count of operator if unique assetOperator has not been created yet
+      if(!assetOperator){
+        if(!doc.operator.asset_count){doc.operator.asset_count=0}
+        doc.operator.asset_count += 1
+        await this._db.getRepository(Operator).save(doc.operator)
+      }else{
+        doc.uuid=assetOperator.uuid
+      }
       await repo.save(doc)
-      //increment asset_count of operator is newOperator saved
-      if(doc.operator.asset_count){doc.operator.asset_count += 1}
-      await this._db.getRepository(Operator).save(doc.operator)
     }catch(error){
       throw new Error(`Cannot create asset_operator relation:: ${error}`)       
     }
@@ -33,10 +43,24 @@ export class AssetOperatorRepo implements AssetOperatorDbInterface{
     return await this._db.getRepository(AssetOperator).find()
   }
 
+
+
   public countAssetOwners = async (): Promise<number> => {
     return await this._db.getRepository(AssetOperator).count()
   }
 
+  public select = async (
+    bundles: Array<QueryBundle>,
+  ): Promise<Array<AssetOperatorInterface>> => {
+    let selectBuilder: SelectQueryBuilder<AssetOperator> = await this._db.getRepository(AssetOperator).createQueryBuilder("asset_operator")
+    // category by issuer address
+    selectBuilder = buildQueries('asset_operator', selectBuilder, bundles 
+        ,[AssetOperator,OilAndGasAsset])
+    return selectBuilder    
+      .innerJoin("asset_operator.asset", "oil_and_gas_asset")
+      .getMany();
+  }
+
   public selectAssetsPaginated = async (
     offset: number, 
     limit: number, 

data/src/repositories/common.ts

@@ -182,7 +182,7 @@ export function buildQueries(table: string,builder: SelectQueryBuilder<any>,quer
     // make case insensitive for issuee issuer cases
     let cond = '';
     if(query.op == "like" || query.field == 'issuedTo' || query.field == 'issuedBy' || query.field == 'issuedFrom') {
-      cond = `LOWER(${alias}.${query.field}) ${query.op} LOWER(:${query.field})`
+      cond = `LOWER(${alias}.${query.field}) ${query.op} LOWER(:${query_field_label})`
     }else if(query.op == "vector"){
       cond = `to_tsvector(${alias}.${query.field}) @@ to_tsquery(query.value)`
     } else {

data/src/repositories/oilAndGasAsset.repo.ts

@@ -33,8 +33,11 @@ export class OilAndGasAssetRepo implements OilAndGasAssetDbInterface {
   ): Promise<Array<OilAndGasAssetInterface>> => {
     let selectBuilder: SelectQueryBuilder<OilAndGasAsset> = await this._db.getRepository(OilAndGasAsset).createQueryBuilder("oil_and_gas_asset")
     // category by issuer address
-    selectBuilder = buildQueries('oil_and_gas_asset', selectBuilder, bundles)
-    return selectBuilder.getMany();
+    selectBuilder = buildQueries('oil_and_gas_asset', selectBuilder, bundles 
+        )//,[OilAndGasAsset, AssetOperator])
+    return selectBuilder    
+      //.innerJoin("oil_and_gas_asset.asset_operators", "asset_operator")
+      .getMany();
   }
 
   public selectPaginated = async (
@@ -50,7 +53,7 @@ export class OilAndGasAssetRepo implements OilAndGasAssetDbInterface {
       .limit(limit)
       .offset(offset)
       .innerJoin("oil_and_gas_asset.asset_operators", "asset_operator")
-      .orderBy('oil_and_gas_asset.name', 'ASC')
+      .orderBy('oil_and_gas_asset.operator', 'ASC')
       .getMany();
   }
 

data/src/repositories/operator.repo.ts

@@ -26,7 +26,11 @@ export class OperatorRepo implements OperatorDbInterface {
   }
 
   public findByName = async (name: string): Promise<OperatorInterface | null> => {
-    return await this._db.getRepository(Operator).findOneBy({name})
+    return await this._db.getRepository(Operator).createQueryBuilder('operator')
+      .where('operator.name=:name', { name })
+      //.leftJoin('operator.asset_operators', 'asset_operators')
+      .loadRelationCountAndMap("operator.assetsCount", "operator.asset_operators")
+      .getOne()
   }
 
   public getOperators = async (): Promise<OperatorInterface[]> => {