clean codes, add test case and make circuit more robust (#6)

README.md

@@ -1,5 +1,7 @@
 # zkmerkle-proof-of-solvency
+## Circuit Design
 
+See the [technical blog](https://gusty-radon-13b.notion.site/Proof-of-solvency-61414c3f7c1e46c5baec32b9491b2b3d) for more details about background and circuit design
 ## How to run
 
 ### Run third-party services
@@ -57,7 +59,7 @@ The `witness` service is used to generate witness for `prover` service.
 ```json
 {
   "PostgresDataSource" : "host=127.0.0.1 user=postgres password=zkpos@123 dbname=zkpos port=5432 sslmode=disable",
-  "UserDataFile": "/server/Result_11.csv",
+  "UserDataFile": "/server/data/20230118",
   "DbSuffix": "0",
   "TreeDB": {
     "Driver": "redis",
@@ -71,7 +73,7 @@ The `witness` service is used to generate witness for `prover` service.
 Where
 
 - `PostgresDataSource`: this is the postgres sql config;
-- `UserDataFile`: all users balance sheet file;
+- `UserDataFile`: the directory which contains all users balance sheet files;
 - `DbSuffix`: this suffix will be appended to the ending of table name, such as `proof0`, `witness0` table;
 - `TreeDB`:
   - `Driver`: `redis` means account tree use kvrocks as its storage engine;
@@ -131,7 +133,7 @@ The `userproof` service is used to generate and persist user merkle proof. It us
 ```json
 {
   "PostgresDataSource" : "host=127.0.0.1 user=postgres password=zkpos@123 dbname=zkpos port=5432 sslmode=disable",
-  "UserDataFile": "/server/Result_11.csv",
+  "UserDataFile": "/server/data/20230118",
   "DbSuffix": "0",
   "TreeDB": {
     "Driver": "redis",
@@ -145,7 +147,7 @@ The `userproof` service is used to generate and persist user merkle proof. It us
 Where
 
 - `PostgresDataSource`: this is the postgres sql config;
-- `UserDataFile`: all users balance sheet;
+- `UserDataFile`: the directory which contains all users balance sheet files;
 - `DbSuffix`: this suffix will be appended to the ending of table name, such as `proof0`, `witness0` table;
 - `TreeDB`:
   - `Driver`: `redis` means account tree use kvrocks as its storage engine;
@@ -179,7 +181,7 @@ Where
 - `ZkKeyName`: the key name generated by `keygen` service;
 - `CexAssetsInfo`: this is published by CEX, it represents CEX's liability;
 
-Run the following command to verify batch proof:
+You can get `CexAssetsInfo` using `dbtool` command after `witness` service run finished. Run the following command to verify batch proof:
 ```shell
 cd verifier; go run main.go
 ```
@@ -225,6 +227,11 @@ Run the following command to delete kvrocks data and postgresql:
 cd src/dbtool; go run main.go -delete_all
 ```
 
+Run the following command to get cex assets info in json format:
+```shell
+cd src/dbtool; go run main.go -query_cex_assets
+```
+
 ### Check data correctness
 
 #### check account tree construct correctness

circuit/batch_create_user_circuit.go

@@ -102,6 +102,8 @@ func (b BatchCreateUserCircuit) Define(api API) error {
 	}
 
 	for j := 0; j < len(tempAfterCexAssets); j++ {
+		CheckValueInRange(api, afterCexAssets[j].TotalEquity)
+		CheckValueInRange(api, afterCexAssets[j].TotalDebt)
 		tempAfterCexAssets[j] = api.Add(api.Mul(afterCexAssets[j].TotalEquity, utils.Uint64MaxValueFrSquare),
 			api.Mul(afterCexAssets[j].TotalDebt, utils.Uint64MaxValueFr), afterCexAssets[j].BasePrice)
 	}

go.mod

@@ -10,7 +10,9 @@ require (
 	github.com/consensys/gnark v0.7.0
 	github.com/consensys/gnark-crypto v0.7.0
 	github.com/go-redis/redis/v8 v8.11.5
+	github.com/gocarina/gocsv v0.0.0-20230123225133-763e25b40669
 	github.com/shopspring/decimal v1.3.1
+	github.com/stretchr/testify v1.8.0
 	github.com/zeromicro/go-zero v1.4.2
 	gorm.io/driver/postgres v1.4.5
 	gorm.io/gorm v1.24.2
@@ -29,13 +31,13 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/ethereum/go-ethereum v1.10.26 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/gocarina/gocsv v0.0.0-20230123225133-763e25b40669 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20221011183528-d4900dc688bf // indirect
@@ -58,6 +60,7 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
@@ -83,6 +86,7 @@ require (
 	google.golang.org/grpc v1.50.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 replace (

src/prover/prover/prover.go

@@ -93,7 +93,6 @@ func NewProver(config *config.Config) *Prover {
 }
 
 func (p *Prover) Run(flag bool) {
-	// TODO remove
 	p.proofModel.CreateProofTable()
 	batchWitnessFetch := func() (*witness.BatchWitness, error) {
 		lock := utils.GetRedisLockByKey(p.redisConn, utils.RedisLockKey)

src/userproof/main.go

@@ -126,7 +126,6 @@ func main() {
 		ComputeAccountRootHash(userProofConfig)
 		return
 	}
-	// ComputeAccountRootHash(userProofConfig)
 	accountTree, err := utils.NewAccountTree(userProofConfig.TreeDB.Driver, userProofConfig.TreeDB.Option.Addr)
 	accounts := HandleUserData(userProofConfig)
 	fmt.Println("num", len(accounts))
@@ -139,7 +138,6 @@ func main() {
 		latestAccountIndex += 1
 	}
 	accountTreeRoot := hex.EncodeToString(accountTree.Root())
-	// proofs := make([]model.UserProof, 1)
 	jobs := make(chan Job, 1000)
 	nums := make(chan int, 1)
 	results := make(chan *model.UserProof, 1000)

src/utils/utils.go

@@ -23,10 +23,6 @@ import (
 
 func ConvertAssetInfoToBytes(value any) []byte {
 	switch t := value.(type) {
-	//case AccountAsset:
-	//	equityBigInt := new(big.Int).SetUint64(t.Equity)
-	//	debtBigInt := new(big.Int).SetUint64(t.Debt)
-	//	return new(big.Int).Add(new(big.Int).Mul(equityBigInt, Uint64MaxValueBigInt), debtBigInt).Bytes()
 	case CexAssetInfo:
 		equityBigInt := new(big.Int).SetUint64(t.TotalEquity)
 		debtBigInt := new(big.Int).SetUint64(t.TotalDebt)
@@ -184,8 +180,6 @@ func ReadUserDataFromCsvFile(name string) ([]AccountInfo, []CexAssetInfo, error)
 	defer f.Close()
 	csvReader := csv.NewReader(f)
 	data, err := csvReader.ReadAll()
-	//fmt.Println(data[0])
-	//fmt.Println(data[1])
 	accountIndex := 0
 	cexAssetsInfo := make([]CexAssetInfo, AssetCounts)
 	accounts := make([]AccountInfo, len(data)-1)
@@ -256,7 +250,6 @@ func ReadUserDataFromCsvFile(name string) ([]AccountInfo, []CexAssetInfo, error)
 		}
 
 		account.Assets = assets
-		// AccountStatistics[len(assets)] += 1
 		if account.TotalEquity.Cmp(account.TotalDebt) >= 0 {
 			accounts[accountIndex] = account
 			accountIndex += 1
@@ -279,7 +272,6 @@ func ConvertFloatStrToUint64(f string, multiplier int64) (uint64, error) {
 		return 0, nil
 	}
 	numFloat, err := decimal.NewFromString(f)
-	// equityFloat, err := strconv.ParseFloat(data[i][j*3+1], 64)
 	if err != nil {
 		return 0, err
 	}
@@ -294,7 +286,6 @@ func ConvertFloatStrToUint64(f string, multiplier int64) (uint64, error) {
 
 func DecodeBatchWitness(data string) *BatchCreateUserWitness {
 	var witnessForCircuit BatchCreateUserWitness
-	// err = json.Unmarshal([]byte(batchWitness.WitnessData), &witnessForCircuit)
 	b, err := base64.StdEncoding.DecodeString(data)
 	if err != nil {
 		fmt.Println("deserialize batch witness failed: ", err.Error())
@@ -319,26 +310,6 @@ func DecodeBatchWitness(data string) *BatchCreateUserWitness {
 }
 
 func AccountInfoToHash(account *AccountInfo, hasher *hash.Hash) []byte {
-	//zeroByte := []byte{0}
-	//startAssetIndex := 0
-	//for p := 0; p < len(account.Assets); p++ {
-	//	if p != 0 {
-	//		startAssetIndex = int(account.Assets[p-1].Index) + 1
-	//	}
-	//	for assetIndex := startAssetIndex; assetIndex < int(account.Assets[p].Index); assetIndex++ {
-	//		(*hasher).Write(zeroByte)
-	//	}
-	//	commitment := ConvertAssetInfoToBytes(account.Assets[p])
-	//	(*hasher).Write(commitment)
-	//}
-	//
-	//if len(account.Assets) != 0 {
-	//	startAssetIndex = int(account.Assets[len(account.Assets)-1].Index) + 1
-	//}
-	//for p := startAssetIndex; p < AssetCounts; p++ {
-	//	(*hasher).Write(zeroByte)
-	//}
-	//assetCommitment := (*hasher).Sum(nil)
 	assetCommitment := ComputeUserAssetsCommitment(hasher, account.Assets)
 	(*hasher).Reset()
 	// compute new account leaf node hash