diff --git a/app/env.js b/app/env.js index df52c45..46a45e4 100644 --- a/app/env.js +++ b/app/env.js @@ -30,6 +30,8 @@ const vars = envalid.cleanEnv( LOG_LEVEL: envalid.str({ default: 'info', devDefault: 'debug' }), PORT: envalid.port({ default: 80, devDefault: 3000 }), NODE_HOST: envalid.host({ devDefault: 'localhost' }), + IPFS_API_HOST: envalid.host({ devDefault: 'localhost' }), + IPFS_API_PORT: envalid.port({ default: 5001 }), NODE_PORT: envalid.port({ default: 9944 }), IPFS_PATH: envalid.str({ default: '/ipfs', devDefault: path.resolve(__dirname, '..', `data`) }), IPFS_EXECUTABLE: envalid.str({ diff --git a/app/utils/ServiceWatcher.js b/app/utils/ServiceWatcher.js index a2f100a..c8bcdf9 100644 --- a/app/utils/ServiceWatcher.js +++ b/app/utils/ServiceWatcher.js @@ -1,3 +1,6 @@ +import * as client from 'prom-client' +import axios from 'axios' + import { TimeoutError } from './Errors.js' import env from '../env.js' @@ -9,8 +12,20 @@ class ServiceWatcher { constructor(apis) { this.report = {} this.#pollPeriod = env.HEALTHCHECK_POLL_PERIOD_MS + this.ipfsApiUrl = `http://${env.IPFS_API_HOST}:${env.IPFS_API_PORT}/api/v0/` this.#timeout = env.HEALTHCHECK_TIMEOUT_MS this.services = this.#init(apis) + this.metrics = { + peerCount: () => { + if (!this.metrics.peerCount) { + return new client.Gauge({ + name: 'dscp_ipfs_swarm_peer_count', + help: 'a number of discovered and connected peers', + labelNames: ['type'], + }) + } + }, + } } delay(ms, service = false) { @@ -44,6 +59,21 @@ class ServiceWatcher { .filter(Boolean) } + async #updateMetrics() { + const { data: connectedPeers } = await axios({ + url: `${this.ipfsApiUrl}swarm/peers`, + method: 'POST', + }) + const { data: discoveredPeers } = await axios({ + url: `${this.ipfsApiUrl}swarm/addrs`, + method: 'POST', + }) + + // update instance's metrics object + this.metrics.peerCount.set({ type: 'discovered' }, Object.keys(discoveredPeers.Addrs).length) + this.metrics.peerCount.set({ type: 'connected' }, connectedPeers.Peers?.length || 0) + } + // starts the generator resolving after the first update // use ServiceWatcher.gen.return() to stop async start() { @@ -54,6 +84,7 @@ class ServiceWatcher { try { const services = await getAll services.forEach(({ name, ...rest }) => this.update(name, rest)) + await this.#updateMetrics().catch((err) => (this.metrics.error = err)) } catch (error) { // if no service assume that this is server error e.g. TypeError, Parse... const name = error.service || 'server' diff --git a/helm/dscp-ipfs/Chart.yaml b/helm/dscp-ipfs/Chart.yaml index 91f68d1..8273f19 100644 --- a/helm/dscp-ipfs/Chart.yaml +++ b/helm/dscp-ipfs/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: dscp-ipfs -appVersion: '2.8.2' +appVersion: '2.9.0' description: A Helm chart for dscp-ipfs -version: '2.8.2' +version: '2.9.0' type: application dependencies: - name: dscp-node diff --git a/helm/dscp-ipfs/templates/statefulset.yaml b/helm/dscp-ipfs/templates/statefulset.yaml index e78b563..c522612 100644 --- a/helm/dscp-ipfs/templates/statefulset.yaml +++ b/helm/dscp-ipfs/templates/statefulset.yaml @@ -98,6 +98,11 @@ spec: configMapKeyRef: name: {{ include "dscp-ipfs.fullname" . }}-config key: healthCheckPort + - name: IPFS_API_PORT + valueFrom: + configMapKeyRef: + name: {{ include "dscp-ipfs.fullname" . }}-config + key: ipfsApiPort - name: LOG_LEVEL valueFrom: configMapKeyRef: diff --git a/helm/dscp-ipfs/values.yaml b/helm/dscp-ipfs/values.yaml index c2d04cd..318cd85 100644 --- a/helm/dscp-ipfs/values.yaml +++ b/helm/dscp-ipfs/values.yaml @@ -52,7 +52,7 @@ statefulSet: image: repository: digicatapult/dscp-ipfs pullPolicy: IfNotPresent - tag: 'v2.8.2' + tag: 'v2.9.0' storage: storageClass: "" diff --git a/package-lock.json b/package-lock.json index dcbb115..9aa7711 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@digicatapult/dscp-ipfs", - "version": "2.8.2", + "version": "2.9.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@digicatapult/dscp-ipfs", - "version": "2.8.2", + "version": "2.9.0", "license": "Apache-2.0", "dependencies": { "@polkadot/api": "^9.9.4", + "axios": "^1.2.0", "dotenv": "^16.0.3", "envalid": "^7.3.1", "express": "^4.18.2", @@ -29,7 +30,7 @@ "formdata-node": "^5.0.0", "go-ipfs": "^0.16.0", "mocha": "^10.1.0", - "node-fetch": "^3.2.10", + "node-fetch": "^3.3.0", "nodemon": "^2.0.20", "nyc": "^15.1.0", "pino-colada": "^2.2.2", @@ -1621,6 +1622,29 @@ "node": ">=8.0.0" } }, + "node_modules/axios": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.0.tgz", + "integrity": "sha512-zT7wZyNYu3N5Bu0wuZ6QccIf93Qk1eV8LOewxgjOZFd2DenOs98cJ7+Y6703d0wkaXGY6/nZd4EweJaHz9uzQw==", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -3330,6 +3354,25 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -5936,6 +5979,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", @@ -8503,6 +8551,28 @@ "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==" }, + "axios": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.2.0.tgz", + "integrity": "sha512-zT7wZyNYu3N5Bu0wuZ6QccIf93Qk1eV8LOewxgjOZFd2DenOs98cJ7+Y6703d0wkaXGY6/nZd4EweJaHz9uzQw==", + "requires": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -9790,6 +9860,11 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + }, "foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -11723,6 +11798,11 @@ "ipaddr.js": "1.9.1" } }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "pstree.remy": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", diff --git a/package.json b/package.json index cedd326..6e41d70 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@digicatapult/dscp-ipfs", - "version": "2.8.2", + "version": "2.9.0", "description": "Service for DSCP", "main": "app/index.js", "type": "module", @@ -8,7 +8,7 @@ "test": "NODE_ENV=test mocha --config ./test/mocharc.cjs ./test", "test:integration": "NODE_ENV=test mocha --config ./test/mocharc.cjs ./test/integration", "test:unit": "NODE_ENV=test mocha --config ./test/mocharc-unit.cjs ./test/unit", - "lint": "eslint .", + "lint": "eslint . --fix", "depcheck": "depcheck", "start": "node app/index.js", "dev": "NODE_ENV=dev nodemon app/index.js | pino-colada", @@ -33,6 +33,7 @@ "homepage": "https://github.com/digicatapult/dscp-ipfs#readme", "dependencies": { "@polkadot/api": "^9.9.4", + "axios": "^1.2.0", "dotenv": "^16.0.3", "envalid": "^7.3.1", "express": "^4.18.2", @@ -52,7 +53,7 @@ "formdata-node": "^5.0.0", "go-ipfs": "^0.16.0", "mocha": "^10.1.0", - "node-fetch": "^3.2.10", + "node-fetch": "^3.3.0", "nodemon": "^2.0.20", "nyc": "^15.1.0", "pino-colada": "^2.2.2", diff --git a/test/integration/fixtures.js b/test/integration/fixtures.js index 1fa4626..960a040 100644 --- a/test/integration/fixtures.js +++ b/test/integration/fixtures.js @@ -17,7 +17,7 @@ export const mochaGlobalSetup = async function () { this.stopServer = stopServer await startServer(this) - await waitForIpfsApi(`5001`) + await waitForIpfsApi(env.IPFS_API_PORT) } export const mochaGlobalTeardown = async function () { diff --git a/test/integration/ipfs.test.js b/test/integration/ipfs.test.js index db83544..c5c315d 100644 --- a/test/integration/ipfs.test.js +++ b/test/integration/ipfs.test.js @@ -4,13 +4,14 @@ import { FormData, Blob } from 'formdata-node' import { expect } from 'chai' import delay from 'delay' +import env from '../../app/env.js' import { getSwarmKey, setSwarmKey } from './helper/api.js' import { setupIPFS, waitForIpfsApi } from './helper/ipfs.js' const uploadA = async (fileName, contents) => { const form = new FormData() form.append('file', new Blob([contents]), fileName) - const body = await fetch(`http://localhost:5001/api/v0/add?cid-version=0`, { + const body = await fetch(`http://localhost:${env.IPFS_API_PORT}/api/v0/add?cid-version=0`, { method: 'POST', body: form, }) @@ -113,7 +114,7 @@ describe('ipfs', function () { context.hash = await uploadA('test-file-4.txt', 'Test 4') await setSwarmKey(context.swarmKey) await delay(500) - await waitForIpfsApi(`5001`) + await waitForIpfsApi(env.IPFS_API_PORT) }) it('should be retrievable', async function () { diff --git a/test/test.env b/test/test.env index b0cda9b..c6a5331 100644 --- a/test/test.env +++ b/test/test.env @@ -3,3 +3,5 @@ IPFS_LOG_LEVEL=fatal NODE_HOST=localhost HEALTHCHECK_POLL_PERIOD_MS=1000 HEALTHCHECK_TIMEOUT_MS=1000 +IPFS_API_HOST=localhost +IPFS_API_PORT=5001