Merge remote-tracking branch 'origin/main' into chore_logger

packages/sdk-socket-server-next/.env.sample

@@ -1,4 +1,10 @@
 SEGMENT_API_KEY_PRODUCTION=123456789
 SEGMENT_API_KEY_DEBUG=123456789
+# REDIS_URL=redis://redis:6379/0
+# Example REDIS_NODES format: "redis://host1:6379,redis://host2:6379"
+# REDIS_NODES=redis://localhost:6380,redis://localhost:6381,redis://localhost:6382
 REDIS_URL=redis://redis:6379/0
 RATE_LIMITER=false
+RATE_LIMITER_HTTP_WINDOW_MINUTE=1
+RATE_LIMITER_HTTP_LIMIT=100000
+ADMIN_UI=false

packages/sdk-socket-server-next/docker-compose.yml

@@ -1,6 +1,16 @@
 version: '3.9'
 
 services:
+
+  app:
+    image: node:latest
+    volumes:
+      - ./:/usr/src/app
+    working_dir: /usr/src/app
+    command: yarn debug:redis
+    # ports:
+    #   - "4000:4000"
+
   app1:
     build:
       context: .
@@ -40,8 +50,57 @@ services:
     depends_on:
       - cache
 
+  redis-master1:
+    # build:
+    #   context: .
+    #   dockerfile: Dockerfile.redis
+    image: redis:7.2-alpine
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6380
+    environment:
+      - REDIS_ROLE=master
+    ports:
+      - "6380:6380"
+
+  redis-master2:
+    image: redis:7.2-alpine
+    # build:
+    #   context: .
+    #   dockerfile: Dockerfile.redis
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6381
+    environment:
+      - REDIS_ROLE=master
+    ports:
+      - "6381:6381"
+
+  redis-master3:
+    image: redis:7.2-alpine
+    # build:
+    #   context: .
+    #   dockerfile: Dockerfile.redis
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6382
+    environment:
+      - REDIS_ROLE=master
+    ports:
+      - "6382:6382"
+
+  redis-cluster-init:
+    image: redis:7.2-alpine
+    volumes:
+      - ./init-redis-cluster.sh:/usr/local/bin/init-redis-cluster.sh
+    depends_on:
+      - redis-master1
+      - redis-master2
+      - redis-master3
+    entrypoint: ["/bin/sh", "/usr/local/bin/init-redis-cluster.sh"]
+    # environment:
+    #   - "REDISCLI_AUTH=yourpassword" # Optional: Include if your Redis nodes are password-protected
+    # To connect and debug the cluster use
+    # docker run -it --network  sdk-socket-server-next_default  --rm redis redis-cli -c -p 6379 -h redis-master1
+    # set mykey "Hello, Redis Cluster!"
+
+  # cache is used if want to simulate single node redis architecture
   cache:
-    image: redis:latest
+    image: redis:7.2-alpine
     command: redis-server --maxmemory 100mb --maxmemory-policy volatile-lru --loglevel debug
     ports:
       - '6379:6379'

packages/sdk-socket-server-next/package.json

@@ -44,6 +44,7 @@
     "express-rate-limit": "^7.1.5",
     "helmet": "^5.1.1",
     "ioredis": "^5.3.2",
+    "logform": "^2.6.0",
     "lru-cache": "^10.0.0",
     "rate-limiter-flexible": "^2.3.8",
     "redis": "^4.6.12",

packages/sdk-socket-server-next/src/api-config.ts

@@ -15,25 +15,50 @@ import { isDevelopment, isDevelopmentServer } from '.';
 const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
 export const redisClient = createClient({ url: redisUrl });
 const THIRTY_DAYS_IN_SECONDS = 30 * 24 * 60 * 60; // expiration time of entries in Redis
+const hasRateLimit = process.env.RATE_LIMITER === 'true';
 
 const app = express();
 
-const limiter = rateLimit({
-  windowMs: 5 * 60 * 1000, // 5 minutes
-  limit: 100, // Limit each IP to 100 requests per `window` (here, per 5 minutes).
-  standardHeaders: 'draft-7', // draft-6: `RateLimit-*` headers; draft-7: combined `RateLimit` header
-  legacyHeaders: false, // Disable the `X-RateLimit-*` headers.
-  // store: ... , // Use an external store for consistency across multiple server instances.
-});
-
 app.use(bodyParser.urlencoded({ extended: true }));
 app.use(bodyParser.json());
 app.use(cors());
 app.options('*', cors());
 app.use(helmet());
 app.disable('x-powered-by');
-// Apply the rate limiting middleware to all requests.
-app.use(limiter);
+
+if (hasRateLimit) {
+  // Conditionally apply the rate limiting middleware to all requests.
+  let windowMin = 1; // every 1minute
+  try {
+    if (process.env.RATE_LIMITER_HTTP_LIMIT) {
+      windowMin = parseInt(
+        process.env.RATE_LIMITER_HTTP_WINDOW_MINUTE ?? '1',
+        10,
+      );
+    }
+  } catch (error) {
+    // Ignore parsing errors
+  }
+  let limit = 100_000; // 100,000 requests per minute by default (unlimited...)
+  try {
+    if (process.env.RATE_LIMITER_HTTP_LIMIT) {
+      limit = parseInt(process.env.RATE_LIMITER_HTTP_LIMIT, 10);
+    }
+  } catch (error) {
+    // Ignore parsing errors
+  }
+
+  const limiterConfig = {
+    windowMs: windowMin * 60 * 1000,
+    limit,
+    legacyHeaders: false, // Disable the `X-RateLimit-*` headers.
+    // store: ... , // Use an external store for consistency across multiple server instances.
+  };
+  const limiter = rateLimit(limiterConfig);
+
+  logger.info('Rate limiter enabled', limiterConfig);
+  app.use(limiter);
+}
 
 async function inspectRedis(key?: string) {
   if (key && typeof key === 'string') {

packages/sdk-socket-server-next/src/index.ts

@@ -5,30 +5,41 @@ import dotenv from 'dotenv';
 // Dotenv must be loaded before importing local files
 dotenv.config();
 
+import { instrument } from '@socket.io/admin-ui';
 import { analytics, app } from './api-config';
 import { logger } from './logger';
-import { cleanupAndExit } from './utils';
-import { configureSocketServer } from './socket-config';
 import { extractMetrics } from './metrics';
+import { configureSocketServer } from './socket-config';
+import { cleanupAndExit } from './utils';
 
 export const isDevelopment: boolean = process.env.NODE_ENV === 'development';
 export const isDevelopmentServer: boolean =
   process.env.ENVIRONMENT === 'development';
+export const withAdminUI: boolean = process.env.ADMIN_UI === 'true';
 
 const server = http.createServer(app);
 
-configureSocketServer(server)
-  .then((ioServer) => {
-    logger.info('INFO> isDevelopment?', isDevelopment);
+// Register event listeners for process termination events
+process.on('SIGINT', async () => {
+  await cleanupAndExit(server, analytics);
+});
 
-    // Register event listeners for process termination events
-    process.on('SIGINT', async () => {
-      await cleanupAndExit(server, analytics);
-    });
+process.on('SIGTERM', async () => {
+  await cleanupAndExit(server, analytics);
+});
 
-    process.on('SIGTERM', async () => {
-      await cleanupAndExit(server, analytics);
-    });
+configureSocketServer(server)
+  .then((ioServer) => {
+    logger.info(
+      `socker.io server started development=${isDevelopment} adminUI=${withAdminUI}`,
+    );
+
+    if (isDevelopmentServer && withAdminUI) {
+      instrument(ioServer, {
+        auth: false,
+        mode: 'development',
+      });
+    }
 
     // Make sure to protect the endpoint to be only available within the cluster for prometheus
     app.get('/metrics', async (_req, res) => {

packages/sdk-socket-server-next/src/logger.ts

@@ -1,14 +1,57 @@
-import winston from 'winston';
+import winston, { format } from 'winston';
 
 const isDevelopment: boolean = process.env.NODE_ENV === 'development';
 
+const customFormat = format.printf((ti) => {
+  const { level, message, timestamp } = ti;
+  const args = ti[Symbol.for('splat')];
+
+  const color = {
+    info: '\x1b[36m',
+    error: '\x1b[31m',
+    warn: '\x1b[33m',
+    debug: '\x1b[32m',
+  };
+
+  // add color to the level of the log
+  // let msg = `${timestamp} [${level}] : `;
+  let printTsLevel = level;
+  if (color[level as keyof typeof color]) {
+    printTsLevel = `${
+      color[level as keyof typeof color]
+    }${timestamp} [${level}]\x1b[0m`;
+  }
+  let msg = `${printTsLevel}: `;
+
+  const extras =
+    args
+      ?.map((arg: unknown) => {
+        if (typeof arg === 'object' && arg !== null) {
+          return JSON.stringify(args);
+        }
+        return arg;
+      })
+      .join(' ') ?? '';
+
+  if (isDevelopment) {
+    const searchContext = message;
+    if (searchContext.indexOf('wallet') !== -1) {
+      msg += `\x1b[36m${message} ${extras}\x1b[0m`;
+      // eslint-disable-next-line no-negated-condition
+    } else if (searchContext.indexOf('dapp') !== -1) {
+      msg += `\x1b[35m${message} ${extras}\x1b[0m`;
+    } else {
+      msg += `${message} ${extras}`;
+    }
+  } else {
+    msg += `${message} ${extras}`;
+  }
+  return msg;
+});
+
 export const logger = winston.createLogger({
   level: 'debug',
-  format: winston.format.combine(
-    winston.format.timestamp(),
-    isDevelopment ? winston.format.colorize() : winston.format.uncolorize(),
-    winston.format.simple(),
-  ),
+  format: winston.format.combine(winston.format.timestamp(), customFormat),
   transports: [
     new winston.transports.Console(),
     // You can also add file transport or any other transport here