From 83153ee50c56b42fb42c03effa9ccbdcf4992da2 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 14:11:50 +0500 Subject: [PATCH 1/6] feat: expose user agent list to http module --- apps/cli | 2 +- packages/core/lib/http.ts | 2 ++ packages/core/lib/store.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/cli b/apps/cli index 978bcc8..e340cef 160000 --- a/apps/cli +++ b/apps/cli @@ -1 +1 @@ -Subproject commit 978bcc8f35e3d6f6a2040189c776d87b9e3b44b3 +Subproject commit e340cef1212b266df52d5746f4138e0cab3e4bbf diff --git a/packages/core/lib/http.ts b/packages/core/lib/http.ts index 7d6384b..a2f857e 100644 --- a/packages/core/lib/http.ts +++ b/packages/core/lib/http.ts @@ -56,6 +56,7 @@ const DEFAULT_MAX_REDIRECTIONS = 5; export interface IHttp { headers: Record; cookies: string[]; + userAgents: Record<'chromeWindows' | 'chromeMacOS' | 'chromeLinux' | 'smartTv' | 'tizen', string>; fetch(resource: string | URL | Request, options?: RequestInit): Promise; fetchAsChrome(resource: string | URL | Request, options?: RequestInit): Promise; appendCookies(setCookie: string | string[]): void; @@ -72,6 +73,7 @@ const sessionToken = { id: crypto.randomUUID() }; class Http implements IHttp { headers: Record; cookies: string[]; + userAgents = USER_AGENTS; #sessions: Map; #retryThreshold: number; diff --git a/packages/core/lib/store.ts b/packages/core/lib/store.ts index 5b81296..e893498 100644 --- a/packages/core/lib/store.ts +++ b/packages/core/lib/store.ts @@ -37,7 +37,7 @@ export const createStore = (name: string) => { const hasCookiesInState = cookiesKey && data[cookiesKey]; if (hasCookiesInTxt) http.setCookies(cookies); else if (hasCookiesInState) http.setCookies(data[cookiesKey]); - return data as T; + return state as T; }; const setState = async >(data?: T) => { Object.assign(state, data || {}); From feefffe411b43c71e0e4ba240d7edccea6e75846 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 15:27:31 +0500 Subject: [PATCH 2/6] refactor: move browser utils in separate package --- package-lock.json | 288 ++++++++++++++++++++++++++- packages/browser/.gitignore | 4 + packages/browser/LICENSE | 21 ++ packages/browser/eslint.config.mjs | 10 + packages/browser/index.d.ts | 30 +++ packages/browser/index.js | 139 +++++++++++++ packages/browser/package.json | 59 ++++++ packages/browser/prettier.config.mjs | 8 + packages/core/lib/browser.ts | 134 ------------- packages/core/lib/main.ts | 1 - packages/core/package.json | 3 - 11 files changed, 556 insertions(+), 141 deletions(-) create mode 100644 packages/browser/.gitignore create mode 100644 packages/browser/LICENSE create mode 100644 packages/browser/eslint.config.mjs create mode 100644 packages/browser/index.d.ts create mode 100644 packages/browser/index.js create mode 100644 packages/browser/package.json create mode 100644 packages/browser/prettier.config.mjs delete mode 100644 packages/core/lib/browser.ts diff --git a/package-lock.json b/package-lock.json index 7c78cd0..d25e359 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4633,6 +4633,10 @@ "solid-js": "^1.8.6" } }, + "node_modules/@streamyx/browser": { + "resolved": "packages/browser", + "link": true + }, "node_modules/@streamyx/cli": { "resolved": "apps/cli", "link": true @@ -19808,6 +19812,287 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "packages/browser": { + "name": "@streamyx/browser", + "version": "0.0.1", + "funding": [ + { + "type": "individual", + "url": "https://boosty.to/vitalygashkov" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/vitalygashkov" + } + ], + "license": "MIT", + "dependencies": { + "puppeteer-core": "^23.5.3", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "@types/node": "^22.9.0", + "eslint": "^9.14.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.12.0", + "prettier": "^3.3.3", + "typescript": "^5.6.3" + }, + "engines": { + "node": "20 || 21 || 22 || 23" + } + }, + "packages/browser/node_modules/@eslint/config-array": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.19.0.tgz", + "integrity": "sha512-zdHg2FPIFNKPdcHWtiNT+jEFCHYVplAXRDlQDyqy0zGx/q2parwh7brGJSiTxRk/TSMkbM//zt/f5CHgyTyaSQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.4", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "packages/browser/node_modules/@eslint/eslintrc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.2.0.tgz", + "integrity": "sha512-grOjVNN8P3hjJn/eIETF1wwd12DdnwFDoyceUJLYYdkpbwq3nLi+4fqrTAONx7XDALqlL220wC/RHSC/QTI/0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/browser/node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/browser/node_modules/@eslint/js": { + "version": "9.16.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.16.0.tgz", + "integrity": "sha512-tw2HxzQkrbeuvyj1tG2Yqq+0H9wGoI2IMk4EOsQeX+vmd75FtJAzf+gTA69WF+baUKRYQ3x2kbLE08js5OsTVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "packages/browser/node_modules/@humanwhocodes/retry": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.1.tgz", + "integrity": "sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "packages/browser/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "packages/browser/node_modules/eslint": { + "version": "9.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.16.0.tgz", + "integrity": "sha512-whp8mSQI4C8VXd+fLgSM0lh3UlmcFtVwUQjyKCFfsp+2ItAIYhlq/hqGahGqHE6cv9unM41VlqKk2VtKYR2TaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.19.0", + "@eslint/core": "^0.9.0", + "@eslint/eslintrc": "^3.2.0", + "@eslint/js": "9.16.0", + "@eslint/plugin-kit": "^0.2.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.1", + "@types/estree": "^1.0.6", + "@types/json-schema": "^7.0.15", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.5", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.2.0", + "eslint-visitor-keys": "^4.2.0", + "espree": "^10.3.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "packages/browser/node_modules/eslint-scope": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz", + "integrity": "sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/browser/node_modules/eslint-visitor-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", + "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/browser/node_modules/espree": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", + "integrity": "sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.14.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/browser/node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "packages/browser/node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "packages/browser/node_modules/globals": { + "version": "15.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-15.12.0.tgz", + "integrity": "sha512-1+gLErljJFhbOVyaetcwJiJ4+eLe45S2E7P5UiZ9xGfeq3ATQf5DOv9G7MH3gGbKQLkzmNh2DxfZwLdw+j6oTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/browser/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "packages/core": { "name": "@streamyx/core", "version": "2.2.0", @@ -19817,9 +20102,6 @@ "got-scraping": "^3.2.15", "pino": "^9.5.0", "pino-pretty": "^11.3.0", - "puppeteer-core": "^23.5.3", - "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2", "undici": "^6.21.0" }, "devDependencies": { diff --git a/packages/browser/.gitignore b/packages/browser/.gitignore new file mode 100644 index 0000000..becd09b --- /dev/null +++ b/packages/browser/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +build +.DS_Store diff --git a/packages/browser/LICENSE b/packages/browser/LICENSE new file mode 100644 index 0000000..6ed8df4 --- /dev/null +++ b/packages/browser/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Vitaly Gashkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/browser/eslint.config.mjs b/packages/browser/eslint.config.mjs new file mode 100644 index 0000000..96072c8 --- /dev/null +++ b/packages/browser/eslint.config.mjs @@ -0,0 +1,10 @@ +import globals from 'globals'; +import pluginJs from '@eslint/js'; +import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'; + +export default [ + { files: ['**/*.js'], languageOptions: { sourceType: 'commonjs' } }, + { languageOptions: { globals: globals.node } }, + pluginJs.configs.recommended, + eslintPluginPrettierRecommended, +]; diff --git a/packages/browser/index.d.ts b/packages/browser/index.d.ts new file mode 100644 index 0000000..e209eea --- /dev/null +++ b/packages/browser/index.d.ts @@ -0,0 +1,30 @@ +import type { + Browser, + BrowserLaunchArgumentOptions, + Page, +} from 'puppeteer-core'; + +type LaunchBrowserOptions = BrowserLaunchArgumentOptions & { + chromePath?: string; + proxy?: string | null; + onChromePathPrompt?: (message: string) => Promise; +}; + +type LaunchBrowserResponse = { + browser: Browser; + page: Page; + chromePath: string | null; +}; + +export function launchBrowser( + options?: LaunchBrowserOptions, +): Promise; + +export function fetchViaBrowser( + resource: string, + options: RequestInit, + browser?: Browser, +): Promise<{ + response: Response; + cookies: Cookie[]; +}>; diff --git a/packages/browser/index.js b/packages/browser/index.js new file mode 100644 index 0000000..3df1c8e --- /dev/null +++ b/packages/browser/index.js @@ -0,0 +1,139 @@ +'use strict'; + +const { stat } = require('node:fs/promises'); +const puppeteer = require('puppeteer-extra'); +const StealthPlugin = require('puppeteer-extra-plugin-stealth'); + +puppeteer.use(StealthPlugin()); + +const findChromePath = async () => { + const paths = [ + 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', + 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', + 'C:\\Program Files (Arm)\\Google\\Chrome\\Application\\chrome.exe', + ]; + const getPathStat = (p) => + stat(p) + .catch(() => null) + .then((s) => (s ? p : null)); + return await Promise.any(paths.map(getPathStat)).catch(() => null); +}; + +const launchBrowser = async (options = {}) => { + const { chromePath, proxy, onChromePathPrompt, ...rest } = options; + let executablePath = chromePath || (await findChromePath()); + let browser = null; + let page = null; + const args = ['--no-sandbox', '--start-maximized', '--lang=ru']; + if (proxy) args.push(`--proxy-server=${proxy}`); + const mainOptions = { + headless: true, + args, + userDataDir: './config/chrome', + ...rest, + }; + while (!browser || !page) { + try { + const launchOptions = executablePath + ? { executablePath, ...mainOptions } + : { channel: 'chrome', ...mainOptions }; + browser = await puppeteer.launch(launchOptions); + page = (await browser?.newPage()) ?? null; + } catch (e) { + if (!onChromePathPrompt) throw e; + const answer = await onChromePathPrompt?.( + 'Enter valid Chrome executable path', + ); + if (answer) executablePath = answer; + } + } + const aboutBlankPage = (await browser.pages())[0]; + if (aboutBlankPage) await aboutBlankPage.close(); + await page.setBypassCSP(true); + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, 'language', { + get: function () { + return 'ru'; + }, + }); + Object.defineProperty(navigator, 'languages', { + get: function () { + return ['ru']; + }, + }); + }); + await page.setExtraHTTPHeaders({ 'Accept-Language': 'ru' }); + + return { browser, page, chromePath: executablePath }; +}; + +const fetchViaBrowser = async (resource, options, browser) => { + const browserInstance = browser ? browser : (await launchBrowser()).browser; + // Load base url on page and parse cookies + const page = await browserInstance.newPage(); + await page.bringToFront(); + const { origin } = new URL(resource); + await page.evaluate((url) => { + window.open(url, '_self'); + }, origin); + await page.waitForNavigation(); + + const isWaitingForRedirect = options?.redirect === 'manual'; + const response = + (await new Promise()) < + Response > + ((resolve) => { + if (isWaitingForRedirect) { + page.on('response', async (httpResponse) => { + const url = httpResponse.request().url(); + if (url !== resource) return; + const response = new Response(null, { + headers: httpResponse.headers(), + status: httpResponse.status(), + statusText: httpResponse.statusText(), + }); + resolve(response); + }); + } + page + .evaluate( + (resource, init) => { + const fetchData = async () => { + const initBody = init.body; + const body = + typeof initBody === 'object' && initBody.type === 'Buffer' + ? Uint8Array.from(initBody.data) + : init.body; + init.body = body; + const response = await globalThis.fetch(resource, init); + return { + body: new Uint8Array(await response.arrayBuffer()), + init: { + headers: response.headers, + status: response.status, + statusText: response.statusText, + }, + }; + }; + return fetchData(); + }, + resource, + options, + ) + .then(({ body, init }) => { + return isWaitingForRedirect + ? {} + : resolve(new Response(Buffer.from(Object.values(body)), init)); + }) + .catch(() => ({ body: null, init: undefined })); + }); + + const cookies = await page.cookies(); + + page.removeAllListeners('response'); + if (!page.isClosed) await page.close(); + + return { response, cookies }; +}; + +module.exports = { launchBrowser, fetchViaBrowser }; diff --git a/packages/browser/package.json b/packages/browser/package.json new file mode 100644 index 0000000..154def7 --- /dev/null +++ b/packages/browser/package.json @@ -0,0 +1,59 @@ +{ + "name": "@streamyx/browser", + "version": "0.0.1", + "description": "Browser utilities for Streamyx", + "main": "index.js", + "types": "index.d.ts", + "files": [ + "index.js", + "index.d.ts" + ], + "scripts": { + "lint": "eslint . && prettier --check .", + "fix": "eslint . --fix && prettier --write ." + }, + "repository": { + "type": "git", + "url": "git+https://github.com/vitalygashkov/streamyx.git" + }, + "keywords": [ + "streamyx", + "puppeteer", + "browser" + ], + "bugs": { + "url": "https://github.com/vitalygashkov/streamyx/issues", + "email": "vitalygashkov@vk.com" + }, + "author": "Vitaly Gashkov ", + "license": "MIT", + "readmeFilename": "README.md", + "funding": [ + { + "type": "individual", + "url": "https://boosty.to/vitalygashkov" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/vitalygashkov" + } + ], + "engines": { + "node": "20 || 21 || 22 || 23" + }, + "dependencies": { + "puppeteer-core": "^23.5.3", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" + }, + "devDependencies": { + "@eslint/js": "^9.14.0", + "@types/node": "^22.9.0", + "eslint": "^9.14.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.12.0", + "prettier": "^3.3.3", + "typescript": "^5.6.3" + } +} diff --git a/packages/browser/prettier.config.mjs b/packages/browser/prettier.config.mjs new file mode 100644 index 0000000..0edf463 --- /dev/null +++ b/packages/browser/prettier.config.mjs @@ -0,0 +1,8 @@ +export default { + printWidth: 80, + singleQuote: true, + trailingComma: 'all', + tabWidth: 2, + useTabs: false, + semi: true, +}; diff --git a/packages/core/lib/browser.ts b/packages/core/lib/browser.ts deleted file mode 100644 index 65d3e55..0000000 --- a/packages/core/lib/browser.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { Browser, BrowserLaunchArgumentOptions, Page } from 'puppeteer-core'; -import puppeteer from 'puppeteer-extra'; -import StealthPlugin from 'puppeteer-extra-plugin-stealth'; -import { logger } from './logger'; -import { prompt } from './prompt'; -import { getSettings, saveSettings } from './settings'; -import { getAnyValidPath } from './bin'; -import { parseUrlFromResource } from './utils'; -import { browserCookiesToList } from './cookies'; - -puppeteer.use(StealthPlugin()); - -const findChromePath = async () => { - const paths = [ - 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe', - 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', - 'C:\\Program Files (Arm)\\Google\\Chrome\\Application\\chrome.exe', - ]; - return getAnyValidPath(paths); -}; - -export const launchBrowser = async (options: BrowserLaunchArgumentOptions = {}, proxy?: string | null) => { - const { chromePath } = getSettings(); - let executablePath: string | null = chromePath || (await findChromePath()); - if (!chromePath && executablePath) await saveSettings({ chromePath: executablePath }); - let browser: Browser | null = null; - let page: Page | null = null; - const args = ['--no-sandbox', '--start-maximized', '--lang=ru']; - if (proxy) args.push(`--proxy-server=${proxy}`); - const mainOptions: BrowserLaunchArgumentOptions = { - headless: true, - args, - userDataDir: './config/chrome', - ...options, - }; - while (!browser || !page) { - try { - const launchOptions = executablePath ? { executablePath, ...mainOptions } : { channel: 'chrome', ...mainOptions }; - browser = await puppeteer.launch(launchOptions); - page = (await browser?.newPage()) ?? null; - } catch (e) { - logger.error((e as Error).message); - const answer = await prompt.ask({ - executablePath: { label: 'Enter valid Chrome executable path' }, - }); - executablePath = answer.executablePath; - } - } - if (executablePath !== chromePath) saveSettings({ chromePath: executablePath }); - const aboutBlankPage = (await browser.pages())[0]; - if (aboutBlankPage) await aboutBlankPage.close(); - await page.setBypassCSP(true); - await page.evaluateOnNewDocument(() => { - Object.defineProperty(navigator, 'language', { - get: function () { - return 'ru'; - }, - }); - Object.defineProperty(navigator, 'languages', { - get: function () { - return ['ru']; - }, - }); - }); - await page.setExtraHTTPHeaders({ 'Accept-Language': 'ru' }); - - return { browser, page, chromePath: executablePath }; -}; - -export const fetchViaBrowser = async (resource: string | URL | Request, options: RequestInit, browser?: Browser) => { - const browserInstance = browser ? browser : (await launchBrowser()).browser; - - // Load base url on page and parse cookies - const page = await browserInstance.newPage(); - await page.bringToFront(); - const { origin } = parseUrlFromResource(resource); - await page.evaluate((url) => { - window.open(url, '_self'); - }, origin); - await page.waitForNavigation(); - const browserCookies = await page.cookies(); - const cookies = browserCookiesToList(browserCookies); - - const isWaitingForRedirect = options?.redirect === 'manual'; - const response = await new Promise((resolve) => { - if (isWaitingForRedirect) { - page.on('response', async (httpResponse) => { - const url = httpResponse.request().url(); - if (url !== resource) return; - const response = new Response(null, { - headers: httpResponse.headers(), - status: httpResponse.status(), - statusText: httpResponse.statusText(), - }); - resolve(response); - }); - } - page - .evaluate( - (resource, init: globalThis.RequestInit) => { - const fetchData = async () => { - const initBody = init.body as string | { type: 'Buffer'; data: number[] }; - const body = - typeof initBody === 'object' && initBody.type === 'Buffer' ? Uint8Array.from(initBody.data) : init.body; - init.body = body; - const response = await globalThis.fetch(resource as globalThis.RequestInfo, init); - return { - body: new Uint8Array(await response.arrayBuffer()), - init: { - headers: response.headers as unknown as Headers, - status: response.status, - statusText: response.statusText, - } as ResponseInit, - }; - }; - return fetchData(); - }, - resource, - options! - ) - .then(({ body, init }) => { - return isWaitingForRedirect ? {} : resolve(new Response(Buffer.from(Object.values(body)), init)); - }) - .catch((e) => { - logger.debug(`Error while evaluate browser fetch: ${e?.message}`); - return { body: null, init: undefined }; - }); - }); - - page.removeAllListeners('response'); - if (!page.isClosed) await page.close(); - - return { response, cookies }; -}; diff --git a/packages/core/lib/main.ts b/packages/core/lib/main.ts index 689fb32..2e1cc9c 100644 --- a/packages/core/lib/main.ts +++ b/packages/core/lib/main.ts @@ -3,7 +3,6 @@ export * from './http'; export * from './cookies'; export * from './settings'; export * from './logger'; -export * from './browser'; export * from './bin'; export * from './prompt'; export * from './store'; diff --git a/packages/core/package.json b/packages/core/package.json index 66a1031..1305dc0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -27,9 +27,6 @@ "got-scraping": "^3.2.15", "pino": "^9.5.0", "pino-pretty": "^11.3.0", - "puppeteer-core": "^23.5.3", - "puppeteer-extra": "^3.3.6", - "puppeteer-extra-plugin-stealth": "^2.11.2", "undici": "^6.21.0" }, "devDependencies": { From fb5cf348599ea71954b4c44c8826e1f7b8037935 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 15:56:56 +0500 Subject: [PATCH 3/6] refactor: move logger from core to separate package --- apps/cli | 2 +- package-lock.json | 528 +++++++++++++++++++--------- packages/browser/package.json | 12 +- packages/core/lib/bin.ts | 2 +- packages/core/lib/http.ts | 2 +- packages/core/lib/log.ts | 15 + packages/core/lib/logger.ts | 185 +++++----- packages/core/lib/main.ts | 2 +- packages/core/lib/service.ts | 2 +- packages/core/lib/settings.ts | 2 +- packages/core/package.json | 4 +- packages/logger/.gitignore | 4 + packages/logger/LICENSE | 21 ++ packages/logger/eslint.config.mjs | 10 + packages/logger/index.d.ts | 10 + packages/logger/index.js | 110 ++++++ packages/logger/package.json | 58 +++ packages/logger/prettier.config.mjs | 8 + 18 files changed, 703 insertions(+), 274 deletions(-) create mode 100644 packages/core/lib/log.ts create mode 100644 packages/logger/.gitignore create mode 100644 packages/logger/LICENSE create mode 100644 packages/logger/eslint.config.mjs create mode 100644 packages/logger/index.d.ts create mode 100644 packages/logger/index.js create mode 100644 packages/logger/package.json create mode 100644 packages/logger/prettier.config.mjs diff --git a/apps/cli b/apps/cli index e340cef..2a6bfef 160000 --- a/apps/cli +++ b/apps/cli @@ -1 +1 @@ -Subproject commit e340cef1212b266df52d5746f4138e0cab3e4bbf +Subproject commit 2a6bfef00092e230940634eac2b02b77ff4bde6f diff --git a/package-lock.json b/package-lock.json index d25e359..09e066f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4230,12 +4230,12 @@ "license": "BSD-3-Clause" }, "node_modules/@puppeteer/browsers": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.4.0.tgz", - "integrity": "sha512-x8J1csfIygOwf6D6qUAZ0ASk3z63zPb7wkNeHRerCMh82qWKUrOgkuP005AJC8lDL6/evtXETGEJVcwykKT4/g==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.4.1.tgz", + "integrity": "sha512-0kdAbmic3J09I6dT8e9vE2JOCSt13wHCW5x/ly8TSt2bDtuIWe2TgLZZDHdcziw9AVCzflMAXCrVyRIhIs44Ng==", "license": "Apache-2.0", "dependencies": { - "debug": "^4.3.6", + "debug": "^4.3.7", "extract-zip": "^2.0.1", "progress": "^2.0.3", "proxy-agent": "^6.4.0", @@ -4649,6 +4649,10 @@ "resolved": "packages/crunchyroll", "link": true }, + "node_modules/@streamyx/logger": { + "resolved": "packages/logger", + "link": true + }, "node_modules/@streamyx/services": { "resolved": "packages/services", "link": true @@ -4827,9 +4831,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.0.tgz", - "integrity": "sha512-XC70cRZVElFHfIUB40FgZOBbgJYFKKMa5nb9lxcwYstFG/Mi+/Y0bGS+rs6Dmhmkpq4pnNiLiuZAbc02YCOnmA==", + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", "license": "MIT", "dependencies": { "undici-types": "~6.20.0" @@ -5778,18 +5782,6 @@ "dev": true, "license": "ISC" }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -6584,9 +6576,9 @@ } }, "node_modules/b4a": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", - "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", "license": "Apache-2.0" }, "node_modules/balanced-match": { @@ -6596,16 +6588,16 @@ "license": "MIT" }, "node_modules/bare-events": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", - "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", "license": "Apache-2.0", "optional": true }, "node_modules/bare-fs": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.3.tgz", - "integrity": "sha512-7RYKL+vZVCyAsMLi5SPu7QGauGGT8avnP/HO571ndEuV4MYdGXvLhtW67FuLPeEI8EiIY7zbbRR9x7x7HU0kgw==", + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", "license": "Apache-2.0", "optional": true, "dependencies": { @@ -6615,9 +6607,9 @@ } }, "node_modules/bare-os": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.2.tgz", - "integrity": "sha512-HZoJwzC+rZ9lqEemTMiO0luOePoGYNBgsLLgegKR/cljiJvcDNhDZQkzC+NC5Oh0aHbdBNSOHpghwMuB5tqhjg==", + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", "license": "Apache-2.0", "optional": true }, @@ -6632,14 +6624,13 @@ } }, "node_modules/bare-stream": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.2.1.tgz", - "integrity": "sha512-YTB47kHwBW9zSG8LD77MIBAAQXjU2WjAkMHeeb7hUplVs6+IoM5I7uEVQNPMB7lj9r8I76UMdoMkGnCodHOLqg==", + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.4.2.tgz", + "integrity": "sha512-XZ4ln/KV4KT+PXdIWTKjsLY+quqCaEtqqtgGJVPw9AoM73By03ij64YjepK0aQvHSWDb6AfAZwqKaFu68qkrdA==", "license": "Apache-2.0", "optional": true, "dependencies": { - "b4a": "^1.6.6", - "streamx": "^2.18.0" + "streamx": "^2.20.0" } }, "node_modules/base32-encode": { @@ -8095,9 +8086,9 @@ "optional": true }, "node_modules/devtools-protocol": { - "version": "0.0.1342118", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1342118.tgz", - "integrity": "sha512-75fMas7PkYNDTmDyb6PRJCH7ILmHLp+BhrZGeMsa4bCh40DTxgCz2NRy5UDzII4C5KuD0oBMZ9vXKhEl6UD/3w==", + "version": "0.0.1367902", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz", + "integrity": "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg==", "license": "BSD-3-Clause" }, "node_modules/didyoumean": { @@ -9817,15 +9808,6 @@ "node": ">= 0.6" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/eventemitter3": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", @@ -9833,15 +9815,6 @@ "dev": true, "license": "MIT" }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/execa": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", @@ -14350,100 +14323,6 @@ "split2": "^4.0.0" } }, - "node_modules/pino-pretty": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-11.3.0.tgz", - "integrity": "sha512-oXwn7ICywaZPHmu3epHGU2oJX4nPmKvHvB/bwrJHlGcbEWaVcotkpyVHMKLKmiVryWYByNp0jpgAcXpFJDXJzA==", - "license": "MIT", - "dependencies": { - "colorette": "^2.0.7", - "dateformat": "^4.6.3", - "fast-copy": "^3.0.2", - "fast-safe-stringify": "^2.1.1", - "help-me": "^5.0.0", - "joycon": "^3.1.1", - "minimist": "^1.2.6", - "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "^2.0.0", - "pump": "^3.0.0", - "readable-stream": "^4.0.0", - "secure-json-parse": "^2.4.0", - "sonic-boom": "^4.0.1", - "strip-json-comments": "^3.1.1" - }, - "bin": { - "pino-pretty": "bin.js" - } - }, - "node_modules/pino-pretty/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, - "node_modules/pino-pretty/node_modules/readable-stream": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", - "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", - "license": "MIT", - "dependencies": { - "abort-controller": "^3.0.0", - "buffer": "^6.0.3", - "events": "^3.3.0", - "process": "^0.11.10", - "string_decoder": "^1.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/pino-pretty/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/pino-pretty/node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/pino-std-serializers": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", @@ -15001,15 +14880,15 @@ } }, "node_modules/puppeteer-core": { - "version": "23.5.3", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.5.3.tgz", - "integrity": "sha512-V58MZD/B3CwkYsqSEQlHKbavMJptF04fzhMdUpiCRCmUVhwZNwSGEPhaiZ1f8I3ABQUirg3VNhXVB6Z1ubHXtQ==", + "version": "23.9.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.9.0.tgz", + "integrity": "sha512-hLVrav2HYMVdK0YILtfJwtnkBAwNOztUdR4aJ5YKDvgsbtagNr6urUJk9HyjRA9e+PaLI3jzJ0wM7A4jSZ7Qxw==", "license": "Apache-2.0", "dependencies": { - "@puppeteer/browsers": "2.4.0", + "@puppeteer/browsers": "2.4.1", "chromium-bidi": "0.8.0", "debug": "^4.3.7", - "devtools-protocol": "0.0.1342118", + "devtools-protocol": "0.0.1367902", "typed-query-selector": "^2.12.0", "ws": "^8.18.0" }, @@ -16599,9 +16478,9 @@ "license": "MIT" }, "node_modules/streamx": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.0.tgz", - "integrity": "sha512-ZGd1LhDeGFucr1CUCTBOS58ZhEendd0ttpGT3usTvosS4ntIwKN9LJFp+OeCSprsCPL14BXVRZlHGRY1V9PVzQ==", + "version": "2.20.2", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.2.tgz", + "integrity": "sha512-aDGDLU+j9tJcUdPGOaHmVF1u/hhI+CsGkT02V3OKlHDV7IukOI+nTWAGkiZEKCO35rWN1wIr4tS7YFr1f4qSvA==", "license": "MIT", "dependencies": { "fast-fifo": "^1.3.2", @@ -17122,13 +17001,10 @@ } }, "node_modules/text-decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", - "integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==", - "license": "Apache-2.0", - "dependencies": { - "b4a": "^1.6.4" - } + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.1.tgz", + "integrity": "sha512-x9v3H/lTKIJKQQe7RPQkLfKAnc9lUTkWDypIQgTzPJAq+5/GCDHonmshfvlsNSj58yyshbIJJDLmU15qNERrXQ==", + "license": "Apache-2.0" }, "node_modules/text-table": { "version": "0.2.0", @@ -19827,19 +19703,19 @@ ], "license": "MIT", "dependencies": { - "puppeteer-core": "^23.5.3", + "puppeteer-core": "^23.9.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2" }, "devDependencies": { - "@eslint/js": "^9.14.0", - "@types/node": "^22.9.0", - "eslint": "^9.14.0", + "@eslint/js": "^9.16.0", + "@types/node": "^22.10.1", + "eslint": "^9.16.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", "globals": "^15.12.0", - "prettier": "^3.3.3", - "typescript": "^5.6.3" + "prettier": "^3.4.1", + "typescript": "^5.7.2" }, "engines": { "node": "20 || 21 || 22 || 23" @@ -20095,13 +19971,11 @@ }, "packages/core": { "name": "@streamyx/core", - "version": "2.2.0", + "version": "2.2.1", "license": "AGPL-3.0", "dependencies": { "dasha": "^3.1.1", "got-scraping": "^3.2.15", - "pino": "^9.5.0", - "pino-pretty": "^11.3.0", "undici": "^6.21.0" }, "devDependencies": { @@ -20407,6 +20281,310 @@ "node": "*" } }, + "packages/logger": { + "name": "@streamyx/logger", + "version": "0.0.1", + "funding": [ + { + "type": "individual", + "url": "https://boosty.to/vitalygashkov" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/vitalygashkov" + } + ], + "license": "MIT", + "dependencies": { + "pino": "^9.5.0", + "pino-pretty": "^13.0.0" + }, + "devDependencies": { + "@eslint/js": "^9.16.0", + "@types/node": "^22.10.1", + "eslint": "^9.16.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.12.0", + "prettier": "^3.4.1", + "typescript": "^5.7.2" + }, + "engines": { + "node": "20 || 21 || 22 || 23" + } + }, + "packages/logger/node_modules/@eslint/config-array": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.19.0.tgz", + "integrity": "sha512-zdHg2FPIFNKPdcHWtiNT+jEFCHYVplAXRDlQDyqy0zGx/q2parwh7brGJSiTxRk/TSMkbM//zt/f5CHgyTyaSQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.4", + "debug": "^4.3.1", + "minimatch": "^3.1.2" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "packages/logger/node_modules/@eslint/eslintrc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.2.0.tgz", + "integrity": "sha512-grOjVNN8P3hjJn/eIETF1wwd12DdnwFDoyceUJLYYdkpbwq3nLi+4fqrTAONx7XDALqlL220wC/RHSC/QTI/0w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/logger/node_modules/@eslint/eslintrc/node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/logger/node_modules/@eslint/js": { + "version": "9.16.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.16.0.tgz", + "integrity": "sha512-tw2HxzQkrbeuvyj1tG2Yqq+0H9wGoI2IMk4EOsQeX+vmd75FtJAzf+gTA69WF+baUKRYQ3x2kbLE08js5OsTVg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "packages/logger/node_modules/@humanwhocodes/retry": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.1.tgz", + "integrity": "sha512-c7hNEllBlenFTHBky65mhq8WD2kbN9Q6gk0bTk8lSBvc554jpXSkST1iePudpt7+A/AQvuHs9EMqjHDXMY1lrA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "packages/logger/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "packages/logger/node_modules/eslint": { + "version": "9.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.16.0.tgz", + "integrity": "sha512-whp8mSQI4C8VXd+fLgSM0lh3UlmcFtVwUQjyKCFfsp+2ItAIYhlq/hqGahGqHE6cv9unM41VlqKk2VtKYR2TaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.19.0", + "@eslint/core": "^0.9.0", + "@eslint/eslintrc": "^3.2.0", + "@eslint/js": "9.16.0", + "@eslint/plugin-kit": "^0.2.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.1", + "@types/estree": "^1.0.6", + "@types/json-schema": "^7.0.15", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.5", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.2.0", + "eslint-visitor-keys": "^4.2.0", + "espree": "^10.3.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "packages/logger/node_modules/eslint-scope": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.2.0.tgz", + "integrity": "sha512-PHlWUfG6lvPc3yvP5A4PNyBL1W8fkDUccmI21JUu/+GKZBoH/W5u6usENXUrWFRsyoW5ACUjFGgAFQp5gUlb/A==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/logger/node_modules/eslint-visitor-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", + "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/logger/node_modules/espree": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", + "integrity": "sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.14.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/logger/node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "packages/logger/node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "packages/logger/node_modules/globals": { + "version": "15.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-15.12.0.tgz", + "integrity": "sha512-1+gLErljJFhbOVyaetcwJiJ4+eLe45S2E7P5UiZ9xGfeq3ATQf5DOv9G7MH3gGbKQLkzmNh2DxfZwLdw+j6oTQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/logger/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "packages/logger/node_modules/pino-pretty": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.0.0.tgz", + "integrity": "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.2", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, "packages/molnia": { "version": "0.0.15", "funding": [ diff --git a/packages/browser/package.json b/packages/browser/package.json index 154def7..8f5d827 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -42,18 +42,18 @@ "node": "20 || 21 || 22 || 23" }, "dependencies": { - "puppeteer-core": "^23.5.3", + "puppeteer-core": "^23.9.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2" }, "devDependencies": { - "@eslint/js": "^9.14.0", - "@types/node": "^22.9.0", - "eslint": "^9.14.0", + "@eslint/js": "^9.16.0", + "@types/node": "^22.10.1", + "eslint": "^9.16.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-prettier": "^5.2.1", "globals": "^15.12.0", - "prettier": "^3.3.3", - "typescript": "^5.6.3" + "prettier": "^3.4.1", + "typescript": "^5.7.2" } } diff --git a/packages/core/lib/bin.ts b/packages/core/lib/bin.ts index 85aab57..92a0f6b 100644 --- a/packages/core/lib/bin.ts +++ b/packages/core/lib/bin.ts @@ -3,7 +3,7 @@ import { ChildProcessWithoutNullStreams, spawn } from 'node:child_process'; import { chmodSync } from 'node:fs'; import { delimiter } from 'node:path'; import { stat } from 'node:fs/promises'; -import { logger } from './logger'; +import { logger } from './log'; import { fs } from './fs'; import { getSettings } from './settings'; diff --git a/packages/core/lib/http.ts b/packages/core/lib/http.ts index a2f857e..f8ec419 100644 --- a/packages/core/lib/http.ts +++ b/packages/core/lib/http.ts @@ -6,7 +6,7 @@ import http2, { import { URL } from 'node:url'; import { fetch, ProxyAgent, Agent, buildConnector } from 'undici'; import { gotScraping } from 'got-scraping'; -import { logger } from './logger'; +import { logger } from './log'; import { randomizeCiphers } from './tls'; import { browserCookiesToList, Cookie } from './cookies'; import { parseUrlFromResource } from './utils'; diff --git a/packages/core/lib/log.ts b/packages/core/lib/log.ts new file mode 100644 index 0000000..19f3f02 --- /dev/null +++ b/packages/core/lib/log.ts @@ -0,0 +1,15 @@ +import { join } from 'node:path'; +import { promises as fsp } from 'node:fs'; +import { BaseDirectory } from './fs'; +import { createLogger } from './logger'; +import { isExecutable } from './utils'; + +const logger = createLogger({ dir: BaseDirectory.AppLog, writeToFile: isExecutable }); + +const showLogsList = async (dir: string) => { + const files = await fsp.readdir(dir); + const toPath = (name: string) => join(dir, name); + for (const file of files) console.log(toPath(file)); +}; + +export { logger, showLogsList }; diff --git a/packages/core/lib/logger.ts b/packages/core/lib/logger.ts index c35cfc2..2cca556 100644 --- a/packages/core/lib/logger.ts +++ b/packages/core/lib/logger.ts @@ -3,97 +3,114 @@ import { join } from 'node:path'; import { Stats, WriteStream, createWriteStream, promises as fsp } from 'node:fs'; import pino from 'pino'; import pretty from 'pino-pretty'; -import { BaseDirectory } from './fs'; -import { getCurrentDateTimeString, isExecutable } from './utils'; -const MAX_LOGS_COUNT = 50; -const CURRENT_DATETIME = getCurrentDateTimeString(); -const LOG_DIR = BaseDirectory.AppLog; -const LOG_PATH = join(LOG_DIR, `${CURRENT_DATETIME}_${pid}.log`); - -// Enable debug mode if needed -if ( - process.argv.includes('-d') || - process.argv.includes('--debug') || - process.env.NODE_ENV_ELECTRON_VITE === 'development' || - !!process.env.ELECTRON_CLI_ARGS -) { - process.env.DEBUG = 'streamyx:*'; -} - -// https://github.com/pinojs/pino/issues/1722 -// Use native Node.js streams on Windows due to lack of Cyrillic support in pino's sonic-boom -const isWindows = process.platform === 'win32'; -const prettyDestination = isWindows ? process.stdout : undefined; - -const level = process.env.DEBUG?.startsWith('streamyx') ? 'debug' : 'info'; - -type LogStream = { level: string; stream: WriteStream | any }; - -const streams: LogStream[] = [ - { - level, - stream: pretty({ - colorize: true, - sync: true, - translateTime: 'SYS:HH:MM:ss.l', - customPrettifiers: { - time: (timestamp) => `${timestamp}`, - level: (_logLevel, _key, _log, { labelColorized }: any) => `${labelColorized}`.padEnd(15, ' '), - }, - destination: prettyDestination, - messageFormat: (log, messageKey, _levelLabel, { colors }) => { - const message = log[messageKey]; - if (typeof message === 'string') return colors.whiteBright(message); - else return message as string; - }, - }), - }, -]; - -const clearOutdatedLogs = async () => { - const files = await fsp.readdir(LOG_DIR); - const toPath = (name: string) => join(LOG_DIR, name); - const statsQueue = files.map((name: string) => - fsp - .stat(toPath(name)) - .catch(() => null) - .then((stats) => ({ stats, path: toPath(name) })) - ); - const results = await Promise.all(statsQueue); - const stats = results.filter(Boolean) as { stats: Stats; path: string }[]; - stats.sort((a, b) => b.stats.mtime.getTime() - a.stats.mtime.getTime()); - const oldLogs = stats.slice(MAX_LOGS_COUNT); - const deleteQueue = oldLogs.map((log) => fsp.unlink(log.path)); - await Promise.allSettled(deleteQueue).catch(logger.error); +const dateTimeFormatter = new Intl.DateTimeFormat('en-US', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, +}); + +const getCurrentDateTimeString = () => { + const date = new Date(); + const formatted = dateTimeFormatter.format(date); + return formatted.replaceAll('/', '-').replace(', ', '_').replaceAll(':', '-'); }; -export const showLogsList = async () => { - const files = await fsp.readdir(LOG_DIR); - const toPath = (name: string) => join(LOG_DIR, name); - for (const file of files) console.log(toPath(file)); +const MAX_LOGS_COUNT = 50; + +type CreateLoggerOptions = { + dir?: string; + writeToFile?: boolean; }; -// If we're inside executable, then also use file logging -if (isExecutable) { - clearOutdatedLogs(); +const createLogger = (options: CreateLoggerOptions = {}) => { + const dateTimeString = getCurrentDateTimeString(); + const logDir = options.dir || process.cwd(); + const logPath = join(logDir, `${dateTimeString}_${pid}.log`); + + // Enable debug mode if needed + if ( + process.argv.includes('-d') || + process.argv.includes('--debug') || + process.env.NODE_ENV_ELECTRON_VITE === 'development' || + !!process.env.ELECTRON_CLI_ARGS + ) { + process.env.DEBUG = 'streamyx:*'; + } - const fileStream = isWindows - ? createWriteStream(LOG_PATH, { flags: 'a' }) - : pino.destination({ dest: LOG_PATH, append: true, mkdir: true }); + // https://github.com/pinojs/pino/issues/1722 + // Use native Node.js streams on Windows due to lack of Cyrillic support in pino's sonic-boom + const isWindows = process.platform === 'win32'; + const prettyDestination = isWindows ? process.stdout : undefined; - streams.unshift({ level: 'debug', stream: fileStream }); -} + const level = process.env.DEBUG?.startsWith('streamyx') ? 'debug' : 'info'; -const logger = pino( - { - level: 'debug', - formatters: { - bindings: () => ({}), + type LogStream = { level: string; stream: WriteStream | any }; + + const streams: LogStream[] = [ + { + level, + stream: pretty({ + colorize: true, + sync: true, + translateTime: 'SYS:HH:MM:ss.l', + customPrettifiers: { + time: (timestamp) => `${timestamp}`, + level: (_logLevel, _key, _log, { labelColorized }: any) => `${labelColorized}`.padEnd(15, ' '), + }, + destination: prettyDestination, + messageFormat: (log, messageKey, _levelLabel, { colors }) => { + const message = log[messageKey]; + if (typeof message === 'string') return colors.whiteBright(message); + else return message as string; + }, + }), }, - timestamp: pino.stdTimeFunctions.isoTime, - }, - pino.multistream(streams) -); + ]; + + const clearOutdatedLogs = async () => { + const files = await fsp.readdir(logDir); + const toPath = (name: string) => join(logDir, name); + const statsQueue = files.map((name: string) => + fsp + .stat(toPath(name)) + .catch(() => null) + .then((stats) => ({ stats, path: toPath(name) })) + ); + const results = await Promise.all(statsQueue); + const stats = results.filter(Boolean) as { stats: Stats; path: string }[]; + stats.sort((a, b) => b.stats.mtime.getTime() - a.stats.mtime.getTime()); + const oldLogs = stats.slice(MAX_LOGS_COUNT); + const deleteQueue = oldLogs.map((log) => fsp.unlink(log.path)); + await Promise.allSettled(deleteQueue).catch(logger.error); + }; + + if (options.writeToFile) { + clearOutdatedLogs(); + + const fileStream = isWindows + ? createWriteStream(logPath, { flags: 'a' }) + : pino.destination({ dest: logPath, append: true, mkdir: true }); + + streams.unshift({ level: 'debug', stream: fileStream }); + } + + const logger = pino( + { + level: 'debug', + formatters: { + bindings: () => ({}), + }, + timestamp: pino.stdTimeFunctions.isoTime, + }, + pino.multistream(streams) + ); + + return logger; +}; -export { logger, LOG_DIR, LOG_PATH }; +export { createLogger }; diff --git a/packages/core/lib/main.ts b/packages/core/lib/main.ts index 2e1cc9c..711f663 100644 --- a/packages/core/lib/main.ts +++ b/packages/core/lib/main.ts @@ -1,8 +1,8 @@ +export * from './log'; export * from './fs'; export * from './http'; export * from './cookies'; export * from './settings'; -export * from './logger'; export * from './bin'; export * from './prompt'; export * from './store'; diff --git a/packages/core/lib/service.ts b/packages/core/lib/service.ts index eecb4b2..a9725c2 100644 --- a/packages/core/lib/service.ts +++ b/packages/core/lib/service.ts @@ -4,7 +4,7 @@ import { IPrompt } from './prompt'; import { createStore } from './store'; import { Http, http } from './http'; import { fs } from './fs'; -import { logger as log } from './logger'; +import { logger as log } from './log'; import { prompt } from './prompt'; import { execUrlPatterns, sanitizeString, safeEval, extendEpisodes } from './utils'; diff --git a/packages/core/lib/settings.ts b/packages/core/lib/settings.ts index 3ded298..350c9ff 100644 --- a/packages/core/lib/settings.ts +++ b/packages/core/lib/settings.ts @@ -1,6 +1,6 @@ import { join } from 'node:path'; import { fs, BaseDirectory, initDir } from './fs'; -import { logger } from './logger'; +import { logger } from './log'; export type VideoQuality = (typeof VIDEO_QUALITY)[keyof typeof VIDEO_QUALITY]; export const VIDEO_QUALITY = { diff --git a/packages/core/package.json b/packages/core/package.json index 1305dc0..191c760 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@streamyx/core", - "version": "2.2.0", + "version": "2.2.1", "author": "Vitaly Gashkov ", "description": "Core modules for Streamyx", "license": "AGPL-3.0", @@ -25,8 +25,6 @@ "dependencies": { "dasha": "^3.1.1", "got-scraping": "^3.2.15", - "pino": "^9.5.0", - "pino-pretty": "^11.3.0", "undici": "^6.21.0" }, "devDependencies": { diff --git a/packages/logger/.gitignore b/packages/logger/.gitignore new file mode 100644 index 0000000..becd09b --- /dev/null +++ b/packages/logger/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +build +.DS_Store diff --git a/packages/logger/LICENSE b/packages/logger/LICENSE new file mode 100644 index 0000000..6ed8df4 --- /dev/null +++ b/packages/logger/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Vitaly Gashkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/logger/eslint.config.mjs b/packages/logger/eslint.config.mjs new file mode 100644 index 0000000..96072c8 --- /dev/null +++ b/packages/logger/eslint.config.mjs @@ -0,0 +1,10 @@ +import globals from 'globals'; +import pluginJs from '@eslint/js'; +import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended'; + +export default [ + { files: ['**/*.js'], languageOptions: { sourceType: 'commonjs' } }, + { languageOptions: { globals: globals.node } }, + pluginJs.configs.recommended, + eslintPluginPrettierRecommended, +]; diff --git a/packages/logger/index.d.ts b/packages/logger/index.d.ts new file mode 100644 index 0000000..8066388 --- /dev/null +++ b/packages/logger/index.d.ts @@ -0,0 +1,10 @@ +import { Logger } from 'pino'; + +type CreateLoggerOptions = { + dir?: string; + writeToFile?: boolean; +}; + +export function createLogger( + options?: LaunchBrowserOptions, +): Logger; diff --git a/packages/logger/index.js b/packages/logger/index.js new file mode 100644 index 0000000..c852ef3 --- /dev/null +++ b/packages/logger/index.js @@ -0,0 +1,110 @@ +const { pid } = require('node:process'); +const { join } = require('node:path'); +const { createWriteStream } = require('node:fs'); +const { readdir, stat, unlink } = require('node:fs/promises'); +const pino = require('pino'); +const pretty = require('pino-pretty'); + +const dateTimeFormatter = new Intl.DateTimeFormat('en-US', { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, +}); + +const getCurrentDateTimeString = () => { + const date = new Date(); + const formatted = dateTimeFormatter.format(date); + return formatted.replaceAll('/', '-').replace(', ', '_').replaceAll(':', '-'); +}; + +const MAX_LOGS_COUNT = 50; + +const createLogger = (options = {}) => { + const dateTimeString = getCurrentDateTimeString(); + const logDir = options.dir || process.cwd(); + const logPath = join(logDir, `${dateTimeString}_${pid}.log`); + + // Enable debug mode if needed + if ( + process.argv.includes('-d') || + process.argv.includes('--debug') || + process.env.NODE_ENV_ELECTRON_VITE === 'development' || + !!process.env.ELECTRON_CLI_ARGS + ) { + process.env.DEBUG = 'streamyx:*'; + } + + // https://github.com/pinojs/pino/issues/1722 + // Use native Node.js streams on Windows due to lack of Cyrillic support in pino's sonic-boom + const isWindows = process.platform === 'win32'; + const prettyDestination = isWindows ? process.stdout : undefined; + + const level = process.env.DEBUG?.startsWith('streamyx') ? 'debug' : 'info'; + + const streams = [ + { + level, + stream: pretty({ + colorize: true, + sync: true, + translateTime: 'SYS:HH:MM:ss.l', + customPrettifiers: { + time: (timestamp) => `${timestamp}`, + level: (_logLevel, _key, _log, { labelColorized }) => + `${labelColorized}`.padEnd(15, ' '), + }, + destination: prettyDestination, + messageFormat: (log, messageKey, _levelLabel, { colors }) => { + const message = log[messageKey]; + if (typeof message === 'string') return colors.whiteBright(message); + else return message; + }, + }), + }, + ]; + + const clearOutdatedLogs = async () => { + const files = await readdir(logDir); + const toPath = (name) => join(logDir, name); + const statsQueue = files.map((name) => + stat(toPath(name)) + .catch(() => null) + .then((stats) => ({ stats, path: toPath(name) })), + ); + const results = await Promise.all(statsQueue); + const stats = results.filter(Boolean); + stats.sort((a, b) => b.stats.mtime.getTime() - a.stats.mtime.getTime()); + const oldLogs = stats.slice(MAX_LOGS_COUNT); + const deleteQueue = oldLogs.map((log) => unlink(log.path)); + await Promise.allSettled(deleteQueue).catch(logger.error); + }; + + if (options.writeToFile) { + clearOutdatedLogs(); + + const fileStream = isWindows + ? createWriteStream(logPath, { flags: 'a' }) + : pino.destination({ dest: logPath, append: true, mkdir: true }); + + streams.unshift({ level: 'debug', stream: fileStream }); + } + + const logger = pino( + { + level: 'debug', + formatters: { + bindings: () => ({}), + }, + timestamp: pino.stdTimeFunctions.isoTime, + }, + pino.multistream(streams), + ); + + return logger; +}; + +module.exports = { createLogger }; diff --git a/packages/logger/package.json b/packages/logger/package.json new file mode 100644 index 0000000..0e2bceb --- /dev/null +++ b/packages/logger/package.json @@ -0,0 +1,58 @@ +{ + "name": "@streamyx/logger", + "version": "0.0.1", + "description": "Logger for Streamyx", + "main": "index.js", + "types": "index.d.ts", + "files": [ + "index.js", + "index.d.ts" + ], + "scripts": { + "lint": "eslint . && prettier --check .", + "fix": "eslint . --fix && prettier --write ." + }, + "repository": { + "type": "git", + "url": "git+https://github.com/vitalygashkov/streamyx.git" + }, + "keywords": [ + "streamyx", + "puppeteer", + "browser" + ], + "bugs": { + "url": "https://github.com/vitalygashkov/streamyx/issues", + "email": "vitalygashkov@vk.com" + }, + "author": "Vitaly Gashkov ", + "license": "MIT", + "readmeFilename": "README.md", + "funding": [ + { + "type": "individual", + "url": "https://boosty.to/vitalygashkov" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/vitalygashkov" + } + ], + "engines": { + "node": "20 || 21 || 22 || 23" + }, + "dependencies": { + "pino": "^9.5.0", + "pino-pretty": "^13.0.0" + }, + "devDependencies": { + "@eslint/js": "^9.16.0", + "@types/node": "^22.10.1", + "eslint": "^9.16.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.2.1", + "globals": "^15.12.0", + "prettier": "^3.4.1", + "typescript": "^5.7.2" + } +} diff --git a/packages/logger/prettier.config.mjs b/packages/logger/prettier.config.mjs new file mode 100644 index 0000000..0edf463 --- /dev/null +++ b/packages/logger/prettier.config.mjs @@ -0,0 +1,8 @@ +export default { + printWidth: 80, + singleQuote: true, + trailingComma: 'all', + tabWidth: 2, + useTabs: false, + semi: true, +}; From f4da16f7ad7ac12886aac545d490eb816e53f28f Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 16:00:05 +0500 Subject: [PATCH 4/6] chore: fix keywords for logger --- package-lock.json | 2 +- packages/core/lib/logger.ts | 116 ----------------------------------- packages/logger/package.json | 6 +- 3 files changed, 4 insertions(+), 120 deletions(-) delete mode 100644 packages/core/lib/logger.ts diff --git a/package-lock.json b/package-lock.json index 09e066f..a0ba468 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20283,7 +20283,7 @@ }, "packages/logger": { "name": "@streamyx/logger", - "version": "0.0.1", + "version": "0.0.2", "funding": [ { "type": "individual", diff --git a/packages/core/lib/logger.ts b/packages/core/lib/logger.ts deleted file mode 100644 index 2cca556..0000000 --- a/packages/core/lib/logger.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { pid } from 'node:process'; -import { join } from 'node:path'; -import { Stats, WriteStream, createWriteStream, promises as fsp } from 'node:fs'; -import pino from 'pino'; -import pretty from 'pino-pretty'; - -const dateTimeFormatter = new Intl.DateTimeFormat('en-US', { - year: 'numeric', - month: '2-digit', - day: '2-digit', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - hour12: false, -}); - -const getCurrentDateTimeString = () => { - const date = new Date(); - const formatted = dateTimeFormatter.format(date); - return formatted.replaceAll('/', '-').replace(', ', '_').replaceAll(':', '-'); -}; - -const MAX_LOGS_COUNT = 50; - -type CreateLoggerOptions = { - dir?: string; - writeToFile?: boolean; -}; - -const createLogger = (options: CreateLoggerOptions = {}) => { - const dateTimeString = getCurrentDateTimeString(); - const logDir = options.dir || process.cwd(); - const logPath = join(logDir, `${dateTimeString}_${pid}.log`); - - // Enable debug mode if needed - if ( - process.argv.includes('-d') || - process.argv.includes('--debug') || - process.env.NODE_ENV_ELECTRON_VITE === 'development' || - !!process.env.ELECTRON_CLI_ARGS - ) { - process.env.DEBUG = 'streamyx:*'; - } - - // https://github.com/pinojs/pino/issues/1722 - // Use native Node.js streams on Windows due to lack of Cyrillic support in pino's sonic-boom - const isWindows = process.platform === 'win32'; - const prettyDestination = isWindows ? process.stdout : undefined; - - const level = process.env.DEBUG?.startsWith('streamyx') ? 'debug' : 'info'; - - type LogStream = { level: string; stream: WriteStream | any }; - - const streams: LogStream[] = [ - { - level, - stream: pretty({ - colorize: true, - sync: true, - translateTime: 'SYS:HH:MM:ss.l', - customPrettifiers: { - time: (timestamp) => `${timestamp}`, - level: (_logLevel, _key, _log, { labelColorized }: any) => `${labelColorized}`.padEnd(15, ' '), - }, - destination: prettyDestination, - messageFormat: (log, messageKey, _levelLabel, { colors }) => { - const message = log[messageKey]; - if (typeof message === 'string') return colors.whiteBright(message); - else return message as string; - }, - }), - }, - ]; - - const clearOutdatedLogs = async () => { - const files = await fsp.readdir(logDir); - const toPath = (name: string) => join(logDir, name); - const statsQueue = files.map((name: string) => - fsp - .stat(toPath(name)) - .catch(() => null) - .then((stats) => ({ stats, path: toPath(name) })) - ); - const results = await Promise.all(statsQueue); - const stats = results.filter(Boolean) as { stats: Stats; path: string }[]; - stats.sort((a, b) => b.stats.mtime.getTime() - a.stats.mtime.getTime()); - const oldLogs = stats.slice(MAX_LOGS_COUNT); - const deleteQueue = oldLogs.map((log) => fsp.unlink(log.path)); - await Promise.allSettled(deleteQueue).catch(logger.error); - }; - - if (options.writeToFile) { - clearOutdatedLogs(); - - const fileStream = isWindows - ? createWriteStream(logPath, { flags: 'a' }) - : pino.destination({ dest: logPath, append: true, mkdir: true }); - - streams.unshift({ level: 'debug', stream: fileStream }); - } - - const logger = pino( - { - level: 'debug', - formatters: { - bindings: () => ({}), - }, - timestamp: pino.stdTimeFunctions.isoTime, - }, - pino.multistream(streams) - ); - - return logger; -}; - -export { createLogger }; diff --git a/packages/logger/package.json b/packages/logger/package.json index 0e2bceb..a1762f9 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@streamyx/logger", - "version": "0.0.1", + "version": "0.0.2", "description": "Logger for Streamyx", "main": "index.js", "types": "index.d.ts", @@ -18,8 +18,8 @@ }, "keywords": [ "streamyx", - "puppeteer", - "browser" + "logger", + "pino" ], "bugs": { "url": "https://github.com/vitalygashkov/streamyx/issues", From 2f5dca09a00cc607688d3a9c863d5bca6584d675 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 16:01:11 +0500 Subject: [PATCH 5/6] fix: use logger from separate package in core --- package-lock.json | 1 + packages/core/lib/log.ts | 2 +- packages/core/package.json | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index a0ba468..873efa4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19974,6 +19974,7 @@ "version": "2.2.1", "license": "AGPL-3.0", "dependencies": { + "@streamyx/logger": "^0.0.2", "dasha": "^3.1.1", "got-scraping": "^3.2.15", "undici": "^6.21.0" diff --git a/packages/core/lib/log.ts b/packages/core/lib/log.ts index 19f3f02..20a6434 100644 --- a/packages/core/lib/log.ts +++ b/packages/core/lib/log.ts @@ -1,7 +1,7 @@ import { join } from 'node:path'; import { promises as fsp } from 'node:fs'; +import { createLogger } from '@streamyx/logger'; import { BaseDirectory } from './fs'; -import { createLogger } from './logger'; import { isExecutable } from './utils'; const logger = createLogger({ dir: BaseDirectory.AppLog, writeToFile: isExecutable }); diff --git a/packages/core/package.json b/packages/core/package.json index 191c760..496b6ed 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -23,6 +23,7 @@ "dev": "tsup ./lib/main.ts --format esm,cjs --watch --dts --clean" }, "dependencies": { + "@streamyx/logger": "^0.0.2", "dasha": "^3.1.1", "got-scraping": "^3.2.15", "undici": "^6.21.0" From d7d0a01549f99fd3ec3406cb4b4ef6999d807975 Mon Sep 17 00:00:00 2001 From: Vitaly Gashkov Date: Sat, 30 Nov 2024 16:39:03 +0500 Subject: [PATCH 6/6] update submodule --- apps/cli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli b/apps/cli index 2a6bfef..44150b1 160000 --- a/apps/cli +++ b/apps/cli @@ -1 +1 @@ -Subproject commit 2a6bfef00092e230940634eac2b02b77ff4bde6f +Subproject commit 44150b1d5ba6e2267d299240e44f5552f0f30bc1