diff --git a/docs/docs/integrations/text_embedding/voyageai.mdx b/docs/docs/integrations/text_embedding/voyageai.mdx new file mode 100644 index 000000000000..7e413c0386a3 --- /dev/null +++ b/docs/docs/integrations/text_embedding/voyageai.mdx @@ -0,0 +1,11 @@ +# Voyage AI + +The `VoyageEmbeddings` class uses the Voyage AI REST API to generate embeddings for a given text. + +```typescript +import { VoyageEmbeddings } from "langchain/embeddings/voyage"; + +const embeddings = new VoyageEmbeddings({ + apiKey: "YOUR-API-KEY", // In Node.js defaults to process.env.VOYAGEAI_API_KEY +}); +``` diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 57637ddb516b..910e629351d5 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake"; export * from "langchain/embeddings/ollama"; export * from "langchain/embeddings/openai"; export * from "langchain/embeddings/minimax"; +export * from "langchain/embeddings/voyage"; export * from "langchain/llms/base"; export * from "langchain/llms/openai"; export * from "langchain/llms/ai21"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 57637ddb516b..910e629351d5 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake"; export * from "langchain/embeddings/ollama"; export * from "langchain/embeddings/openai"; export * from "langchain/embeddings/minimax"; +export * from "langchain/embeddings/voyage"; export * from "langchain/llms/base"; export * from "langchain/llms/openai"; export * from "langchain/llms/ai21"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 87cd8f1e2240..5d901d5f1d8a 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -21,6 +21,7 @@ const embeddings_fake = require("langchain/embeddings/fake"); const embeddings_ollama = require("langchain/embeddings/ollama"); const embeddings_openai = require("langchain/embeddings/openai"); const embeddings_minimax = require("langchain/embeddings/minimax"); +const embeddings_voyage = require("langchain/embeddings/voyage"); const llms_base = require("langchain/llms/base"); const llms_openai = require("langchain/llms/openai"); const llms_ai21 = require("langchain/llms/ai21"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index 8d5ba32b04d3..da609092add0 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -21,6 +21,7 @@ import * as embeddings_fake from "langchain/embeddings/fake"; import * as embeddings_ollama from "langchain/embeddings/ollama"; import * as embeddings_openai from "langchain/embeddings/openai"; import * as embeddings_minimax from "langchain/embeddings/minimax"; +import * as embeddings_voyage from "langchain/embeddings/voyage"; import * as llms_base from "langchain/llms/base"; import * as llms_openai from "langchain/llms/openai"; import * as llms_ai21 from "langchain/llms/ai21"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index 8d5ba32b04d3..da609092add0 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -21,6 +21,7 @@ import * as embeddings_fake from "langchain/embeddings/fake"; import * as embeddings_ollama from "langchain/embeddings/ollama"; import * as embeddings_openai from "langchain/embeddings/openai"; import * as embeddings_minimax from "langchain/embeddings/minimax"; +import * as embeddings_voyage from "langchain/embeddings/voyage"; import * as llms_base from "langchain/llms/base"; import * as llms_openai from "langchain/llms/openai"; import * as llms_ai21 from "langchain/llms/ai21"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 57637ddb516b..910e629351d5 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake"; export * from "langchain/embeddings/ollama"; export * from "langchain/embeddings/openai"; export * from "langchain/embeddings/minimax"; +export * from "langchain/embeddings/voyage"; export * from "langchain/llms/base"; export * from "langchain/llms/openai"; export * from "langchain/llms/ai21"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 57637ddb516b..910e629351d5 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -21,6 +21,7 @@ export * from "langchain/embeddings/fake"; export * from "langchain/embeddings/ollama"; export * from "langchain/embeddings/openai"; export * from "langchain/embeddings/minimax"; +export * from "langchain/embeddings/voyage"; export * from "langchain/llms/base"; export * from "langchain/llms/openai"; export * from "langchain/llms/ai21"; diff --git a/langchain/.eslintrc.cjs b/langchain/.eslintrc.cjs index 01d949d5df89..bab396aedde2 100644 --- a/langchain/.eslintrc.cjs +++ b/langchain/.eslintrc.cjs @@ -15,6 +15,7 @@ module.exports = { ignorePatterns: [ "src/util/@cfworker", "src/util/fast-json-patch", + "src/util/js-sha1", ".eslintrc.cjs", "scripts", "node_modules", diff --git a/langchain/.gitignore b/langchain/.gitignore index 3ebfca11f7b8..3a1dbb7ed6d2 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -133,6 +133,9 @@ embeddings/googlepalm.d.ts embeddings/minimax.cjs embeddings/minimax.js embeddings/minimax.d.ts +embeddings/voyage.cjs +embeddings/voyage.js +embeddings/voyage.d.ts embeddings/llama_cpp.cjs embeddings/llama_cpp.js embeddings/llama_cpp.d.ts diff --git a/langchain/package.json b/langchain/package.json index 00bdacabb07b..7e0cab4a6d8e 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -1,6 +1,6 @@ { "name": "langchain", - "version": "0.0.180", + "version": "0.0.181", "description": "Typescript bindings for langchain", "type": "module", "engines": { @@ -145,6 +145,9 @@ "embeddings/minimax.cjs", "embeddings/minimax.js", "embeddings/minimax.d.ts", + "embeddings/voyage.cjs", + "embeddings/voyage.js", + "embeddings/voyage.d.ts", "embeddings/llama_cpp.cjs", "embeddings/llama_cpp.js", "embeddings/llama_cpp.d.ts", @@ -867,7 +870,6 @@ "@types/jsdom": "^21.1.1", "@types/lodash": "^4", "@types/mozilla-readability": "^0.2.1", - "@types/object-hash": "^3.0.2", "@types/pdf-parse": "^1.1.1", "@types/pg": "^8", "@types/pg-copy-streams": "^1.2.2", @@ -1360,7 +1362,6 @@ "langchainhub": "~0.0.6", "langsmith": "~0.0.48", "ml-distance": "^4.0.0", - "object-hash": "^3.0.0", "openai": "~4.4.0", "openapi-types": "^12.1.3", "p-queue": "^6.6.2", @@ -1613,6 +1614,11 @@ "import": "./embeddings/minimax.js", "require": "./embeddings/minimax.cjs" }, + "./embeddings/voyage": { + "types": "./embeddings/voyage.d.ts", + "import": "./embeddings/voyage.js", + "require": "./embeddings/voyage.cjs" + }, "./embeddings/llama_cpp": { "types": "./embeddings/llama_cpp.d.ts", "import": "./embeddings/llama_cpp.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index 8f17d6b4d97d..8506518b01ee 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -59,6 +59,7 @@ const entrypoints = { "embeddings/googlevertexai": "embeddings/googlevertexai", "embeddings/googlepalm": "embeddings/googlepalm", "embeddings/minimax": "embeddings/minimax", + "embeddings/voyage": "embeddings/voyage", "embeddings/llama_cpp": "embeddings/llama_cpp", // llms "llms/load": "llms/load", diff --git a/langchain/src/cache/base.ts b/langchain/src/cache/base.ts index 6114cf65b4b5..41a44f15e282 100644 --- a/langchain/src/cache/base.ts +++ b/langchain/src/cache/base.ts @@ -1,4 +1,4 @@ -import hash from "object-hash"; +import { insecureHash } from "../util/js-sha1/hash.js"; import { ChatGeneration, @@ -18,7 +18,7 @@ import { * TODO: Make cache key consistent across versions of langchain. */ export const getCacheKey = (...strings: string[]): string => - hash(strings.join("_")); + insecureHash(strings.join("_")); export function deserializeStoredGeneration( storedGeneration: StoredGeneration diff --git a/langchain/src/cache/tests/ioredis.test.ts b/langchain/src/cache/tests/ioredis.test.ts index 59d68716ec28..60f40906b50b 100644 --- a/langchain/src/cache/tests/ioredis.test.ts +++ b/langchain/src/cache/tests/ioredis.test.ts @@ -1,14 +1,14 @@ import { test, expect, jest } from "@jest/globals"; -import hash from "object-hash"; +import { insecureHash } from "../../util/js-sha1/hash.js"; import { RedisCache } from "../ioredis.js"; -const sha256 = (str: string) => hash(str); +const sha1 = (str: string) => insecureHash(str); test("RedisCache", async () => { const redis = { get: jest.fn(async (key: string) => { - if (key === sha256("foo_bar_0")) { + if (key === sha1("foo_bar_0")) { return JSON.stringify({ text: "baz" }); } return null; diff --git a/langchain/src/cache/tests/redis.test.ts b/langchain/src/cache/tests/redis.test.ts index 59469b38c45c..eb53b0fe1b92 100644 --- a/langchain/src/cache/tests/redis.test.ts +++ b/langchain/src/cache/tests/redis.test.ts @@ -1,14 +1,14 @@ import { test, expect, jest } from "@jest/globals"; -import hash from "object-hash"; +import { insecureHash } from "../../util/js-sha1/hash.js"; import { RedisCache } from "../redis.js"; -const sha256 = (str: string) => hash(str); +const sha1 = (str: string) => insecureHash(str); test("RedisCache", async () => { const redis = { get: jest.fn(async (key: string) => { - if (key === sha256("foo_bar_0")) { + if (key === sha1("foo_bar_0")) { return JSON.stringify({ text: "baz" }); } return null; diff --git a/langchain/src/cache/tests/upstash_redis.test.ts b/langchain/src/cache/tests/upstash_redis.test.ts index f79644a3f3c5..caa41903d337 100644 --- a/langchain/src/cache/tests/upstash_redis.test.ts +++ b/langchain/src/cache/tests/upstash_redis.test.ts @@ -1,15 +1,15 @@ import { test, expect, jest } from "@jest/globals"; -import hash from "object-hash"; +import { insecureHash } from "../../util/js-sha1/hash.js"; import { UpstashRedisCache } from "../upstash_redis.js"; import { StoredGeneration } from "../../schema/index.js"; -const sha256 = (str: string) => hash(str); +const sha1 = (str: string) => insecureHash(str); test("UpstashRedisCache", async () => { const redis = { get: jest.fn(async (key: string): Promise => { - if (key === sha256("foo_bar_0")) { + if (key === sha1("foo_bar_0")) { return { text: "baz" }; } return null; diff --git a/langchain/src/embeddings/cache_backed.ts b/langchain/src/embeddings/cache_backed.ts index ae4ac3cc6a2b..126a49f1e633 100644 --- a/langchain/src/embeddings/cache_backed.ts +++ b/langchain/src/embeddings/cache_backed.ts @@ -1,4 +1,4 @@ -import hash from "object-hash"; +import { insecureHash } from "../util/js-sha1/hash.js"; import { BaseStore } from "../schema/storage.js"; import { EncoderBackedStore } from "../storage/encoder_backed.js"; @@ -112,7 +112,7 @@ export class CacheBackedEmbeddings extends Embeddings { Uint8Array >({ store: documentEmbeddingStore, - keyEncoder: (key) => (options?.namespace ?? "") + hash(key), + keyEncoder: (key) => (options?.namespace ?? "") + insecureHash(key), valueSerializer: (value) => encoder.encode(JSON.stringify(value)), valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue)), diff --git a/langchain/src/embeddings/tests/voyage.int.test.ts b/langchain/src/embeddings/tests/voyage.int.test.ts new file mode 100644 index 000000000000..1e6259a474ea --- /dev/null +++ b/langchain/src/embeddings/tests/voyage.int.test.ts @@ -0,0 +1,36 @@ +import { test, expect } from "@jest/globals"; +import { VoyageEmbeddings } from "../voyage.js"; + +test.skip("Test VoyageEmbeddings.embedQuery", async () => { + const embeddings = new VoyageEmbeddings(); + const res = await embeddings.embedQuery("Hello world"); + + expect(typeof res[0]).toBe("number"); +}); + +test.skip("Test VoyageEmbeddings.embedDocuments", async () => { + const embeddings = new VoyageEmbeddings(); + const res = await embeddings.embedDocuments(["Hello world", "Bye bye"]); + expect(res).toHaveLength(2); + expect(typeof res[0][0]).toBe("number"); + expect(typeof res[1][0]).toBe("number"); +}); + +test.skip("Test VoyageEmbeddings concurrency", async () => { + const embeddings = new VoyageEmbeddings({ + batchSize: 1, + maxConcurrency: 2, + }); + const res = await embeddings.embedDocuments([ + "Hello world", + "Bye bye", + "Hello world", + "Bye bye", + "Hello world", + "Bye bye", + ]); + expect(res).toHaveLength(6); + expect(res.find((embedding) => typeof embedding[0] !== "number")).toBe( + undefined + ); +}); diff --git a/langchain/src/embeddings/voyage.ts b/langchain/src/embeddings/voyage.ts new file mode 100644 index 000000000000..6b4d03c7e210 --- /dev/null +++ b/langchain/src/embeddings/voyage.ts @@ -0,0 +1,152 @@ +import { chunkArray } from "../util/chunk.js"; +import { getEnvironmentVariable } from "../util/env.js"; +import { Embeddings, EmbeddingsParams } from "./base.js"; + +/** + * Interface that extends EmbeddingsParams and defines additional + * parameters specific to the VoyageEmbeddings class. + */ +export interface VoyageEmbeddingsParams extends EmbeddingsParams { + modelName: string; + + /** + * The maximum number of documents to embed in a single request. This is + * limited by the Voyage AI API to a maximum of 8. + */ + batchSize?: number; +} + +/** + * Interface for the request body to generate embeddings. + */ +export interface CreateVoyageEmbeddingRequest { + /** + * @type {string} + * @memberof CreateVoyageEmbeddingRequest + */ + model: string; + + /** + * Text to generate vector expectation + * @type {CreateEmbeddingRequestInput} + * @memberof CreateVoyageEmbeddingRequest + */ + input: string | string[]; +} + +/** + * A class for generating embeddings using the Voyage AI API. + */ +export class VoyageEmbeddings + extends Embeddings + implements VoyageEmbeddingsParams +{ + modelName = "voyage-01"; + + batchSize = 8; + + private apiKey: string; + + basePath?: string = "https://api.voyageai.com/v1"; + + apiUrl: string; + + headers?: Record; + + /** + * Constructor for the VoyageEmbeddings class. + * @param fields - An optional object with properties to configure the instance. + */ + constructor( + fields?: Partial & { + verbose?: boolean; + apiKey?: string; + } + ) { + const fieldsWithDefaults = { ...fields }; + + super(fieldsWithDefaults); + + const apiKey = + fieldsWithDefaults?.apiKey || getEnvironmentVariable("VOYAGEAI_API_KEY"); + + if (!apiKey) { + throw new Error("Voyage AI API key not found"); + } + + this.modelName = fieldsWithDefaults?.modelName ?? this.modelName; + this.batchSize = fieldsWithDefaults?.batchSize ?? this.batchSize; + this.apiKey = apiKey; + this.apiUrl = `${this.basePath}/embeddings`; + } + + /** + * Generates embeddings for an array of texts. + * @param texts - An array of strings to generate embeddings for. + * @returns A Promise that resolves to an array of embeddings. + */ + async embedDocuments(texts: string[]): Promise { + const batches = chunkArray(texts, this.batchSize); + + const batchRequests = batches.map((batch) => + this.embeddingWithRetry({ + model: this.modelName, + input: batch, + }) + ); + + const batchResponses = await Promise.all(batchRequests); + + const embeddings: number[][] = []; + + for (let i = 0; i < batchResponses.length; i += 1) { + const batch = batches[i]; + const { data: batchResponse } = batchResponses[i]; + for (let j = 0; j < batch.length; j += 1) { + embeddings.push(batchResponse[j].embedding); + } + } + + return embeddings; + } + + /** + * Generates an embedding for a single text. + * @param text - A string to generate an embedding for. + * @returns A Promise that resolves to an array of numbers representing the embedding. + */ + async embedQuery(text: string): Promise { + const { data } = await this.embeddingWithRetry({ + model: this.modelName, + input: text, + }); + + return data[0].embedding; + } + + /** + * Makes a request to the Voyage AI API to generate embeddings for an array of texts. + * @param request - An object with properties to configure the request. + * @returns A Promise that resolves to the response from the Voyage AI API. + */ + + private async embeddingWithRetry(request: CreateVoyageEmbeddingRequest) { + const makeCompletionRequest = async () => { + const url = `${this.apiUrl}`; + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + ...this.headers, + }, + body: JSON.stringify(request), + }); + + const json = await response.json(); + return json; + }; + + return this.caller.call(makeCompletionRequest); + } +} diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index 33e3fe3704b8..c31a66622d73 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -22,6 +22,7 @@ export * as embeddings__fake from "../embeddings/fake.js"; export * as embeddings__ollama from "../embeddings/ollama.js"; export * as embeddings__openai from "../embeddings/openai.js"; export * as embeddings__minimax from "../embeddings/minimax.js"; +export * as embeddings__voyage from "../embeddings/voyage.js"; export * as llms__base from "../llms/base.js"; export * as llms__openai from "../llms/openai.js"; export * as llms__ai21 from "../llms/ai21.js"; diff --git a/langchain/src/util/js-sha1/LICENSE.md b/langchain/src/util/js-sha1/LICENSE.md new file mode 100644 index 000000000000..74ac20d909c7 --- /dev/null +++ b/langchain/src/util/js-sha1/LICENSE.md @@ -0,0 +1,20 @@ +Copyright 2014-2017 Chen, Yi-Cyuan + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/langchain/src/util/js-sha1/hash.ts b/langchain/src/util/js-sha1/hash.ts new file mode 100644 index 000000000000..d2d3d192b665 --- /dev/null +++ b/langchain/src/util/js-sha1/hash.ts @@ -0,0 +1,417 @@ +// @ts-nocheck + +// Inlined to deal with portability issues with importing crypto module + +/* + * [js-sha1]{@link https://github.com/emn178/js-sha1} + * + * @version 0.6.0 + * @author Chen, Yi-Cyuan [emn178@gmail.com] + * @copyright Chen, Yi-Cyuan 2014-2017 + * @license MIT + */ +/*jslint bitwise: true */ +"use strict"; + +var root = typeof window === "object" ? window : {}; +var HEX_CHARS = "0123456789abcdef".split(""); +var EXTRA = [-2147483648, 8388608, 32768, 128]; +var SHIFT = [24, 16, 8, 0]; +var OUTPUT_TYPES = ["hex", "array", "digest", "arrayBuffer"]; + +var blocks = []; + +function Sha1(sharedMemory) { + if (sharedMemory) { + blocks[0] = + blocks[16] = + blocks[1] = + blocks[2] = + blocks[3] = + blocks[4] = + blocks[5] = + blocks[6] = + blocks[7] = + blocks[8] = + blocks[9] = + blocks[10] = + blocks[11] = + blocks[12] = + blocks[13] = + blocks[14] = + blocks[15] = + 0; + this.blocks = blocks; + } else { + this.blocks = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + } + + this.h0 = 0x67452301; + this.h1 = 0xefcdab89; + this.h2 = 0x98badcfe; + this.h3 = 0x10325476; + this.h4 = 0xc3d2e1f0; + + this.block = this.start = this.bytes = this.hBytes = 0; + this.finalized = this.hashed = false; + this.first = true; +} + +Sha1.prototype.update = function (message) { + if (this.finalized) { + return; + } + var notString = typeof message !== "string"; + if (notString && message.constructor === root.ArrayBuffer) { + message = new Uint8Array(message); + } + var code, + index = 0, + i, + length = message.length || 0, + blocks = this.blocks; + + while (index < length) { + if (this.hashed) { + this.hashed = false; + blocks[0] = this.block; + blocks[16] = + blocks[1] = + blocks[2] = + blocks[3] = + blocks[4] = + blocks[5] = + blocks[6] = + blocks[7] = + blocks[8] = + blocks[9] = + blocks[10] = + blocks[11] = + blocks[12] = + blocks[13] = + blocks[14] = + blocks[15] = + 0; + } + + if (notString) { + for (i = this.start; index < length && i < 64; ++index) { + blocks[i >> 2] |= message[index] << SHIFT[i++ & 3]; + } + } else { + for (i = this.start; index < length && i < 64; ++index) { + code = message.charCodeAt(index); + if (code < 0x80) { + blocks[i >> 2] |= code << SHIFT[i++ & 3]; + } else if (code < 0x800) { + blocks[i >> 2] |= (0xc0 | (code >> 6)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } else if (code < 0xd800 || code >= 0xe000) { + blocks[i >> 2] |= (0xe0 | (code >> 12)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 6) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } else { + code = + 0x10000 + + (((code & 0x3ff) << 10) | (message.charCodeAt(++index) & 0x3ff)); + blocks[i >> 2] |= (0xf0 | (code >> 18)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 12) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | ((code >> 6) & 0x3f)) << SHIFT[i++ & 3]; + blocks[i >> 2] |= (0x80 | (code & 0x3f)) << SHIFT[i++ & 3]; + } + } + } + + this.lastByteIndex = i; + this.bytes += i - this.start; + if (i >= 64) { + this.block = blocks[16]; + this.start = i - 64; + this.hash(); + this.hashed = true; + } else { + this.start = i; + } + } + if (this.bytes > 4294967295) { + this.hBytes += (this.bytes / 4294967296) << 0; + this.bytes = this.bytes % 4294967296; + } + return this; +}; + +Sha1.prototype.finalize = function () { + if (this.finalized) { + return; + } + this.finalized = true; + var blocks = this.blocks, + i = this.lastByteIndex; + blocks[16] = this.block; + blocks[i >> 2] |= EXTRA[i & 3]; + this.block = blocks[16]; + if (i >= 56) { + if (!this.hashed) { + this.hash(); + } + blocks[0] = this.block; + blocks[16] = + blocks[1] = + blocks[2] = + blocks[3] = + blocks[4] = + blocks[5] = + blocks[6] = + blocks[7] = + blocks[8] = + blocks[9] = + blocks[10] = + blocks[11] = + blocks[12] = + blocks[13] = + blocks[14] = + blocks[15] = + 0; + } + blocks[14] = (this.hBytes << 3) | (this.bytes >>> 29); + blocks[15] = this.bytes << 3; + this.hash(); +}; + +Sha1.prototype.hash = function () { + var a = this.h0, + b = this.h1, + c = this.h2, + d = this.h3, + e = this.h4; + var f, + j, + t, + blocks = this.blocks; + + for (j = 16; j < 80; ++j) { + t = blocks[j - 3] ^ blocks[j - 8] ^ blocks[j - 14] ^ blocks[j - 16]; + blocks[j] = (t << 1) | (t >>> 31); + } + + for (j = 0; j < 20; j += 5) { + f = (b & c) | (~b & d); + t = (a << 5) | (a >>> 27); + e = (t + f + e + 1518500249 + blocks[j]) << 0; + b = (b << 30) | (b >>> 2); + + f = (a & b) | (~a & c); + t = (e << 5) | (e >>> 27); + d = (t + f + d + 1518500249 + blocks[j + 1]) << 0; + a = (a << 30) | (a >>> 2); + + f = (e & a) | (~e & b); + t = (d << 5) | (d >>> 27); + c = (t + f + c + 1518500249 + blocks[j + 2]) << 0; + e = (e << 30) | (e >>> 2); + + f = (d & e) | (~d & a); + t = (c << 5) | (c >>> 27); + b = (t + f + b + 1518500249 + blocks[j + 3]) << 0; + d = (d << 30) | (d >>> 2); + + f = (c & d) | (~c & e); + t = (b << 5) | (b >>> 27); + a = (t + f + a + 1518500249 + blocks[j + 4]) << 0; + c = (c << 30) | (c >>> 2); + } + + for (; j < 40; j += 5) { + f = b ^ c ^ d; + t = (a << 5) | (a >>> 27); + e = (t + f + e + 1859775393 + blocks[j]) << 0; + b = (b << 30) | (b >>> 2); + + f = a ^ b ^ c; + t = (e << 5) | (e >>> 27); + d = (t + f + d + 1859775393 + blocks[j + 1]) << 0; + a = (a << 30) | (a >>> 2); + + f = e ^ a ^ b; + t = (d << 5) | (d >>> 27); + c = (t + f + c + 1859775393 + blocks[j + 2]) << 0; + e = (e << 30) | (e >>> 2); + + f = d ^ e ^ a; + t = (c << 5) | (c >>> 27); + b = (t + f + b + 1859775393 + blocks[j + 3]) << 0; + d = (d << 30) | (d >>> 2); + + f = c ^ d ^ e; + t = (b << 5) | (b >>> 27); + a = (t + f + a + 1859775393 + blocks[j + 4]) << 0; + c = (c << 30) | (c >>> 2); + } + + for (; j < 60; j += 5) { + f = (b & c) | (b & d) | (c & d); + t = (a << 5) | (a >>> 27); + e = (t + f + e - 1894007588 + blocks[j]) << 0; + b = (b << 30) | (b >>> 2); + + f = (a & b) | (a & c) | (b & c); + t = (e << 5) | (e >>> 27); + d = (t + f + d - 1894007588 + blocks[j + 1]) << 0; + a = (a << 30) | (a >>> 2); + + f = (e & a) | (e & b) | (a & b); + t = (d << 5) | (d >>> 27); + c = (t + f + c - 1894007588 + blocks[j + 2]) << 0; + e = (e << 30) | (e >>> 2); + + f = (d & e) | (d & a) | (e & a); + t = (c << 5) | (c >>> 27); + b = (t + f + b - 1894007588 + blocks[j + 3]) << 0; + d = (d << 30) | (d >>> 2); + + f = (c & d) | (c & e) | (d & e); + t = (b << 5) | (b >>> 27); + a = (t + f + a - 1894007588 + blocks[j + 4]) << 0; + c = (c << 30) | (c >>> 2); + } + + for (; j < 80; j += 5) { + f = b ^ c ^ d; + t = (a << 5) | (a >>> 27); + e = (t + f + e - 899497514 + blocks[j]) << 0; + b = (b << 30) | (b >>> 2); + + f = a ^ b ^ c; + t = (e << 5) | (e >>> 27); + d = (t + f + d - 899497514 + blocks[j + 1]) << 0; + a = (a << 30) | (a >>> 2); + + f = e ^ a ^ b; + t = (d << 5) | (d >>> 27); + c = (t + f + c - 899497514 + blocks[j + 2]) << 0; + e = (e << 30) | (e >>> 2); + + f = d ^ e ^ a; + t = (c << 5) | (c >>> 27); + b = (t + f + b - 899497514 + blocks[j + 3]) << 0; + d = (d << 30) | (d >>> 2); + + f = c ^ d ^ e; + t = (b << 5) | (b >>> 27); + a = (t + f + a - 899497514 + blocks[j + 4]) << 0; + c = (c << 30) | (c >>> 2); + } + + this.h0 = (this.h0 + a) << 0; + this.h1 = (this.h1 + b) << 0; + this.h2 = (this.h2 + c) << 0; + this.h3 = (this.h3 + d) << 0; + this.h4 = (this.h4 + e) << 0; +}; + +Sha1.prototype.hex = function () { + this.finalize(); + + var h0 = this.h0, + h1 = this.h1, + h2 = this.h2, + h3 = this.h3, + h4 = this.h4; + + return ( + HEX_CHARS[(h0 >> 28) & 0x0f] + + HEX_CHARS[(h0 >> 24) & 0x0f] + + HEX_CHARS[(h0 >> 20) & 0x0f] + + HEX_CHARS[(h0 >> 16) & 0x0f] + + HEX_CHARS[(h0 >> 12) & 0x0f] + + HEX_CHARS[(h0 >> 8) & 0x0f] + + HEX_CHARS[(h0 >> 4) & 0x0f] + + HEX_CHARS[h0 & 0x0f] + + HEX_CHARS[(h1 >> 28) & 0x0f] + + HEX_CHARS[(h1 >> 24) & 0x0f] + + HEX_CHARS[(h1 >> 20) & 0x0f] + + HEX_CHARS[(h1 >> 16) & 0x0f] + + HEX_CHARS[(h1 >> 12) & 0x0f] + + HEX_CHARS[(h1 >> 8) & 0x0f] + + HEX_CHARS[(h1 >> 4) & 0x0f] + + HEX_CHARS[h1 & 0x0f] + + HEX_CHARS[(h2 >> 28) & 0x0f] + + HEX_CHARS[(h2 >> 24) & 0x0f] + + HEX_CHARS[(h2 >> 20) & 0x0f] + + HEX_CHARS[(h2 >> 16) & 0x0f] + + HEX_CHARS[(h2 >> 12) & 0x0f] + + HEX_CHARS[(h2 >> 8) & 0x0f] + + HEX_CHARS[(h2 >> 4) & 0x0f] + + HEX_CHARS[h2 & 0x0f] + + HEX_CHARS[(h3 >> 28) & 0x0f] + + HEX_CHARS[(h3 >> 24) & 0x0f] + + HEX_CHARS[(h3 >> 20) & 0x0f] + + HEX_CHARS[(h3 >> 16) & 0x0f] + + HEX_CHARS[(h3 >> 12) & 0x0f] + + HEX_CHARS[(h3 >> 8) & 0x0f] + + HEX_CHARS[(h3 >> 4) & 0x0f] + + HEX_CHARS[h3 & 0x0f] + + HEX_CHARS[(h4 >> 28) & 0x0f] + + HEX_CHARS[(h4 >> 24) & 0x0f] + + HEX_CHARS[(h4 >> 20) & 0x0f] + + HEX_CHARS[(h4 >> 16) & 0x0f] + + HEX_CHARS[(h4 >> 12) & 0x0f] + + HEX_CHARS[(h4 >> 8) & 0x0f] + + HEX_CHARS[(h4 >> 4) & 0x0f] + + HEX_CHARS[h4 & 0x0f] + ); +}; + +Sha1.prototype.toString = Sha1.prototype.hex; + +Sha1.prototype.digest = function () { + this.finalize(); + + var h0 = this.h0, + h1 = this.h1, + h2 = this.h2, + h3 = this.h3, + h4 = this.h4; + + return [ + (h0 >> 24) & 0xff, + (h0 >> 16) & 0xff, + (h0 >> 8) & 0xff, + h0 & 0xff, + (h1 >> 24) & 0xff, + (h1 >> 16) & 0xff, + (h1 >> 8) & 0xff, + h1 & 0xff, + (h2 >> 24) & 0xff, + (h2 >> 16) & 0xff, + (h2 >> 8) & 0xff, + h2 & 0xff, + (h3 >> 24) & 0xff, + (h3 >> 16) & 0xff, + (h3 >> 8) & 0xff, + h3 & 0xff, + (h4 >> 24) & 0xff, + (h4 >> 16) & 0xff, + (h4 >> 8) & 0xff, + h4 & 0xff, + ]; +}; + +Sha1.prototype.array = Sha1.prototype.digest; + +Sha1.prototype.arrayBuffer = function () { + this.finalize(); + + var buffer = new ArrayBuffer(20); + var dataView = new DataView(buffer); + dataView.setUint32(0, this.h0); + dataView.setUint32(4, this.h1); + dataView.setUint32(8, this.h2); + dataView.setUint32(12, this.h3); + dataView.setUint32(16, this.h4); + return buffer; +}; + +export const insecureHash = (message) => { + return new Sha1(true).update(message)["hex"](); +}; diff --git a/yarn.lock b/yarn.lock index 6b85168406bf..b251fc94671a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11478,13 +11478,6 @@ __metadata: languageName: node linkType: hard -"@types/object-hash@npm:^3.0.2": - version: 3.0.2 - resolution: "@types/object-hash@npm:3.0.2" - checksum: 0332e59074e7df2e74c093a7419c05c1e1c5ae1e12d3779f3240b3031835ff045b4ac591362be0b411ace24d3b5e760386b434761c33af25904f7a3645cb3785 - languageName: node - linkType: hard - "@types/offscreencanvas@npm:~2019.3.0": version: 2019.3.0 resolution: "@types/offscreencanvas@npm:2019.3.0" @@ -22210,7 +22203,6 @@ __metadata: "@types/jsdom": ^21.1.1 "@types/lodash": ^4 "@types/mozilla-readability": ^0.2.1 - "@types/object-hash": ^3.0.2 "@types/pdf-parse": ^1.1.1 "@types/pg": ^8 "@types/pg-copy-streams": ^1.2.2 @@ -22280,7 +22272,6 @@ __metadata: neo4j-driver: ^5.12.0 node-llama-cpp: 2.7.3 notion-to-md: ^3.1.0 - object-hash: ^3.0.0 openai: ~4.4.0 openapi-types: ^12.1.3 p-queue: ^6.6.2