diff --git a/packages/api/src/magic.link.js b/packages/api/src/magic.link.js index 1aff0f0ffc..8a357c2ba4 100644 --- a/packages/api/src/magic.link.js +++ b/packages/api/src/magic.link.js @@ -62,7 +62,7 @@ function isMagicTestModeToken (token) { try { parsed = JSON.parse(globalThis.atob(token)) } catch (error) { - return false; + return false } if (parsed.length !== 2) { // unexpeced parse diff --git a/packages/db/db-client-types.ts b/packages/db/db-client-types.ts index 61095164e9..92c4a1412f 100644 --- a/packages/db/db-client-types.ts +++ b/packages/db/db-client-types.ts @@ -206,6 +206,7 @@ export type UploadItem = { created?: definitions['upload']['inserted_at'] updated?: definitions['upload']['updated_at'] content: ContentItem + backupUrls: definitions['upload']['backup_urls'] } export type UploadItemOutput = { @@ -218,6 +219,8 @@ export type UploadItemOutput = { dagSize?: definitions['content']['dag_size'] pins: Array, deals: Array + // array of links to things containing this Upload (e.g. CARs) + partOf: Array } export type UploadOutput = definitions['upload'] & { diff --git a/packages/db/index.js b/packages/db/index.js index 21d28b33f4..df5d7d042f 100644 --- a/packages/db/index.js +++ b/packages/db/index.js @@ -21,6 +21,7 @@ const uploadQuery = ` sourceCid:source_cid, created:inserted_at, updated:updated_at, + backupUrls:backup_urls, content(cid, dagSize:dag_size, pins:pin(status, updated:updated_at, location:pin_location(_id:id, peerId:peer_id, peerName:peer_name, ipfsPeerId:ipfs_peer_id, region))) ` @@ -555,7 +556,6 @@ export class DBClient { // Get deals const cids = uploads?.map((u) => u.content.cid) const deals = await this.getDealsForCids(cids) - return { count, uploads: uploads?.map((u) => ({ diff --git a/packages/db/postgres/functions.sql b/packages/db/postgres/functions.sql index b91f23e80f..e3085b6811 100644 --- a/packages/db/postgres/functions.sql +++ b/packages/db/postgres/functions.sql @@ -10,6 +10,7 @@ DROP FUNCTION IF EXISTS user_used_storage; DROP FUNCTION IF EXISTS user_auth_keys_list; DROP FUNCTION IF EXISTS find_deals_by_content_cids; DROP FUNCTION IF EXISTS upsert_user; +DROP TYPE IF EXISTS stored_bytes; -- transform a JSON array property into an array of SQL text elements CREATE OR REPLACE FUNCTION json_arr_to_text_arr(_json json) diff --git a/packages/db/postgres/reset.sql b/packages/db/postgres/reset.sql index 74b1006e73..badcba45a7 100644 --- a/packages/db/postgres/reset.sql +++ b/packages/db/postgres/reset.sql @@ -11,12 +11,18 @@ DROP TABLE IF EXISTS pin_sync_request; DROP TABLE IF EXISTS psa_pin_request; DROP TABLE IF EXISTS content; DROP TABLE IF EXISTS backup; +DROP TABLE IF EXISTS auth_key_history; DROP TABLE IF EXISTS auth_key; -DROP TABLE IF EXISTS public.user; DROP TABLE IF EXISTS user_tag; DROP TABLE IF EXISTS user_tag_proposal; +DROP TABLE IF EXISTS email_history; +DROP TABLE IF EXISTS user_customer; +DROP TABLE IF EXISTS agreement; +DROP TABLE IF EXISTS public.user; DROP TABLE IF EXISTS terms_of_service; +DROP TYPE IF EXISTS stored_bytes; + DROP SCHEMA IF EXISTS cargo CASCADE; DROP SERVER IF EXISTS dag_cargo_server CASCADE; DROP MATERIALIZED VIEW IF EXISTS public.aggregate_entry CASCADE; diff --git a/packages/db/postgres/tables.sql b/packages/db/postgres/tables.sql index 40d2a78514..c46edf4cbc 100644 --- a/packages/db/postgres/tables.sql +++ b/packages/db/postgres/tables.sql @@ -300,6 +300,7 @@ CREATE INDEX IF NOT EXISTS pin_sync_request_inserted_at_idx ON pin_sync_request -- Setting search_path to public scope for uuid function(s) SET search_path TO public; +DROP TABLE IF EXISTS psa_pin_request; DROP extension IF EXISTS "uuid-ossp"; CREATE extension "uuid-ossp" SCHEMA public; @@ -356,6 +357,8 @@ CREATE TABLE IF NOT EXISTS email_history sent_at TIMESTAMP WITH TIME ZONE DEFAULT timezone('utc'::text, now()) NOT NULL ); + +DROP VIEW IF EXISTS admin_search; CREATE VIEW admin_search as select u.id::text as user_id, diff --git a/packages/db/test/upload.spec.js b/packages/db/test/upload.spec.js index f80c0e0bfc..4d0c033da7 100644 --- a/packages/db/test/upload.spec.js +++ b/packages/db/test/upload.spec.js @@ -276,6 +276,41 @@ describe('upload', () => { // Default sort {inserted_at, Desc} const { uploads: userUploads } = await client.listUploads(user._id, { page: 1 }) assert.ok(userUploads.find(upload => upload.cid === sourceCid)) + console.log('userUploads', userUploads) + }) + + it('lists user uploads with CAR links in partOf', async () => { + const contentCid = 'bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi' + const sourceCid = 'QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR' + const exampleCarParkUrl = 'https://carpark-dev.web3.storage/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea/bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea.car' + const created = new Date().toISOString() + const name = `rand-${Math.random().toString().slice(2)}` + await client.createUpload({ + user: user._id, + contentCid, + sourceCid, + authKey: authKeys[0]._id, + type, + dagSize, + name, + pins: [initialPinData], + backupUrls: [`https://backup.cid/${created}`, exampleCarParkUrl], + created + }) + + // Default sort {inserted_at, Desc} + const { uploads } = await client.listUploads(user._id, { page: 1 }) + assert.ok(uploads.length > 0) + for (const upload of uploads) { + // backupUrls raw is private + assert.ok(!('backupUrls' in upload), 'upload does not have backupUrls property') + assert.ok(Array.isArray(upload.partOf), 'upload.partOf is an array') + } + const namedUpload = uploads.find(u => u.name === name) + assert.deepEqual(namedUpload.partOf, [ + // this corresponds to `exampleCarParkUrl` + 'ipfs://bagbaiera6xcx7hiicm7sc523axbjf2otuu5nptt6brdzt4a5ulgn6qcfdwea' + ]) }) it('can list user uploads with several options', async () => { diff --git a/packages/db/utils.js b/packages/db/utils.js index 626a851b80..68aafcd0c4 100644 --- a/packages/db/utils.js +++ b/packages/db/utils.js @@ -6,17 +6,39 @@ */ export function normalizeUpload (upload) { const nUpload = { ...upload } + const backupUrls = nUpload.backupUrls ?? [] + delete nUpload.backupUrls delete nUpload.content delete nUpload.sourceCid + const partOf = [...carUrlsFromBackupUrls(backupUrls)] + return { ...nUpload, ...upload.content, cid: upload.sourceCid, // Overwrite cid to source cid pins: normalizePins(upload.content.pins, { isOkStatuses: true - }) + }), + partOf + } +} + +/** + * given array of backup_urls from uploads table, return a set of ipfs:// URIs for any CAR files in the backup_urls + * @param {string[]} backupUrls + * @returns {Iterable} + */ +function carUrlsFromBackupUrls (backupUrls) { + const carCIDUrls = new Set() + for (const backupUrl of backupUrls) { + // match cid v1 starting with 'ba'. + // there are also backupUrls from s3 with .car suffix and path stem is base32(multihash) (not a CID). exclude those. + const carCidFileSuffixMatch = String(backupUrl).match(/\/(ba[^/]+).car$/) + if (!carCidFileSuffixMatch) continue + carCIDUrls.add(`ipfs://${carCidFileSuffixMatch[1]}`) } + return carCIDUrls } /**