From 35c02993c8a4183e66b2c2fc252c7021bd2c8972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Raphael=20Diaz=20Sim=C3=B5es?= Date: Tue, 3 Oct 2023 20:47:51 +0500 Subject: [PATCH 1/6] Add tlon hosting upload method --- ui/src/state/storage/storage.ts | 8 ++ ui/src/state/storage/type.ts | 18 +++- ui/src/state/storage/upload.ts | 181 ++++++++++++++++++++++---------- 3 files changed, 149 insertions(+), 58 deletions(-) diff --git a/ui/src/state/storage/storage.ts b/ui/src/state/storage/storage.ts index 9feef137ae..6f32f1b9f3 100644 --- a/ui/src/state/storage/storage.ts +++ b/ui/src/state/storage/storage.ts @@ -20,6 +20,14 @@ export const useStorage = createState( () => ({ loaded: false, hasCredentials: false, + // XXX: This is for PR testing, the current case is covered by the "s3" + // backend. The endpoint and token are supposed to come from the ship, like + // the S3 credentials. + backend: 'tlon-hosting', + tlonHosting: { + endpoint: 'http://localhost:8888', + token: 'token', + }, s3: { configuration: { buckets: new Set(), diff --git a/ui/src/state/storage/type.ts b/ui/src/state/storage/type.ts index fee9bd3353..491827154d 100644 --- a/ui/src/state/storage/type.ts +++ b/ui/src/state/storage/type.ts @@ -7,9 +7,17 @@ export interface GcpToken { expiresIn: number; } +export interface StorageCredentialsTlonHosting { + endpoint: string; + token: string; +} + +export type StorageBackend = 's3' | 'tlon-hosting'; + export interface BaseStorageState { loaded?: boolean; hasCredentials?: boolean; + backend: StorageBackend; s3: { configuration: { buckets: Set; @@ -18,6 +26,7 @@ export interface BaseStorageState { }; credentials: S3Credentials | null; }; + tlonHosting: StorageCredentialsTlonHosting; [ref: string]: unknown; } @@ -54,10 +63,15 @@ export interface Uploader { } export interface FileStore { - client: S3Client | null; + // Only one among S3 client or Tlon credentials will be set at a given time. + s3Client: S3Client | null; + tlonHostingCredentials: StorageCredentialsTlonHosting | null; uploaders: Record; getUploader: (key: string) => Uploader; - createClient: (s3: S3Credentials, region: string) => void; + createS3Client: (s3: S3Credentials, region: string) => void; + setTlonHostingCredentials: ( + credentials: StorageCredentialsTlonHosting + ) => void; update: (key: string, updateFn: (uploader: Uploader) => void) => void; uploadFiles: ( uploader: string, diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index e1c1c5003a..3f993a6784 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -7,7 +7,7 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getImageSize } from 'react-image-size'; import { useCallback, useEffect, useState } from 'react'; import { Status } from '@/logic/status'; -import { FileStore, Uploader } from './type'; +import { FileStore, StorageCredentialsTlonHosting, Uploader } from './type'; import { useStorage } from './storage'; import { StorageState } from './reducer'; @@ -24,11 +24,12 @@ function imageSize(url: string) { } export const useFileStore = create((set, get) => ({ - client: null, + s3Client: null, + tlonHostingCredentials: null, uploaders: {}, - createClient: (credentials: S3Credentials, region: string) => { + createS3Client: (credentials: S3Credentials, region: string) => { const endpoint = new URL(prefixEndpoint(credentials.endpoint)); - const client = new S3Client({ + const s3Client = new S3Client({ endpoint: { protocol: endpoint.protocol.slice(0, -1), hostname: endpoint.host, @@ -39,7 +40,10 @@ export const useFileStore = create((set, get) => ({ credentials, forcePathStyle: true, }); - set({ client }); + set({ s3Client, tlonHostingCredentials: null }); + }, + setTlonHostingCredentials: (credentials: StorageCredentialsTlonHosting) => { + set({ s3Client: null, tlonHostingCredentials: credentials }); }, getUploader: (key) => { const { uploaders } = get(); @@ -80,46 +84,90 @@ export const useFileStore = create((set, get) => ({ return uploader ? uploader.uploadType : 'prompt'; }, upload: async (uploader, upload, bucket) => { - const { client, updateStatus, updateFile } = get(); - if (!client) { - return; - } + const { s3Client, tlonHostingCredentials, updateStatus, updateFile } = + get(); const { key, file } = upload; updateStatus(uploader, key, 'loading'); - const command = new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: file, - ContentType: file.type, - ContentLength: file.size, - ACL: 'public-read', - }); + // Logic for uploading with Tlon Hosting storage. + if (tlonHostingCredentials) { + // The first step is to send the PUT request to the proxy, which will + // respond with a redirect to a pre-signed url to the actual bucket. The + // token is in the url, not a header, so that it disappears after the + // redirect. + const requestOptions = { + method: 'PUT', + headers: { + 'Content-Type': file.type, + }, + body: file, + }; + const { endpoint, token } = tlonHostingCredentials; + const url = `${endpoint}/${key}`; + const urlWithToken = `${url}?token=${token}`; + fetch(urlWithToken, requestOptions) + .then(async () => { + // When the PUT succeeded, we fetch the actual URL of the file. We do + // this to avoid having to proxy every single GET request, and to + // avoid remembering which file corresponds to which bucket, when + // using multiple buckets internally. + const fileUrlResponse = await fetch(url); + const fileUrl = await fileUrlResponse.json(); + updateStatus(uploader, key, 'success'); + imageSize(fileUrl).then((s) => + updateFile(uploader, key, { + size: s, + url: fileUrl, + }) + ); + }) + .catch((error: any) => { + updateStatus( + uploader, + key, + 'error', + `Tlon Hosting upload error: ${error.message}, contact support if it persists.` + ); + console.log({ error }); + }); + } - const url = await getSignedUrl(client, command); - - client - .send(command) - .then(() => { - const fileUrl = url.split('?')[0]; - updateStatus(uploader, key, 'success'); - imageSize(fileUrl).then((s) => - updateFile(uploader, key, { - size: s, - url: fileUrl, - }) - ); - }) - .catch((error: any) => { - updateStatus( - uploader, - key, - 'error', - `S3 upload error: ${error.message}, check your S3 configuration.` - ); - console.log({ error }); + // Logic for uploading with S3. + if (s3Client) { + const command = new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: file, + ContentType: file.type, + ContentLength: file.size, + ACL: 'public-read', }); + + const url = await getSignedUrl(s3Client, command); + + s3Client + .send(command) + .then(() => { + const fileUrl = url.split('?')[0]; + updateStatus(uploader, key, 'success'); + imageSize(fileUrl).then((s) => + updateFile(uploader, key, { + size: s, + url: fileUrl, + }) + ); + }) + .catch((error: any) => { + updateStatus( + uploader, + key, + 'error', + `S3 upload error: ${error.message}, check your S3 configuration.` + ); + console.log({ error }); + }); + } }, clear: (uploader) => { get().update(uploader, (draft) => { @@ -187,49 +235,70 @@ const emptyUploader = (key: string, bucket: string): Uploader => ({ }, }); -export function useClient() { +function useS3Client() { const { + backend, s3: { credentials, configuration }, + tlonHosting, } = useStorage(); - const { client, createClient } = useFileStore(); + const { s3Client, createS3Client, setTlonHostingCredentials } = + useFileStore(); const [hasCredentials, setHasCredentials] = useState(false); useEffect(() => { const hasCreds = - credentials?.accessKeyId && - credentials?.endpoint && - credentials?.secretAccessKey; + backend === 's3' + ? credentials?.accessKeyId && + credentials?.endpoint && + credentials?.secretAccessKey + : backend === 'tlon-hosting' + ? !!tlonHosting + : false; if (hasCreds) { setHasCredentials(true); } - }, [credentials]); + }, [backend, credentials, tlonHosting]); const initClient = useCallback(async () => { if (credentials) { - await createClient(credentials, configuration.region); + if (backend === 's3') { + await createS3Client(credentials, configuration.region); + } + if (backend === 'tlon-hosting') { + await setTlonHostingCredentials(tlonHosting); + } } - }, [createClient, credentials, configuration]); + }, [ + backend, + createS3Client, + credentials, + configuration, + setTlonHostingCredentials, + tlonHosting, + ]); useEffect(() => { - if (hasCredentials && !client) { + if (hasCredentials && !s3Client) { initClient(); } - }, [client, hasCredentials, initClient]); + }, [s3Client, hasCredentials, initClient]); - return client; + return s3Client; } -const selS3 = (s: StorageState) => s.s3; const selUploader = (key: string) => (s: FileStore) => s.uploaders[key]; export function useUploader(key: string): Uploader | undefined { const { - configuration: { currentBucket }, - } = useStorage(selS3); - const client = useClient(); + tlonHosting: { token }, + s3: { + configuration: { currentBucket }, + }, + } = useStorage(); + const s3Client = useS3Client(); const uploader = useFileStore(selUploader(key)); useEffect(() => { - if (client && currentBucket) { + if ((s3Client && currentBucket) || token) { useFileStore.setState( produce((draft) => { draft.uploaders[key] = emptyUploader(key, currentBucket); @@ -244,7 +313,7 @@ export function useUploader(key: string): Uploader | undefined { }) ); }; - }, [client, currentBucket, key]); + }, [s3Client, currentBucket, key, token]); return uploader; } From d40593eeb0ae32e55a26b4b7602fee04b6924315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Raphael=20Diaz=20Sim=C3=B5es?= Date: Tue, 10 Oct 2023 17:30:17 +0500 Subject: [PATCH 2/6] Better error messages in the UI --- ui/src/state/storage/upload.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index 3f993a6784..df7fb2878f 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -107,7 +107,11 @@ export const useFileStore = create((set, get) => ({ const url = `${endpoint}/${key}`; const urlWithToken = `${url}?token=${token}`; fetch(urlWithToken, requestOptions) - .then(async () => { + .then(async (response) => { + if (response.status !== 200) { + const body = await response.text(); + throw new Error(body || 'Incorrect response status'); + } // When the PUT succeeded, we fetch the actual URL of the file. We do // this to avoid having to proxy every single GET request, and to // avoid remembering which file corresponds to which bucket, when From 40ca4630f08030191e5bc6762f6317cf10ddf5bb Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Fri, 13 Oct 2023 09:44:00 -0500 Subject: [PATCH 3/6] storage: adding secret for tlon hosting upload --- ui/src/state/storage/upload.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index df7fb2878f..7ba2954190 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -7,9 +7,9 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getImageSize } from 'react-image-size'; import { useCallback, useEffect, useState } from 'react'; import { Status } from '@/logic/status'; +import api from '@/api'; import { FileStore, StorageCredentialsTlonHosting, Uploader } from './type'; import { useStorage } from './storage'; -import { StorageState } from './reducer'; export function prefixEndpoint(endpoint: string) { return endpoint.match(/https?:\/\//) ? endpoint : `https://${endpoint}`; @@ -103,7 +103,11 @@ export const useFileStore = create((set, get) => ({ }, body: file, }; - const { endpoint, token } = tlonHostingCredentials; + const { endpoint } = tlonHostingCredentials; + const token = await api.scry({ + app: 'genuine', + path: '/secret', + }); const url = `${endpoint}/${key}`; const urlWithToken = `${url}?token=${token}`; fetch(urlWithToken, requestOptions) From c1857ce1c671cc94993e6724e29bd91b7c43dfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Raphael=20Diaz=20Sim=C3=B5es?= Date: Tue, 17 Oct 2023 16:49:27 +0500 Subject: [PATCH 4/6] Remove token from tlonHosting method --- ui/src/state/storage/storage.ts | 1 - ui/src/state/storage/type.ts | 1 - ui/src/state/storage/upload.ts | 6 +++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ui/src/state/storage/storage.ts b/ui/src/state/storage/storage.ts index 6f32f1b9f3..847a8696b5 100644 --- a/ui/src/state/storage/storage.ts +++ b/ui/src/state/storage/storage.ts @@ -26,7 +26,6 @@ export const useStorage = createState( backend: 'tlon-hosting', tlonHosting: { endpoint: 'http://localhost:8888', - token: 'token', }, s3: { configuration: { diff --git a/ui/src/state/storage/type.ts b/ui/src/state/storage/type.ts index 491827154d..03fba4e603 100644 --- a/ui/src/state/storage/type.ts +++ b/ui/src/state/storage/type.ts @@ -9,7 +9,6 @@ export interface GcpToken { export interface StorageCredentialsTlonHosting { endpoint: string; - token: string; } export type StorageBackend = 's3' | 'tlon-hosting'; diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index 7ba2954190..71c4fba599 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -297,7 +297,7 @@ function useS3Client() { const selUploader = (key: string) => (s: FileStore) => s.uploaders[key]; export function useUploader(key: string): Uploader | undefined { const { - tlonHosting: { token }, + tlonHosting: { endpoint }, s3: { configuration: { currentBucket }, }, @@ -306,7 +306,7 @@ export function useUploader(key: string): Uploader | undefined { const uploader = useFileStore(selUploader(key)); useEffect(() => { - if ((s3Client && currentBucket) || token) { + if ((s3Client && currentBucket) || endpoint) { useFileStore.setState( produce((draft) => { draft.uploaders[key] = emptyUploader(key, currentBucket); @@ -321,7 +321,7 @@ export function useUploader(key: string): Uploader | undefined { }) ); }; - }, [s3Client, currentBucket, key, token]); + }, [s3Client, currentBucket, key, endpoint]); return uploader; } From f2e5164c53efa54b7846cd6061d9c19db0c48d30 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 18 Oct 2023 10:47:17 -0500 Subject: [PATCH 5/6] storage: update to match new tlon hosting --- ui/package-lock.json | 24 +++--- ui/package.json | 2 +- ui/src/state/storage/reducer.ts | 2 + ui/src/state/storage/storage.ts | 10 +-- ui/src/state/storage/type.ts | 110 ++++++++++++++++++------ ui/src/state/storage/upload.ts | 145 ++++++++++++++------------------ 6 files changed, 166 insertions(+), 127 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 27b2ad1922..7f0dd1722d 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -58,7 +58,7 @@ "@tloncorp/mock-http-api": "^1.2.0", "@types/marked": "^4.3.0", "@urbit/api": "^2.2.0", - "@urbit/aura": "^0.4.0", + "@urbit/aura": "^1.0.0", "@urbit/http-api": "^3.0.0", "@urbit/sigil-js": "^2.1.0", "any-ascii": "^0.3.1", @@ -7956,13 +7956,15 @@ } }, "node_modules/@urbit/aura": { - "version": "0.4.0", - "license": "MIT", - "dependencies": { - "big-integer": "^1.6.51" - }, + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@urbit/aura/-/aura-1.0.0.tgz", + "integrity": "sha512-IeP3uoDzZ0Rpn345auXK0y/BCcXTmpgAlOPbgf7n4eD35h56OnSoit1kuXKA21sWE19gFjK/wqZcz5ULjz2ADg==", "engines": { - "node": ">=10" + "node": ">=16", + "npm": ">=8" + }, + "peerDependencies": { + "big-integer": "^1.6.51" } }, "node_modules/@urbit/eslint-config": { @@ -24128,10 +24130,10 @@ } }, "@urbit/aura": { - "version": "0.4.0", - "requires": { - "big-integer": "^1.6.51" - } + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@urbit/aura/-/aura-1.0.0.tgz", + "integrity": "sha512-IeP3uoDzZ0Rpn345auXK0y/BCcXTmpgAlOPbgf7n4eD35h56OnSoit1kuXKA21sWE19gFjK/wqZcz5ULjz2ADg==", + "requires": {} }, "@urbit/eslint-config": { "version": "1.0.3", diff --git a/ui/package.json b/ui/package.json index 84e8392535..8885d53782 100644 --- a/ui/package.json +++ b/ui/package.json @@ -105,7 +105,7 @@ "@tloncorp/mock-http-api": "^1.2.0", "@types/marked": "^4.3.0", "@urbit/api": "^2.2.0", - "@urbit/aura": "^0.4.0", + "@urbit/aura": "^1.0.0", "@urbit/http-api": "^3.0.0", "@urbit/sigil-js": "^2.1.0", "any-ascii": "^0.3.1", diff --git a/ui/src/state/storage/reducer.ts b/ui/src/state/storage/reducer.ts index 404589f0ce..62e519970a 100644 --- a/ui/src/state/storage/reducer.ts +++ b/ui/src/state/storage/reducer.ts @@ -21,6 +21,8 @@ const configuration = (json: S3Update, state: StorageState): StorageState => { buckets: new Set(data.buckets), currentBucket: data.currentBucket, region: data.region, + presignedUrl: data.presignedUrl, + service: data.service, }; } return state; diff --git a/ui/src/state/storage/storage.ts b/ui/src/state/storage/storage.ts index 847a8696b5..09fd4b7f65 100644 --- a/ui/src/state/storage/storage.ts +++ b/ui/src/state/storage/storage.ts @@ -19,19 +19,13 @@ export const useStorage = createState( 'Storage', () => ({ loaded: false, - hasCredentials: false, - // XXX: This is for PR testing, the current case is covered by the "s3" - // backend. The endpoint and token are supposed to come from the ship, like - // the S3 credentials. - backend: 'tlon-hosting', - tlonHosting: { - endpoint: 'http://localhost:8888', - }, s3: { configuration: { buckets: new Set(), currentBucket: '', region: '', + presignedUrl: '', + service: 'credentials', }, credentials: null, }, diff --git a/ui/src/state/storage/type.ts b/ui/src/state/storage/type.ts index 03fba4e603..57df5ed9de 100644 --- a/ui/src/state/storage/type.ts +++ b/ui/src/state/storage/type.ts @@ -1,34 +1,37 @@ -import { S3Credentials } from '@urbit/api'; import { S3Client } from '@aws-sdk/client-s3'; import { Status } from '@/logic/status'; -export interface GcpToken { - accessKey: string; - expiresIn: number; +export type StorageService = 'presigned-url' | 'credentials'; + +export interface StorageConfiguration { + buckets: Set; + currentBucket: string; + region: string; + presignedUrl: string; + service: StorageService; } -export interface StorageCredentialsTlonHosting { +export interface StorageCredentials { endpoint: string; + accessKeyId: string; + secretAccessKey: string; } -export type StorageBackend = 's3' | 'tlon-hosting'; - export interface BaseStorageState { loaded?: boolean; hasCredentials?: boolean; - backend: StorageBackend; s3: { - configuration: { - buckets: Set; - currentBucket: string; - region: string; - }; - credentials: S3Credentials | null; + configuration: StorageConfiguration; + credentials: StorageCredentials | null; }; - tlonHosting: StorageCredentialsTlonHosting; [ref: string]: unknown; } +export interface GcpToken { + accessKey: string; + expiresIn: number; +} + export interface FileStoreFile { key: string; file: File; @@ -62,22 +65,21 @@ export interface Uploader { } export interface FileStore { - // Only one among S3 client or Tlon credentials will be set at a given time. - s3Client: S3Client | null; - tlonHostingCredentials: StorageCredentialsTlonHosting | null; + client: S3Client | null; uploaders: Record; getUploader: (key: string) => Uploader; - createS3Client: (s3: S3Credentials, region: string) => void; - setTlonHostingCredentials: ( - credentials: StorageCredentialsTlonHosting - ) => void; + createClient: (s3: StorageCredentials, region: string) => void; update: (key: string, updateFn: (uploader: Uploader) => void) => void; uploadFiles: ( uploader: string, files: FileList | File[] | null, - bucket: string + config: StorageConfiguration + ) => Promise; + upload: ( + uploader: string, + upload: Upload, + config: StorageConfiguration ) => Promise; - upload: (uploader: string, upload: Upload, bucket: string) => Promise; clear: (uploader: string) => void; setUploadType: (uploaderKey: string, type: Uploader['uploadType']) => void; getUploadType: (uploaderKey: string) => Uploader['uploadType']; @@ -95,3 +97,63 @@ export interface UploadInputProps { multiple?: boolean; id: string; } + +export interface StorageUpdateCredentials { + credentials: StorageCredentials; +} + +export interface StorageUpdateConfiguration { + configuration: { + buckets: string[]; + currentBucket: string; + }; +} + +export interface StorageUpdateCurrentBucket { + setCurrentBucket: string; +} + +export interface StorageUpdateAddBucket { + addBucket: string; +} + +export interface StorageUpdateRemoveBucket { + removeBucket: string; +} + +export interface StorageUpdateEndpoint { + setEndpoint: string; +} + +export interface StorageUpdateAccessKeyId { + setAccessKeyId: string; +} + +export interface StorageUpdateSecretAccessKey { + setSecretAccessKey: string; +} + +export interface StorageUpdateRegion { + setRegion: string; +} + +export interface StorageUpdateToggleService { + toggleService: string; +} + +export interface StorageUpdateSetPresignedUrl { + setPresignedUrl: string; +} + +export declare type StorageUpdate = + | StorageUpdateCredentials + | StorageUpdateConfiguration + | StorageUpdateCurrentBucket + | StorageUpdateAddBucket + | StorageUpdateRemoveBucket + | StorageUpdateEndpoint + | StorageUpdateAccessKeyId + | StorageUpdateSecretAccessKey + | StorageUpdateRegion + | StorageUpdateToggleService + | StorageUpdateSetPresignedUrl; diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index 71c4fba599..a8522091e4 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -1,17 +1,22 @@ import _ from 'lodash'; import create from 'zustand'; import produce from 'immer'; -import { dateToDa, deSig, S3Credentials } from '@urbit/api'; +import { formatDa, unixToDa, deSig } from '@urbit/aura'; import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getImageSize } from 'react-image-size'; import { useCallback, useEffect, useState } from 'react'; -import { Status } from '@/logic/status'; import api from '@/api'; -import { FileStore, StorageCredentialsTlonHosting, Uploader } from './type'; +import { Status } from '@/logic/status'; +import { + FileStore, + StorageConfiguration, + StorageCredentials, + Uploader, +} from './type'; import { useStorage } from './storage'; -export function prefixEndpoint(endpoint: string) { +function prefixEndpoint(endpoint: string) { return endpoint.match(/https?:\/\//) ? endpoint : `https://${endpoint}`; } @@ -24,12 +29,11 @@ function imageSize(url: string) { } export const useFileStore = create((set, get) => ({ - s3Client: null, - tlonHostingCredentials: null, + client: null, uploaders: {}, - createS3Client: (credentials: S3Credentials, region: string) => { + createClient: (credentials: StorageCredentials, region: string) => { const endpoint = new URL(prefixEndpoint(credentials.endpoint)); - const s3Client = new S3Client({ + const client = new S3Client({ endpoint: { protocol: endpoint.protocol.slice(0, -1), hostname: endpoint.host, @@ -40,10 +44,7 @@ export const useFileStore = create((set, get) => ({ credentials, forcePathStyle: true, }); - set({ s3Client, tlonHostingCredentials: null }); - }, - setTlonHostingCredentials: (credentials: StorageCredentialsTlonHosting) => { - set({ s3Client: null, tlonHostingCredentials: credentials }); + set({ client }); }, getUploader: (key) => { const { uploaders } = get(); @@ -53,12 +54,23 @@ export const useFileStore = create((set, get) => ({ update: (key: string, updateFn: (uploader: Uploader) => void) => { set(produce((draft) => updateFn(draft.uploaders[key]))); }, - uploadFiles: async (uploader, files, bucket) => { + setUploadType: (uploaderKey, type) => { + get().update(uploaderKey, (draft) => { + draft.uploadType = type; + }); + }, + getUploadType: (uploaderKey) => { + const uploader = get().getUploader(uploaderKey); + return uploader ? uploader.uploadType : 'prompt'; + }, + uploadFiles: async (uploader, files, config) => { if (!files) return; const fileList = [...files].map((file) => ({ file, - key: `${window.ship}/${deSig(dateToDa(new Date()))}-${file.name}`, + key: `${window.ship}/${deSig(formatDa(unixToDa(new Date().getTime())))}-${ + file.name + }`, status: 'initial' as Status, url: '', size: [0, 0] as [number, number], @@ -72,26 +84,16 @@ export const useFileStore = create((set, get) => ({ draft.files = { ...draft.files, ...newFiles }; }); - fileList.forEach((f) => upload(uploader, f, bucket)); + fileList.forEach((f) => upload(uploader, f, config)); }, - setUploadType: (uploaderKey, type) => { - get().update(uploaderKey, (draft) => { - draft.uploadType = type; - }); - }, - getUploadType: (uploaderKey) => { - const uploader = get().getUploader(uploaderKey); - return uploader ? uploader.uploadType : 'prompt'; - }, - upload: async (uploader, upload, bucket) => { - const { s3Client, tlonHostingCredentials, updateStatus, updateFile } = - get(); + upload: async (uploader, upload, config) => { + const { client, updateStatus, updateFile } = get(); const { key, file } = upload; updateStatus(uploader, key, 'loading'); // Logic for uploading with Tlon Hosting storage. - if (tlonHostingCredentials) { + if (config.service === 'presigned-url' && config.presignedUrl) { // The first step is to send the PUT request to the proxy, which will // respond with a redirect to a pre-signed url to the actual bucket. The // token is in the url, not a header, so that it disappears after the @@ -103,12 +105,12 @@ export const useFileStore = create((set, get) => ({ }, body: file, }; - const { endpoint } = tlonHostingCredentials; + const { presignedUrl } = config; + const url = `${presignedUrl}/${key}`; const token = await api.scry({ app: 'genuine', path: '/secret', }); - const url = `${endpoint}/${key}`; const urlWithToken = `${url}?token=${token}`; fetch(urlWithToken, requestOptions) .then(async (response) => { @@ -142,9 +144,9 @@ export const useFileStore = create((set, get) => ({ } // Logic for uploading with S3. - if (s3Client) { + if (config.service === 'credentials' && client) { const command = new PutObjectCommand({ - Bucket: bucket, + Bucket: config.currentBucket, Key: key, Body: file, ContentType: file.type, @@ -152,9 +154,9 @@ export const useFileStore = create((set, get) => ({ ACL: 'public-read', }); - const url = await getSignedUrl(s3Client, command); + const url = await getSignedUrl(client, command); - s3Client + client .send(command) .then(() => { const fileUrl = url.split('?')[0]; @@ -217,11 +219,14 @@ export const useFileStore = create((set, get) => ({ }, })); -const emptyUploader = (key: string, bucket: string): Uploader => ({ +const emptyUploader = ( + key: string, + config: StorageConfiguration +): Uploader => ({ files: {}, getMostRecent: () => useFileStore.getState().getMostRecent(key), uploadFiles: async (files) => - useFileStore.getState().uploadFiles(key, files, bucket), + useFileStore.getState().uploadFiles(key, files, config), clear: () => useFileStore.getState().clear(key), removeByURL: (url) => useFileStore.getState().removeByURL(key, url), uploadType: 'prompt', @@ -233,7 +238,7 @@ const emptyUploader = (key: string, bucket: string): Uploader => ({ input.accept = 'image/*,video/*,audio/*'; input.addEventListener('change', async (e) => { const { files } = e.target as HTMLInputElement; - useFileStore.getState().uploadFiles(key, files, bucket); + useFileStore.getState().uploadFiles(key, files, config); input.remove(); }); // Add to DOM for mobile Safari support @@ -243,85 +248,59 @@ const emptyUploader = (key: string, bucket: string): Uploader => ({ }, }); -function useS3Client() { +function useClient() { const { - backend, s3: { credentials, configuration }, - tlonHosting, } = useStorage(); - const { s3Client, createS3Client, setTlonHostingCredentials } = - useFileStore(); + const { client, createClient } = useFileStore(); const [hasCredentials, setHasCredentials] = useState(false); useEffect(() => { const hasCreds = - backend === 's3' - ? credentials?.accessKeyId && - credentials?.endpoint && - credentials?.secretAccessKey - : backend === 'tlon-hosting' - ? !!tlonHosting - : false; + configuration.service === 'credentials' && + credentials?.accessKeyId && + credentials?.endpoint && + credentials?.secretAccessKey; if (hasCreds) { setHasCredentials(true); } - }, [backend, credentials, tlonHosting]); + }, [credentials, configuration]); const initClient = useCallback(async () => { if (credentials) { - if (backend === 's3') { - await createS3Client(credentials, configuration.region); - } - if (backend === 'tlon-hosting') { - await setTlonHostingCredentials(tlonHosting); - } + await createClient(credentials, configuration.region); } - }, [ - backend, - createS3Client, - credentials, - configuration, - setTlonHostingCredentials, - tlonHosting, - ]); + }, [createClient, credentials, configuration]); useEffect(() => { - if (hasCredentials && !s3Client) { + if (hasCredentials && !client) { initClient(); } - }, [s3Client, hasCredentials, initClient]); + }, [client, hasCredentials, initClient]); - return s3Client; + return client; } const selUploader = (key: string) => (s: FileStore) => s.uploaders[key]; export function useUploader(key: string): Uploader | undefined { const { - tlonHosting: { endpoint }, - s3: { - configuration: { currentBucket }, - }, + s3: { configuration }, } = useStorage(); - const s3Client = useS3Client(); + const client = useClient(); const uploader = useFileStore(selUploader(key)); useEffect(() => { - if ((s3Client && currentBucket) || endpoint) { + if ( + (client && configuration.service === 'credentials') || + (configuration.service === 'presigned-url' && configuration.presignedUrl) + ) { useFileStore.setState( produce((draft) => { - draft.uploaders[key] = emptyUploader(key, currentBucket); + draft.uploaders[key] = emptyUploader(key, configuration); }) ); } - - return () => { - useFileStore.setState( - produce((draft) => { - delete draft.uploaders[key]; - }) - ); - }; - }, [s3Client, currentBucket, key, endpoint]); + }, [client, configuration, key]); return uploader; } From c044c40c58585dc0e44f08bcbe848691b987f0da Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 18 Oct 2023 13:56:07 -0500 Subject: [PATCH 6/6] storage: adding new reducer functions --- ui/src/state/storage/reducer.ts | 70 ++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/ui/src/state/storage/reducer.ts b/ui/src/state/storage/reducer.ts index 62e519970a..ec8607b927 100644 --- a/ui/src/state/storage/reducer.ts +++ b/ui/src/state/storage/reducer.ts @@ -1,12 +1,14 @@ /* eslint-disable no-param-reassign */ -import { S3Update } from '@urbit/api'; import _ from 'lodash'; -import { BaseStorageState } from './type'; +import { StorageUpdate, BaseStorageState } from './type'; import { BaseState } from '../base'; export type StorageState = BaseStorageState & BaseState; -const credentials = (json: S3Update, state: StorageState): StorageState => { +const credentials = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'credentials', false); if (data) { state.s3.credentials = data; @@ -14,7 +16,10 @@ const credentials = (json: S3Update, state: StorageState): StorageState => { return state; }; -const configuration = (json: S3Update, state: StorageState): StorageState => { +const configuration = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'configuration', false); if (data) { state.s3.configuration = { @@ -28,7 +33,10 @@ const configuration = (json: S3Update, state: StorageState): StorageState => { return state; }; -const currentBucket = (json: S3Update, state: StorageState): StorageState => { +const currentBucket = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setCurrentBucket', false); if (data && state.s3) { state.s3.configuration.currentBucket = data; @@ -36,7 +44,7 @@ const currentBucket = (json: S3Update, state: StorageState): StorageState => { return state; }; -const addBucket = (json: S3Update, state: StorageState): StorageState => { +const addBucket = (json: StorageUpdate, state: StorageState): StorageState => { const data = _.get(json, 'addBucket', false); if (data) { state.s3.configuration.buckets = state.s3.configuration.buckets.add(data); @@ -44,7 +52,10 @@ const addBucket = (json: S3Update, state: StorageState): StorageState => { return state; }; -const removeBucket = (json: S3Update, state: StorageState): StorageState => { +const removeBucket = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'removeBucket', false); if (data) { state.s3.configuration.buckets.delete(data); @@ -52,7 +63,7 @@ const removeBucket = (json: S3Update, state: StorageState): StorageState => { return state; }; -const endpoint = (json: S3Update, state: StorageState): StorageState => { +const endpoint = (json: StorageUpdate, state: StorageState): StorageState => { const data = _.get(json, 'setEndpoint', false); if (data && state.s3.credentials) { state.s3.credentials.endpoint = data; @@ -60,7 +71,10 @@ const endpoint = (json: S3Update, state: StorageState): StorageState => { return state; }; -const accessKeyId = (json: S3Update, state: StorageState): StorageState => { +const accessKeyId = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setAccessKeyId', false); if (data && state.s3.credentials) { state.s3.credentials.accessKeyId = data; @@ -68,7 +82,10 @@ const accessKeyId = (json: S3Update, state: StorageState): StorageState => { return state; }; -const secretAccessKey = (json: S3Update, state: StorageState): StorageState => { +const secretAccessKey = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setSecretAccessKey', false); if (data && state.s3.credentials) { state.s3.credentials.secretAccessKey = data; @@ -76,6 +93,36 @@ const secretAccessKey = (json: S3Update, state: StorageState): StorageState => { return state; }; +const region = (json: StorageUpdate, state: StorageState): StorageState => { + const data = _.get(json, 'setRegion', false); + if (data && state.s3.configuration) { + state.s3.configuration.region = data; + } + return state; +}; + +const presignedUrl = ( + json: StorageUpdate, + state: StorageState +): StorageState => { + const data = _.get(json, 'setPresignedUrl', false); + if (data && state.s3.configuration) { + state.s3.configuration.presignedUrl = data; + } + return state; +}; + +const toggleService = ( + json: StorageUpdate, + state: StorageState +): StorageState => { + const data = _.get(json, 'toggleService', false); + if (data && state.s3.configuration) { + state.s3.configuration.service = data; + } + return state; +}; + const reduce = [ credentials, configuration, @@ -85,6 +132,9 @@ const reduce = [ endpoint, accessKeyId, secretAccessKey, + region, + presignedUrl, + toggleService, ]; export default reduce;