diff --git a/desk/app/storage.hoon b/desk/app/storage.hoon index 938793f9..487f0b4c 100644 --- a/desk/app/storage.hoon +++ b/desk/app/storage.hoon @@ -10,13 +10,15 @@ +$ versioned-state $% state-zero state-one + state-two == :: +$ state-zero [%0 =credentials:zero:past =configuration:zero:past] -+$ state-one [%1 =credentials =configuration] ++$ state-one [%1 =credentials:one:past =configuration:one:past] ++$ state-two [%2 =credentials =configuration] -- :: -=| state-one +=| state-two =* state - :: %- agent:dbug @@ -41,13 +43,15 @@ ++ on-save !>(state) ++ on-load |= =vase + |^ =/ old ((soft versioned-state) q.vase) ?~ old on-init =/ old u.old - |^ + |- ?- -.old - %1 `this(state old) - %0 `this(state (state-0-to-1 old)) + %0 $(old (state-0-to-1 old)) + %1 $(old (state-1-to-2 old)) + %2 `this(state old) == ++ state-0-to-1 |= zer=state-zero @@ -56,12 +60,31 @@ credentials.zer (configuration-0-to-1 configuration.zer) == + :: ++ configuration-0-to-1 |= conf=configuration:zero:past + ^- configuration:one:past + :* buckets.conf + current-bucket.conf + '' + == + :: + ++ state-1-to-2 + |= one=state-one + ^- state-two + :* %2 + credentials.one + (configuration-1-to-2 configuration.one) + == + :: + ++ configuration-1-to-2 + |= conf=configuration:one:past ^- ^configuration :* buckets.conf current-bucket.conf + region.conf '' + %credentials == -- :: @@ -127,6 +150,12 @@ :: %remove-bucket state(buckets.configuration (~(del in buckets.configuration) bucket.act)) + :: + %set-presigned-url + state(presigned-url.configuration url.act) + :: + %toggle-service + state(service.configuration service.act) == -- :: diff --git a/desk/lib/storage-json.hoon b/desk/lib/storage-json.hoon index f4d02be5..8f887214 100644 --- a/desk/lib/storage-json.hoon +++ b/desk/lib/storage-json.hoon @@ -14,6 +14,8 @@ [%add-bucket so:dejs] [%remove-bucket so:dejs] [%set-current-bucket so:dejs] + [%set-presigned-url so:dejs] + [%toggle-service (su:dejs (perk %presigned-url %credentials ~))] == -- :: @@ -30,6 +32,8 @@ %remove-bucket [%'removeBucket' s+bucket.upd] %set-endpoint [%'setEndpoint' s+endpoint.upd] %set-access-key-id [%'setAccessKeyId' s+access-key-id.upd] + %set-presigned-url [%'setPresignedUrl' s+url.upd] + %toggle-service [%'toggleService' s+service.upd] %set-secret-access-key [%'setSecretAccessKey' s+secret-access-key.upd] :: @@ -47,6 +51,8 @@ :~ [%buckets a+(turn ~(tap in buckets.configuration.upd) |=(a=@t s+a))] [%'currentBucket' s+current-bucket.configuration.upd] [%'region' s+region.configuration.upd] + [%'service' s+service.configuration.upd] + [%'presignedUrl' s+presigned-url.configuration.upd] == == == diff --git a/desk/sur/storage-1.hoon b/desk/sur/storage-1.hoon new file mode 100644 index 00000000..c2a6175a --- /dev/null +++ b/desk/sur/storage-1.hoon @@ -0,0 +1,29 @@ +|% ++$ credentials + $: endpoint=@t + access-key-id=@t + secret-access-key=@t + == +:: ++$ configuration + $: buckets=(set @t) + current-bucket=@t + region=@t + == +:: ++$ action + $% [%set-endpoint endpoint=@t] + [%set-access-key-id access-key-id=@t] + [%set-secret-access-key secret-access-key=@t] + [%add-bucket bucket=@t] + [%remove-bucket bucket=@t] + [%set-current-bucket bucket=@t] + [%set-region region=@t] + == +:: ++$ update + $% [%credentials =credentials] + [%configuration =configuration] + action + == +-- diff --git a/desk/sur/storage.hoon b/desk/sur/storage.hoon index 8afea6a8..6a8387c8 100644 --- a/desk/sur/storage.hoon +++ b/desk/sur/storage.hoon @@ -1,19 +1,31 @@ -/- zer=storage-0 +/- zer=storage-0, uno=storage-1 |% ++ past |% ++ zero zer + ++ one uno -- ++$ service ?(%presigned-url %credentials) +$ credentials $: endpoint=@t access-key-id=@t secret-access-key=@t == :: +:: $configuration: the upload configuration +:: +:: $buckets: the buckets available +:: $current-bucket: the current bucket we use to upload +:: $region: the region of the current bucket +:: $presigned-url: the presigned url endpoint +:: $service: whether to use a presigned url service or direct S3 uploads +:: +$ configuration $: buckets=(set @t) current-bucket=@t region=@t + presigned-url=@t + =service == :: +$ action @@ -24,6 +36,8 @@ [%remove-bucket bucket=@t] [%set-current-bucket bucket=@t] [%set-region region=@t] + [%set-presigned-url url=@t] + [%toggle-service =service] == :: +$ update diff --git a/ui/src/gear/storage/lib.ts b/ui/src/gear/storage/lib.ts index 6b397206..3a6337df 100644 --- a/ui/src/gear/storage/lib.ts +++ b/ui/src/gear/storage/lib.ts @@ -1,52 +1,75 @@ import { Poke } from '@urbit/http-api'; -import { StorageUpdate, StorageUpdateCurrentBucket, StorageUpdateAddBucket, StorageUpdateRemoveBucket, StorageUpdateEndpoint, StorageUpdateAccessKeyId, StorageUpdateSecretAccessKey, StorageUpdateRegion } from './types'; +import { + StorageUpdate, + StorageUpdateCurrentBucket, + StorageUpdateAddBucket, + StorageUpdateRemoveBucket, + StorageUpdateEndpoint, + StorageUpdateAccessKeyId, + StorageUpdateSecretAccessKey, + StorageUpdateRegion, + StorageService, + StorageUpdateSetPresignedUrl, + StorageUpdateToggleService, +} from './types'; -const storageAction = ( - data: any -): Poke => ({ +const storageAction = (data: any): Poke => ({ app: 'storage', mark: 'storage-action', - json: data + json: data, }); export const setCurrentBucket = ( bucket: string -): Poke => storageAction({ - 'set-current-bucket': bucket -}); +): Poke => + storageAction({ + 'set-current-bucket': bucket, + }); -export const addBucket = ( - bucket: string -): Poke => storageAction({ - 'add-bucket': bucket -}); +export const addBucket = (bucket: string): Poke => + storageAction({ + 'add-bucket': bucket, + }); -export const removeBucket = ( - bucket: string -): Poke => storageAction({ - 'remove-bucket': bucket -}); +export const removeBucket = (bucket: string): Poke => + storageAction({ + 'remove-bucket': bucket, + }); -export const setEndpoint = ( - endpoint: string -): Poke => storageAction({ - 'set-endpoint': endpoint -}); +export const setEndpoint = (endpoint: string): Poke => + storageAction({ + 'set-endpoint': endpoint, + }); export const setAccessKeyId = ( accessKeyId: string -): Poke => storageAction({ - 'set-access-key-id': accessKeyId -}); +): Poke => + storageAction({ + 'set-access-key-id': accessKeyId, + }); export const setSecretAccessKey = ( secretAccessKey: string -): Poke => storageAction({ - 'set-secret-access-key': secretAccessKey -}); +): Poke => + storageAction({ + 'set-secret-access-key': secretAccessKey, + }); + +export const setRegion = (region: string): Poke => + storageAction({ + 'set-region': region, + }); + +export const setPresignedUrl = ( + presignedUrl: string +): Poke => + storageAction({ + 'set-presigned-url': presignedUrl, + }); -export const setRegion = ( - region: string -): Poke => storageAction({ - 'set-region': region -}) +export const toggleService = ( + service: StorageService +): Poke => + storageAction({ + 'toggle-service': service, + }); diff --git a/ui/src/gear/storage/types.ts b/ui/src/gear/storage/types.ts index 4f37d44b..7887edb8 100644 --- a/ui/src/gear/storage/types.ts +++ b/ui/src/gear/storage/types.ts @@ -2,6 +2,16 @@ import { S3Client } from '@aws-sdk/client-s3'; export type Status = 'initial' | 'idle' | 'loading' | 'success' | 'error'; +export type StorageService = 'presigned-url' | 'credentials'; + +export interface StorageConfiguration { + buckets: Set; + currentBucket: string; + region: string; + presignedUrl: string; + service: StorageService; +} + export interface StorageCredentials { endpoint: string; accessKeyId: string; @@ -12,11 +22,7 @@ export interface BaseStorageState { loaded?: boolean; hasCredentials?: boolean; s3: { - configuration: { - buckets: Set; - currentBucket: string; - region: string; - }; + configuration: StorageConfiguration; credentials: StorageCredentials | null; }; [ref: string]: unknown; @@ -67,9 +73,13 @@ export interface FileStore { uploadFiles: ( uploader: string, files: FileList | null, - bucket: string + config: StorageConfiguration + ) => Promise; + upload: ( + uploader: string, + upload: Upload, + config: StorageConfiguration ) => Promise; - upload: (uploader: string, upload: Upload, bucket: string) => Promise; clear: (uploader: string) => void; updateFile: ( uploader: string, @@ -125,6 +135,14 @@ export interface StorageUpdateRegion { setRegion: string; } +export interface StorageUpdateToggleService { + toggleService: string; +} + +export interface StorageUpdateSetPresignedUrl { + setPresignedUrl: string; +} + export declare type StorageUpdate = | StorageUpdateCredentials | StorageUpdateConfiguration @@ -134,4 +152,6 @@ export declare type StorageUpdate = | StorageUpdateEndpoint | StorageUpdateAccessKeyId | StorageUpdateSecretAccessKey - | StorageUpdateRegion; + | StorageUpdateRegion + | StorageUpdateToggleService + | StorageUpdateSetPresignedUrl; diff --git a/ui/src/preferences/StoragePrefs.tsx b/ui/src/preferences/StoragePrefs.tsx index 1d57211f..2b5c8088 100644 --- a/ui/src/preferences/StoragePrefs.tsx +++ b/ui/src/preferences/StoragePrefs.tsx @@ -7,6 +7,13 @@ import { useStorage } from '../state/storage'; import { Button } from '../components/Button'; import { Spinner } from '../components/Spinner'; import { Urbit } from '@urbit/http-api'; +import { + StorageUpdate, + StorageUpdateToggleService, + toggleService, +} from '@/gear'; +import { isHosted } from '@/logic/utils'; +import { Toggle } from '@/components/Toggle'; interface CredentialsSubmit { endpoint: string; @@ -34,7 +41,8 @@ function storagePoke(data: S3Update | { 'set-region': string }) { } export const StoragePrefs = () => { - const { s3, loaded, ...storageState } = useStorage(); + const { s3, loaded } = useStorage(); + const hostedStorage = s3.configuration.service === 'presigned-url'; const { register, @@ -55,6 +63,21 @@ export const StoragePrefs = () => { }, []) ); + const { call: toggleS3, status: toggleStatus } = useAsyncCall( + useCallback(() => { + return api.trackedPoke< + StorageUpdateToggleService, + { 'storage-update': StorageUpdate } + >( + toggleService(hostedStorage ? 'credentials' : 'presigned-url'), + { app: 'storage', path: '/all' }, + (event) => + 'storage-update' in event && + 'toggleService' in event['storage-update'] + ); + }, [hostedStorage]) + ); + useEffect(() => { useStorage.getState().initialize(api as unknown as Urbit); }, []); @@ -63,133 +86,161 @@ export const StoragePrefs = () => { loaded && reset(); }, [loaded, reset]); + console.log(s3.configuration, toggleStatus); + return ( -
+

Remote Storage

-
-

- Configure your urbit to enable uploading your own images or other - files in Urbit applications. -

-

- Read more about setting up S3 storage in the{' '} - - Urbit Operator's Manual - - . -

-
-
-
- -
- - {!loaded && } -
-
-
- -
- - {!loaded && } -
-
-
- -
- - {!loaded && } -
-
-
- -
- + {isHosted ? ( +
+ + - {!loaded && }
-
-
- -
- - {!loaded && } -
-
- - + ) : null} + {hostedStorage && isHosted ? ( +

+ Your Tlon-hosted urbit comes with free image hosting for Groups and + Talk. If you would like to use your own S3-compatible back-end for + image hosting, you can enable it on this screen. +

+ ) : ( + <> +

+ Configure your urbit to enable uploading your own images or other + files in Urbit applications. +

+

+ Read more about setting up S3 storage in the{' '} + + Urbit Operator's Manual + + . +

+ +
+
+ +
+ + {!loaded && } +
+
+
+ +
+ + {!loaded && } +
+
+
+ +
+ + {!loaded && } +
+
+
+ +
+ + {!loaded && } +
+
+
+ +
+ + {!loaded && } +
+
+ +
+ + )} +
); }; diff --git a/ui/src/state/storage/reducer.ts b/ui/src/state/storage/reducer.ts index a68bbe08..472bfd2c 100644 --- a/ui/src/state/storage/reducer.ts +++ b/ui/src/state/storage/reducer.ts @@ -6,7 +6,10 @@ import { BaseState } from '../base'; export type StorageState = BaseStorageState & BaseState; -const credentials = (json: StorageUpdate, state: StorageState): StorageState => { +const credentials = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'credentials', false); if (data) { state.s3.credentials = data; @@ -14,19 +17,27 @@ const credentials = (json: StorageUpdate, state: StorageState): StorageState => return state; }; -const configuration = (json: StorageUpdate, state: StorageState): StorageState => { +const configuration = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'configuration', false); if (data) { state.s3.configuration = { buckets: new Set(data.buckets), currentBucket: data.currentBucket, region: data.region, + presignedUrl: data.presignedUrl, + service: data.service, }; } return state; }; -const currentBucket = (json: StorageUpdate, state: StorageState): StorageState => { +const currentBucket = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setCurrentBucket', false); if (data && state.s3) { state.s3.configuration.currentBucket = data; @@ -42,7 +53,10 @@ const addBucket = (json: StorageUpdate, state: StorageState): StorageState => { return state; }; -const removeBucket = (json: StorageUpdate, state: StorageState): StorageState => { +const removeBucket = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'removeBucket', false); if (data) { state.s3.configuration.buckets.delete(data); @@ -58,7 +72,10 @@ const endpoint = (json: StorageUpdate, state: StorageState): StorageState => { return state; }; -const accessKeyId = (json: StorageUpdate, state: StorageState): StorageState => { +const accessKeyId = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setAccessKeyId', false); if (data && state.s3.credentials) { state.s3.credentials.accessKeyId = data; @@ -66,7 +83,10 @@ const accessKeyId = (json: StorageUpdate, state: StorageState): StorageState => return state; }; -const secretAccessKey = (json: StorageUpdate, state: StorageState): StorageState => { +const secretAccessKey = ( + json: StorageUpdate, + state: StorageState +): StorageState => { const data = _.get(json, 'setSecretAccessKey', false); if (data && state.s3.credentials) { state.s3.credentials.secretAccessKey = data; @@ -82,6 +102,28 @@ const region = (json: StorageUpdate, state: StorageState): StorageState => { return state; }; +const presignedUrl = ( + json: StorageUpdate, + state: StorageState +): StorageState => { + const data = _.get(json, 'setPresignedUrl', false); + if (data && state.s3.configuration) { + state.s3.configuration.presignedUrl = data; + } + return state; +}; + +const toggleService = ( + json: StorageUpdate, + state: StorageState +): StorageState => { + const data = _.get(json, 'toggleService', false); + if (data && state.s3.configuration) { + state.s3.configuration.service = data; + } + return state; +}; + const reduce = [ credentials, configuration, @@ -92,6 +134,8 @@ const reduce = [ accessKeyId, secretAccessKey, region, + presignedUrl, + toggleService, ]; export default reduce; diff --git a/ui/src/state/storage/storage.ts b/ui/src/state/storage/storage.ts index a342b8b1..61cb2ff2 100644 --- a/ui/src/state/storage/storage.ts +++ b/ui/src/state/storage/storage.ts @@ -25,6 +25,8 @@ export const useStorage = createState( buckets: new Set(), currentBucket: '', region: '', + presignedUrl: '', + service: 'credentials', }, credentials: null, }, diff --git a/ui/src/state/storage/upload.ts b/ui/src/state/storage/upload.ts index 7bfdd728..75d655a0 100644 --- a/ui/src/state/storage/upload.ts +++ b/ui/src/state/storage/upload.ts @@ -6,11 +6,17 @@ import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getImageSize } from 'react-image-size'; import { useCallback, useEffect, useState } from 'react'; -import { FileStore, Status, StorageCredentials, Uploader } from '@/gear'; +import { + FileStore, + Status, + StorageConfiguration, + StorageCredentials, + Uploader, +} from '@/gear'; import { useStorage } from './storage'; -import { StorageState } from './reducer'; +import api from '@/api'; -export function prefixEndpoint(endpoint: string) { +function prefixEndpoint(endpoint: string) { return endpoint.match(/https?:\/\//) ? endpoint : `https://${endpoint}`; } @@ -48,12 +54,14 @@ export const useFileStore = create((set, get) => ({ update: (key: string, updateFn: (uploader: Uploader) => void) => { set(produce((draft) => updateFn(draft.uploaders[key]))); }, - uploadFiles: async (uploader, files, bucket) => { + uploadFiles: async (uploader, files, config) => { if (!files) return; const fileList = [...files].map((file) => ({ file, - key: `${window.ship}/${deSig(formatDa(unixToDa(new Date().getTime())))}-${file.name}`, + key: `${window.ship}/${deSig(formatDa(unixToDa(new Date().getTime())))}-${ + file.name + }`, status: 'initial' as Status, url: '', size: [0, 0] as [number, number], @@ -67,49 +75,100 @@ export const useFileStore = create((set, get) => ({ draft.files = { ...draft.files, ...newFiles }; }); - fileList.forEach((f) => upload(uploader, f, bucket)); + fileList.forEach((f) => upload(uploader, f, config)); }, - upload: async (uploader, upload, bucket) => { + upload: async (uploader, upload, config) => { const { client, updateStatus, updateFile } = get(); - if (!client) { - return; - } const { key, file } = upload; updateStatus(uploader, key, 'loading'); - const command = new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: file, - ContentType: file.type, - ContentLength: file.size, - ACL: 'public-read', - }); + // Logic for uploading with Tlon Hosting storage. + if (config.service === 'presigned-url' && config.presignedUrl) { + // The first step is to send the PUT request to the proxy, which will + // respond with a redirect to a pre-signed url to the actual bucket. The + // token is in the url, not a header, so that it disappears after the + // redirect. + const requestOptions = { + method: 'PUT', + headers: { + 'Content-Type': file.type, + }, + body: file, + }; + const { presignedUrl } = config; + const url = `${presignedUrl}/${key}`; + const token = await api.scry({ + app: 'genuine', + path: '/secret', + }); + const urlWithToken = `${url}?token=${token}`; + fetch(urlWithToken, requestOptions) + .then(async (response) => { + if (response.status !== 200) { + const body = await response.text(); + throw new Error(body || 'Incorrect response status'); + } + // When the PUT succeeded, we fetch the actual URL of the file. We do + // this to avoid having to proxy every single GET request, and to + // avoid remembering which file corresponds to which bucket, when + // using multiple buckets internally. + const fileUrlResponse = await fetch(url); + const fileUrl = await fileUrlResponse.json(); + updateStatus(uploader, key, 'success'); + imageSize(fileUrl).then((s) => + updateFile(uploader, key, { + size: s, + url: fileUrl, + }) + ); + }) + .catch((error: any) => { + updateStatus( + uploader, + key, + 'error', + `Tlon Hosting upload error: ${error.message}, contact support if it persists.` + ); + console.log({ error }); + }); + } - const url = await getSignedUrl(client, command); - - client - .send(command) - .then(() => { - const fileUrl = url.split('?')[0]; - updateStatus(uploader, key, 'success'); - imageSize(fileUrl).then((s) => - updateFile(uploader, key, { - size: s, - url: fileUrl, - }) - ); - }) - .catch((error: any) => { - updateStatus( - uploader, - key, - 'error', - `S3 upload error: ${error.message}, check your S3 configuration.` - ); - console.log({ error }); + // Logic for uploading with S3. + if (config.service === 'credentials' && client) { + const command = new PutObjectCommand({ + Bucket: config.currentBucket, + Key: key, + Body: file, + ContentType: file.type, + ContentLength: file.size, + ACL: 'public-read', }); + + const url = await getSignedUrl(client, command); + + client + .send(command) + .then(() => { + const fileUrl = url.split('?')[0]; + updateStatus(uploader, key, 'success'); + imageSize(fileUrl).then((s) => + updateFile(uploader, key, { + size: s, + url: fileUrl, + }) + ); + }) + .catch((error: any) => { + updateStatus( + uploader, + key, + 'error', + `S3 upload error: ${error.message}, check your S3 configuration.` + ); + console.log({ error }); + }); + } }, clear: (uploader) => { get().update(uploader, (draft) => { @@ -151,11 +210,14 @@ export const useFileStore = create((set, get) => ({ }, })); -const emptyUploader = (key: string, bucket: string): Uploader => ({ +const emptyUploader = ( + key: string, + config: StorageConfiguration +): Uploader => ({ files: {}, getMostRecent: () => useFileStore.getState().getMostRecent(key), uploadFiles: async (files) => - useFileStore.getState().uploadFiles(key, files, bucket), + useFileStore.getState().uploadFiles(key, files, config), clear: () => useFileStore.getState().clear(key), removeByURL: (url) => useFileStore.getState().removeByURL(key, url), prompt: () => { @@ -166,14 +228,17 @@ const emptyUploader = (key: string, bucket: string): Uploader => ({ input.accept = 'image/*,video/*,audio/*'; input.addEventListener('change', async (e) => { const { files } = e.target as HTMLInputElement; - useFileStore.getState().uploadFiles(key, files, bucket); + useFileStore.getState().uploadFiles(key, files, config); input.remove(); }); + // Add to DOM for mobile Safari support + input.classList.add('hidden'); + document.body.appendChild(input); input.click(); }, }); -export function useClient() { +function useClient() { const { s3: { credentials, configuration }, } = useStorage(); @@ -182,13 +247,14 @@ export function useClient() { useEffect(() => { const hasCreds = + configuration.service === 'credentials' && credentials?.accessKeyId && credentials?.endpoint && credentials?.secretAccessKey; if (hasCreds) { setHasCredentials(true); } - }, [credentials]); + }, [credentials, configuration]); const initClient = useCallback(async () => { if (credentials) { @@ -205,24 +271,26 @@ export function useClient() { return client; } -const selS3 = (s: StorageState) => s.s3; const selUploader = (key: string) => (s: FileStore) => s.uploaders[key]; export function useUploader(key: string): Uploader | undefined { const { - configuration: { currentBucket }, - } = useStorage(selS3); + s3: { configuration }, + } = useStorage(); const client = useClient(); const uploader = useFileStore(selUploader(key)); useEffect(() => { - if (client && currentBucket) { + if ( + (client && configuration.service === 'credentials') || + (configuration.service === 'presigned-url' && configuration.presignedUrl) + ) { useFileStore.setState( produce((draft) => { - draft.uploaders[key] = emptyUploader(key, currentBucket); + draft.uploaders[key] = emptyUploader(key, configuration); }) ); } - }, [client, currentBucket, key]); + }, [client, configuration, key]); return uploader; }