Merge remote-tracking branch 'origin/main' into feat/support-multiple…

…-gateway-urls

.env.tpl

@@ -54,6 +54,12 @@ S3_SECRET_ACCESS_KEY_ID = 'minioadmin'
 # URL prefix for the linkdex-api
 # LINKDEX_URL = "https://linkdex.web3.storage"
 
+# Content Claims
+# CONTENT_CLAIMS_PRIVATE_KEY = "base64pad encoded ed25519 key"
+# CONTENT_CLAIMS_PROOF = "base64pad encoded CAR of delegation"
+# CONTENT_CLAIMS_SERVICE_DID = "did:web:claims.web3.storage"
+# CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage"
+
 ## ---- website ---------------------------------------------------------------
 
 # vars to expose to the public website build have to be prefixed with NEXT_PUBLIC_

.github/CODEOWNERS

@@ -6,7 +6,7 @@
 
 # github's syntax docs: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax
 
-* @web3-storage/dag-house
+* @web3-storage/dag-house-devs
 
 # @web3-storage/website is maintained by multiple teams
-/packages/website/  @web3-storage/dag-house @web3-storage/website-codeowners
+/packages/website/  @web3-storage/dag-house-devs @web3-storage/website-codeowners

.github/workflows/api.yml

@@ -23,7 +23,7 @@ jobs:
       - uses: microsoft/playwright-github-action@v1
       - uses: actions/setup-node@v2
         with:
-          node-version: 16
+          node-version: 18
       - uses: bahmutov/npm-install@v1
       - run: npm run build --workspace packages/api
       - run: npm test --workspace packages/api
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 16
+          node-version: 18
       - uses: bahmutov/npm-install@v1
       - name: Publish api worker
         uses: cloudflare/[email protected]
@@ -91,7 +91,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
+          node-version: 18
           registry-url: https://registry.npmjs.org/
       - uses: bahmutov/npm-install@v1
       - name: API - Deploy to Cloudflare

.github/workflows/client.yml

@@ -18,14 +18,23 @@ jobs:
   test:
     runs-on: ubuntu-latest
     name: Test
+    strategy:
+      matrix:
+        node_version:
+          - 16
+          - 18
+          - 20
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2
         with:
-          node-version: 16
+          node-version: ${{ matrix.node_version }}
       - uses: bahmutov/npm-install@v1
       - run: npm run build --workspace packages/client
       - run: npm test --workspace packages/client
+      - name: Bundle Size
+        if: matrix.node_version == 18 # does not work on node.js 20 yet
+        run: npm run test:size --workspace packages/client
       - name: Test upload to staging
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         run: |
@@ -50,7 +59,7 @@ jobs:
       - uses: actions/setup-node@v2
         if: ${{ steps.tag-release.outputs.releases_created }}
         with:
-          node-version: '16'
+          node-version: 18
           registry-url: https://registry.npmjs.org/
       - uses: bahmutov/npm-install@v1
         if: ${{ steps.tag-release.outputs.releases_created }}

.github/workflows/cron-pins.yml

@@ -2,7 +2,7 @@ name: Cron Pins
 
 on:
   schedule:
-    - cron: '*/30 * * * *'
+    - cron: '*/60 * * * *'
   workflow_dispatch:
 
 jobs:
@@ -12,7 +12,12 @@ jobs:
     strategy:
       matrix:
         env: ['staging', 'production']
-    timeout-minutes: 30
+        include:
+          - env: production
+            pickup_url: https://pickup.dag.haus
+          - env: staging
+            pickup_url: https://staging.pickup.dag.haus
+    timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
         with: 
@@ -37,10 +42,8 @@ jobs:
           STAGING_PG_REST_JWT: ${{ secrets.STAGING_PG_REST_JWT }}
           PROD_PG_REST_URL: ${{ secrets.PROD_PG_REST_URL }}
           STAGING_PG_REST_URL: ${{ secrets.STAGING_PG_REST_URL }}
-          CLUSTER_API_URL: ${{ secrets.CLUSTER_API_URL }}
+          CLUSTER_API_URL: ${{ matrix.pickup_url }}
           CLUSTER_BASIC_AUTH_TOKEN: ${{ secrets.CLUSTER_BASIC_AUTH_TOKEN }}
-          CLUSTER_IPFS_PROXY_API_URL: ${{ secrets.CLUSTER_IPFS_PROXY_API_URL }}
-          CLUSTER_IPFS_PROXY_BASIC_AUTH_TOKEN: ${{ secrets.CLUSTER_IPFS_PROXY_BASIC_AUTH_TOKEN }}
         run: npm run start:pins -w packages/cron
 
       - name: Heartbeat

.github/workflows/cron-test-pinning.yml

@@ -0,0 +1,52 @@
+name: Pinning Test
+
+on:
+  push:
+    paths:
+      - '.github/workflows/cron-test-pinning.yml'
+  schedule:
+    - cron: '50 * * * *'
+  workflow_dispatch:
+
+jobs:
+  psa-test:
+    name: Pinning Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Setup kubo
+        uses: ibnesayeed/setup-ipfs@aff2e4ae38368a44ac36ecc244f8c693d2e5b263
+        with:
+          ipfs_version: '0.18.1'
+          run_daemon: true
+
+      - name: IPFS debug info
+        run: |
+          # print multiaddrs for the local node
+          ipfs id -f="<addrs>"
+          ipfs version
+
+      - name: Create a 3MiB file and add it to kubo
+        run: |
+          dd if=/dev/urandom of=./pin-test-3mib bs=1024 count=3072
+          ipfs add --quiet --cid-version=1 ./pin-test-3mib > ./cid-to-pin
+          echo "Added $(cat ./cid-to-pin)"
+
+      # Create a random file. use kubo to pin it to web3.storage and time the duration in seconds
+      # `usr/bin/time -f "%e" -o ./duration_secs` writes the wall-clock runtime in seconds of the command to the file ./duration_secs
+      - name: Pin 3MiB file to web3.storage via kubo
+        run: |
+          ipfs pin remote service add web3 "https://api.web3.storage" ${{ secrets.WEB3_TOKEN }}
+          echo "Pinning $(cat ./cid-to-pin)"
+          /usr/bin/time -f "%e" -o ./pin_duration_secs ipfs pin remote add --service=web3 --name=foo $(cat ./cid-to-pin)
+          cat ./pin_duration_secs
+
+      # format it as a prometheus metric and send it to the push gateway
+      # see: https://github.com/prometheus/pushgateway/blob/master/README.md
+      - name: Push duration metric to prometheus
+        run: |
+          echo "# HELP web3storage_pin_duration_seconds How long it took to pin a file via kubo" > ./pin_metric
+          echo "# TYPE web3storage_pin_duration_seconds gauge" >> ./pin_metric
+          echo "web3storage_pin_duration_seconds{size=\"3MiB\",cmd=\"kubo\"} $(cat ./pin_duration_secs)" >> ./pin_metric
+          cat ./pin_metric
+          curl --silent --show-error --data-binary "@./pin_metric" ${{ secrets.PUSHGATEWAY_URL }}/metrics/job/web3storage_ci/instance/github_action

.github/workflows/cron-test.yml → .github/workflows/cron-test-upload.yml

@@ -3,7 +3,7 @@ name: Upload test
 on:
   push:
     paths:
-      - '.github/workflows/cron-test.yml'
+      - '.github/workflows/cron-test-upload.yml'
   schedule:
     - cron: '*/10 * * * *'
   workflow_dispatch:
@@ -39,7 +39,7 @@ jobs:
           echo "# TYPE web3storage_upload_duration_seconds gauge" >> ./metric
           echo "web3storage_upload_duration_seconds{endpoint=\"/car\",size=\"3MiB\",cmd=\"w3\"} $(cat ./duration_secs)" >> ./metric
           cat ./metric
-          curl --silent --show-error --data-binary "@./metric" --basic --user "${{ secrets.PUSHGATEWAY_BASIC_AUTH }}" https://pushgateway.k8s.locotorp.info/metrics/job/web3storage_ci/instance/github_action
+          curl --silent --show-error --data-binary "@./metric" ${{ secrets.PUSHGATEWAY_URL }}/metrics/job/web3storage_ci/instance/github_action
 
       - name: Heartbeat
         if: ${{ success() }}

.github/workflows/lint.yml

@@ -13,9 +13,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: codepunkt/[email protected]
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/setup-node@v2
         with:
           node-version: 16

.github/workflows/website.yml

@@ -49,6 +49,7 @@ jobs:
       - run: npm run lint -w packages/website
 
   test-e2e:
+    if: false
     name: "Test e2e - ${{ matrix.browser }}"
     strategy:
       fail-fast: false
@@ -57,7 +58,7 @@ jobs:
           - firefox
           - chromium
         os:
-          - ubuntu-18.04
+          - ubuntu-latest
         node-version:
           - 16
         test_results_path:
@@ -88,12 +89,13 @@ jobs:
 
       - run: npx playwright install --with-deps ${{ matrix.browser }}
         working-directory: packages/website
-      
+
       # run e2e tests
       - id: website_test_e2e
         continue-on-error: true # when this fails, we still want to do some cleanup
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- node -r dotenv/config $(which npm) run -w @web3-storage/website test:e2e -- --project=${{ matrix.browser }} --reporter=html
         env:
+          # Timeout is in milliseconds
           PLAYWRIGHT_TIMEOUT: 120000
           NEXT_PUBLIC_MAGIC: ${{ secrets.STAGING_MAGIC_PUBLIC_KEY }}
           MAGIC_SECRET_KEY: ${{ secrets.STAGING_MAGIC_SECRET_KEY }}
@@ -191,6 +193,8 @@ jobs:
           NEXT_PUBLIC_COUNTLY_URL: ${{ secrets.COUNTLY_URL }}
           NEXT_PUBLIC_COUNTLY_KEY: ${{ secrets.COUNTLY_KEY }}
           NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: ${{ secrets.TESTING_STRIPE_PUBLISHABLE_KEY }}
+          DID_DOCUMENT_ID: ${{ secrets.STAGING_DID_DOCUMENT_ID }}
+          DID_DOCUMENT_PRIMARY_DID_KEY: ${{ secrets.STAGING_DID_DOCUMENT_PRIMARY_DID_KEY }}
       - name: Add to web3.storage
         uses: web3-storage/add-to-web3@v2
         id: ipfs
@@ -260,12 +264,12 @@ jobs:
     if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changelog.outputs.releases_created) || inputs.force_release
     name: Release
     runs-on: ubuntu-latest
-    environment: 
+    environment:
       name: production
       url: https://web3.storage
     needs:
       - test
-      - test-e2e
+      # - test-e2e
       - preview
       - changelog
     steps:
@@ -284,6 +288,8 @@ jobs:
           NEXT_PUBLIC_COUNTLY_URL: ${{ secrets.COUNTLY_URL }}
           NEXT_PUBLIC_COUNTLY_KEY: ${{ secrets.COUNTLY_KEY }}
           NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY: ${{ secrets.STRIPE_PUBLISHABLE_KEY }}
+          DID_DOCUMENT_ID: ${{ secrets.PRODUCTION_DID_DOCUMENT_ID }}
+          DID_DOCUMENT_PRIMARY_DID_KEY: ${{ secrets.PRODUCTION_DID_DOCUMENT_PRIMARY_DID_KEY }}
       - name: Add to web3.storage
         uses: web3-storage/add-to-web3@v2
         id: ipfs

PEERS

@@ -9,41 +9,3 @@
 
 # Elastic IPFS
 /dns4/elastic.dag.house/tcp/443/wss/p2p/bafzbeibhqavlasjc7dvbiopygwncnrtvjd2xmryk5laib7zyjor6kf3avm
-# web3-storage-am6
-/ip4/147.75.33.191/tcp/4001/p2p/12D3KooWPySxxWQjBgX9Jp6uAHQfVmdq8HG1gVvS1fRawHNSrmqW
-# web3-storage-am6-3
-/ip4/147.75.80.9/tcp/4001/p2p/12D3KooWQYBPcvxFnnWzPGEx6JuBnrbF1FZq4jTahczuG2teEk1m
-# web3-storage-am6-4
-/ip4/147.75.80.39/tcp/4001/p2p/12D3KooWDdzN3snjaMJEH9zuq3tjKUFpYHeSGNkiAreF6dQSbCiL
-# web3-storage-am6-5
-/ip4/147.75.80.143/tcp/4001/p2p/12D3KooWEzCun34s9qpYEnKkG6epx2Ts9oVGRGnzCvM2s2edioLA
-# web3-storage-am6-6
-/ip4/147.75.84.119/tcp/4001/p2p/12D3KooWQE3CWA3MJ1YhrYNP8EE3JErGbrCtpKRkFrWgi45nYAMn
-# web3-storage-am6-7
-/ip4/147.75.84.175/tcp/4001/p2p/12D3KooWDYVuVFGb9Yj6Gi9jWwSLqdnzZgqJg1a1scQMDc4R6RUJ
-# web3-storage-am6-8
-/ip4/147.75.84.173/tcp/4001/p2p/12D3KooWSafoW6yrSL7waghFAaiCqGy5mdjpQx4jn4CRNqbG7eqG
-# web3-storage-dc13-3
-/ip4/136.144.57.15/tcp/4001/p2p/12D3KooWJEfH2MB4RsUoaJPogDPRWbFTi8iehsxsqrQpiJwFNDrP
-# web3-storage-dc13-4
-/ip4/147.75.63.131/tcp/4001/p2p/12D3KooWHpE5KiQTkqbn8KbU88ZxwJxYJFaqP4mp9Z9bhNPhym9V
-# web3-storage-dc13-5
-/ip4/147.75.62.95/tcp/4001/p2p/12D3KooWBHvsSSKHeragACma3HUodK5FcPUpXccLu2vHooNsDf9k
-# web3-storage-dc13-6
-/ip4/147.75.50.77/tcp/4001/p2p/12D3KooWMaTJKNwQJyP1fw3ftGb5uqqM2U24Kam8aWqMRXzWHNiF
-# web3-storage-dc13-7
-/ip4/147.75.50.141/tcp/4001/p2p/12D3KooWNCmYvqPbeXmNC4rnTr7hbuVtJKDNpL1vvNz6mq9Sr2Xf
-# web3-storage-dc13-8
-/ip4/147.28.147.193/tcp/4001/p2p/12D3KooWDRak1XzURGh9MvGR4EWaP9kcbmdoagAcGMcNxBXXLzTF
-# web3-storage-sv15-3
-/ip4/139.178.69.93/tcp/4001/p2p/12D3KooWRi18oHN1j8McxS9RMnuibcTwxu6VCTYHyLNH2R14qhTy
-# web3-storage-sv15-4
-/ip4/139.178.91.227/tcp/4001/p2p/12D3KooWKhPb9tSnCqBswVfC5EPE7iSTXhbF4Ywwz2MKg5UCagbr
-# web3-storage-sv15-5
-/ip4/139.178.91.231/tcp/4001/p2p/12D3KooWAdxvJCV5KXZ6zveTJmnYGrSzAKuLUKZYkZssLk7UKv4i
-# web3-storage-sv15-6
-/ip4/147.75.49.91/tcp/4001/p2p/12D3KooWRgXWwnZQJgdW1GHW7hJ5UvZ8MLp7HBCSWS596PypAs8M
-# web3-storage-sv15-7
-/ip4/139.178.88.145/tcp/4001/p2p/12D3KooWPbxiW4wFYHs7MwCQNqK9YVedH7QYZXJKMFVduhwR1Lcs
-# web3-storage-sv15-8
-/ip4/145.40.90.155/tcp/4001/p2p/12D3KooWSH5uLrYe7XSFpmnQj1NCsoiGeKSRCV7T5xijpX2Po2aT

README.md

@@ -1,4 +1,5 @@
-<h1 align="center">⁂<br/>web3.storage</h1>
+<h1 align="center"><a href="https://web3.storage"><img width="25%" src="https://user-images.githubusercontent.com/11778450/227262707-1a9674a7-9286-43e3-87b4-98b388677720.png" alt="web3.storage logo" /></a>
+</h1>
 <p align="center">The simple file storage service for IPFS &amp; Filecoin.</p>
 
 ## Usage