Performance Improvements for Buildbarn deployments K8's (Work in Progress)

kubernetes/README.md

@@ -9,3 +9,20 @@ kubectl apply -k .
 
 These files assume that the cluster needs to be created in the
 `buildbarn` namespace. Storage is backed by persistent volumes.
+
+
+**Prototyping/Quickstart**
+To run against the cluster (for protyping & debugging), forward port 8980 in one terminal
+```sh
+while true; do kubectl --n buildbarn port-forward services/frontend <your-bazel-grpc-port>:8980; done
+```
+
+**Debugging**:
+Check Pods are up and get logs within the pods
+```sh
+kubectl get pods -n buildbarn
+kubectl logs <worker-ubuntu-pod-name> -n buildbarn
+```
+
+**Note**
+- Yaml files here work for microk8s and k3s, but will not work straight out of the box for kubeadm

kubernetes/config/storage.yaml

@@ -75,6 +75,43 @@ data:
         getAuthorizer: { allow: {} },
         putAuthorizer: { allow: {} },
       },
+      fileSystemAccessCache: {
+        backend: {
+          'local': {
+            keyLocationMapOnBlockDevice: {
+              file: {
+                path: '/storage-fsac/key_location_map',
+                sizeBytes: 1024 * 1024,
+              },
+            },
+            keyLocationMapMaximumGetAttempts: 8,
+            keyLocationMapMaximumPutAttempts: 32,
+            oldBlocks: 8,
+            currentBlocks: 24,
+            newBlocks: 1,
+            blocksOnBlockDevice: {
+              source: {
+                file: {
+                  path: '/storage-fsac/blocks',
+                  sizeBytes: 20 * 1024 * 1024,
+                },
+              },
+              spareBlocks: 3,
+              dataIntegrityValidationCache: {
+                cacheSize: 10000,
+                cacheDuration: '14400s',
+                cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+              },
+            },
+            persistent: {
+              stateDirectoryPath: '/storage-fsac/persistent_state',
+              minimumEpochInterval: '300s',
+            },
+          },
+        },
+        getAuthorizer: { allow: {} },
+        putAuthorizer: { allow: {} },
+      },
     }
 kind: ConfigMap
 metadata:

kubernetes/config/worker-ubuntu22-04.yaml

@@ -4,24 +4,73 @@ data:
     local common = import 'common.libsonnet';
 
     {
-      blobstore: common.blobstore,
+      blobstore: {
+        actionCache: common.blobstore.actionCache,
+        contentAddressableStorage: {
+          readCaching: {
+            slow: common.blobstore.contentAddressableStorage,
+            fast: {
+              'local': {
+                keyLocationMapOnBlockDevice: {
+                  file: {
+                    path: '/storage-worker-cas/key_location_map',
+                    sizeBytes: 100 * 1024 * 1024,
+                  },
+                },
+                keyLocationMapMaximumGetAttempts: 8,
+                keyLocationMapMaximumPutAttempts: 32,
+                oldBlocks: 8,
+                currentBlocks: 24,
+                newBlocks: 1,
+                blocksOnBlockDevice: {
+                  source: {
+                    file: {
+                    path: '/storage-worker-cas/blocks',
+                    sizeBytes: 8 * 1024 * 1024 * 1024,
+                    },
+                  },
+                  spareBlocks: 3,
+                  dataIntegrityValidationCache: {
+                    cacheSize: 10000,
+                    cacheDuration: '14400s',
+                    cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+                  },
+                },
+                persistent: {
+                stateDirectoryPath: '/storage-worker-cas/persistent_state',
+                minimumEpochInterval: '300s',
+                },
+              },
+            },
+            replicator: { deduplicating: { 'local': {} } },
+          },
+        },
+      },
       browserUrl: common.browserUrl,
       maximumMessageSizeBytes: common.maximumMessageSizeBytes,
       scheduler: { address: 'scheduler:8983' },
       global: common.global + {
         setUmask: { umask: 0 },
       },
       buildDirectories: [{
-        native: {
-          buildDirectoryPath: '/worker/build',
-          cacheDirectoryPath: '/worker/cache',
-          maximumCacheFileCount: 10000,
-          maximumCacheSizeBytes: 1024 * 1024 * 1024,
-          cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+        virtual: {
+          maximumExecutionTimeoutCompensation: "3600s",
+          shuffleDirectoryListings: true,
+          mount: {
+            mountPath: '/worker/build',
+            fuse: {
+              directoryEntryValidity: "300s",
+              inodeAttributeValidity: "300s",
+              allowOther: true,
+              directMount: true,
+            },
+          },
         },
         runners: [{
           endpoint: { address: 'unix:///worker/runner' },
           concurrency: 8,
+          maximumFilePoolFileCount: 10000,
+          maximumFilePoolSizeBytes: 8*1024*1024*1024,
           platform: {
             properties: [
               { name: 'OSFamily', value: 'Linux' },
@@ -34,12 +83,64 @@ data:
           },
         }],
       }],
+      filePool: {
+        blockDevice: {
+          file: {
+            path: '/worker/filepool',
+            sizeBytes:8 * 8*1024*1024*1024, // concurrency * maximumFilePoolSizeBytes
+          },
+        },
+      },
       outputUploadConcurrency: 11,
       directoryCache: {
         maximumCount: 1000,
         maximumSizeBytes: 1000 * 1024,
         cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
       },
+      prefetching: {
+        fileSystemAccessCache:{
+          readCaching: {
+            slow: common.blobstore.contentAddressableStorage,
+            fast: {
+              'local': {
+                keyLocationMapOnBlockDevice: {
+                  file: {
+                    path: '/storage-worker-cas/key_location_map',
+                    sizeBytes: 100 * 1024 * 1024,
+                  },
+                },
+                keyLocationMapMaximumGetAttempts: 8,
+                keyLocationMapMaximumPutAttempts: 32,
+                oldBlocks: 8,
+                currentBlocks: 24,
+                newBlocks: 1,
+                blocksOnBlockDevice: {
+                  source: {
+                    file: {
+                    path: '/storage-worker-cas/blocks',
+                    sizeBytes: 8 * 1024 * 1024 * 1024,
+                    },
+                  },
+                  spareBlocks: 3,
+                  dataIntegrityValidationCache: {
+                    cacheSize: 10000,
+                    cacheDuration: '14400s',
+                    cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+                  },
+                },
+                persistent: {
+                stateDirectoryPath: '/storage-worker-cas/persistent_state',
+                minimumEpochInterval: '300s',
+                },
+              },
+            },
+            replicator: { 'local' : {} },
+          },
+        },
+        bloomFilterBitsPerPath: 14,
+        bloomFilterMaximumSizeBytes: 65536,
+        downloadConcurrency: 24,
+      },
     }
 kind: ConfigMap
 metadata:

kubernetes/storage.yaml

@@ -30,18 +30,22 @@ spec:
           name: cas
         - mountPath: /storage-ac
           name: ac
+        - mountPath: /storage-fsac
+          name: fsac
       initContainers:
       - name: volume-init
         image: busybox:1.31.1-uclibc
         command:
         - sh
         - -c
-        - mkdir -m 0700 -p /storage-cas/persistent_state /storage-ac/persistent_state
+        - mkdir -m 0700 -p /storage-cas/persistent_state /storage-ac/persistent_state /storage-fsac/persistent_state
         volumeMounts:
         - mountPath: /storage-cas
           name: cas
         - mountPath: /storage-ac
           name: ac
+        - mountPath: /storage-fsac
+          name: fsac
       volumes:
       - name: configs
         projected:
@@ -73,6 +77,14 @@ spec:
       resources:
         requests:
           storage: 1Gi
+  - metadata:
+      name: fsac
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 12Gi
 ---
 apiVersion: v1
 kind: Service

kubernetes/worker-ubuntu22-04.yaml

@@ -7,7 +7,7 @@ metadata:
     prometheus.io/port: "80"
     prometheus.io/scrape: "true"
 spec:
-  replicas: 8
+  replicas: 2
   selector:
     matchLabels:
       app: worker
@@ -28,7 +28,12 @@ spec:
           name: configs
           readOnly: true
         - mountPath: /worker
+          mountPropagation: Bidirectional
           name: worker
+        - mountPath: /storage-worker-cas
+          name: storage-worker-cas
+        securityContext:
+          privileged: true
         env:
         - name: NODE_NAME
           valueFrom:
@@ -42,14 +47,16 @@ spec:
         image: ghcr.io/catthehacker/ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448
         name: runner
         securityContext:
-          runAsUser: 65534
-          allowPrivilegeEscalation: false
+          privileged: true
         volumeMounts:
         - mountPath: /config/
           name: configs
           readOnly: true
         - mountPath: /worker
+          mountPropagation: HostToContainer
           name: worker
+        - mountPath: /storage-worker-cas
+          name: storage-worker-cas
         - mountPath: /bb
           name: bb-runner
           readOnly: true
@@ -64,10 +71,15 @@ spec:
         command:
         - sh
         - -c
-        - mkdir -pm 0777 /worker/build && mkdir -pm 0700 /worker/cache && chmod 0777 /worker
+        - mkdir -pm 0777 /worker/build && mkdir -pm 0700 /worker/cache && mkdir -pm 0777 /storage-worker-cas/persistent_state && mkdir -pm 0777 /worker-fuse && chmod 0777 /worker
         volumeMounts:
         - mountPath: /worker
+          mountPropagation: HostToContainer
           name: worker
+        - mountPath: /storage-worker-cas
+          name: storage-worker-cas
+        securityContext:
+          privileged: true
       volumes:
       - name: bb-runner
         emptyDir: {}
@@ -89,5 +101,9 @@ spec:
               items:
               - key: common.libsonnet
                 path: common.libsonnet
-      - emptyDir: {}
-        name: worker
+      - name: worker
+        hostPath:
+          path: /worker-fuse
+          type: Directory
+      - name: storage-worker-cas
+        emptyDir: {}