refactor: route filesystem archives through host

eslint.config.js

@@ -60,6 +60,24 @@ const config = defineConfig([
       'no-shadow': 'off',
     },
   },
+  {
+    files: ['packages/agent-eval/src/**/*.ts'],
+    ignores: ['packages/agent-eval/src/host.ts'],
+    rules: {
+      'no-restricted-imports': [
+        'error',
+        {
+          paths: [
+            {
+              name: 'tar-fs',
+              allowTypeImports: true,
+              message: 'Use host.archive so archive operations use the host filesystem.',
+            },
+          ],
+        },
+      ],
+    },
+  },
 ])
 
 export default config

packages/agent-eval/src/host.test.ts

@@ -2,6 +2,7 @@ import os from 'node:os'
 import path from 'node:path'
 import fs from 'node:fs/promises'
 import {pathToFileURL} from 'node:url'
+import {pipeline} from 'node:stream/promises'
 import {expect, test} from 'vitest'
 import {DefaultHost, SystemHost, VirtualHost} from './host'
 
@@ -24,6 +25,11 @@ test('SystemHost provides access to the system filesystem and module loader', as
     await expect(host.loadModule<{value: string}>(pathToFileURL(moduleFilepath).href)).resolves.toMatchObject({
       value: 'loaded',
     })
+
+    const destination = path.join(directory, 'extracted')
+    await pipeline(host.archive.pack(directory, {entries: ['example.txt']}), host.archive.extract(destination))
+
+    await expect(fs.readFile(path.join(destination, 'example.txt'), 'utf8')).resolves.toBe('example')
   } finally {
     await fs.rm(directory, {recursive: true, force: true})
   }
@@ -54,3 +60,27 @@ test('VirtualHost creates an empty writable filesystem by default', async () =>
 
   expect(host.vol.readFileSync('/example.txt', 'utf8')).toBe('example')
 })
+
+test('VirtualHost archives read and write only their own filesystem', async () => {
+  const source = VirtualHost.create({
+    '/source/nested/example.txt': 'Source contents',
+  })
+  const destination = VirtualHost.create({
+    '/source/nested/example.txt': 'Destination contents',
+  })
+  const binary = Buffer.from([0, 255, 128, 1])
+  await source.fs.promises.writeFile('/source/binary', binary)
+  await source.fs.promises.symlink('nested/example.txt', '/source/link')
+
+  await pipeline(source.archive.pack('/source'), destination.archive.extract('/extracted'))
+
+  await expect(destination.fs.promises.readFile('/extracted/nested/example.txt', 'utf8')).resolves.toBe(
+    'Source contents',
+  )
+  await expect(destination.fs.promises.readFile('/extracted/binary')).resolves.toEqual(binary)
+  await expect(destination.fs.promises.readlink('/extracted/link')).resolves.toBe('nested/example.txt')
+  await expect(destination.fs.promises.readFile('/source/nested/example.txt', 'utf8')).resolves.toBe(
+    'Destination contents',
+  )
+  expect(source.fs.existsSync('/extracted')).toBe(false)
+})

packages/agent-eval/src/host.ts

@@ -1,12 +1,14 @@
 import fs from 'node:fs'
 import {pathToFileURL} from 'node:url'
 import {memfs, Volume, type NestedDirectoryJSON} from 'memfs'
+import tarFs from 'tar-fs'
 import {SystemSandbox, VirtualSandbox, type Sandbox, type SandboxCreateOptions} from './sandbox'
 
 type FileSystem = typeof import('node:fs')
 
 interface Host {
   fs: FileSystem
+  archive: Pick<typeof tarFs, 'pack' | 'extract'>
   loadModule<T = unknown>(filepath: string): Promise<T>
   createSandbox: (options?: SandboxCreateOptions) => Promise<Sandbox>
 }
@@ -17,9 +19,11 @@ class SystemHost implements Host {
   }
 
   fs: FileSystem
+  archive: Host['archive']
 
   constructor() {
     this.fs = fs
+    this.archive = createArchive(this.fs)
   }
 
   loadModule<T = unknown>(filepath: string): Promise<T> {
@@ -40,6 +44,7 @@ class VirtualHost implements Host {
   }
 
   fs: FileSystem
+  archive: Host['archive']
   vol: Volume
 
   constructor(json?: NestedDirectoryJSON) {
@@ -48,6 +53,7 @@ class VirtualHost implements Host {
     // @ts-expect-error - not every constant is exposed but memfs should match
     // for our test cases
     this.fs = virtualFs
+    this.archive = createArchive(this.fs)
   }
 
   async loadModule<T = unknown>(filepath: string): Promise<T> {
@@ -65,6 +71,25 @@ class VirtualHost implements Host {
   }
 }
 
+function createArchive(fs: FileSystem): Host['archive'] {
+  return {
+    pack(directory, options) {
+      const packOptions = {
+        ...options,
+        fs,
+      }
+      return tarFs.pack(directory, packOptions)
+    },
+    extract(directory, options) {
+      const extractOptions = {
+        ...options,
+        fs,
+      }
+      return tarFs.extract(directory, extractOptions)
+    },
+  }
+}
+
 const DefaultHost = new SystemHost()
 
 export {SystemHost, VirtualHost, DefaultHost}

packages/agent-eval/src/sandbox/system.test.ts

@@ -1,8 +1,9 @@
 import path from 'node:path'
 import {PassThrough, Readable} from 'node:stream'
+import {pipeline} from 'node:stream/promises'
 import Docker from 'dockerode'
 import {afterEach, expect, test, vi} from 'vitest'
-import {createContainer} from '../docker'
+import * as docker from '../docker'
 import {VirtualHost} from '../host'
 import {SystemSandbox} from './system'
 import {VirtualSandbox} from './virtual'
@@ -17,14 +18,69 @@ async function createSandbox(host = VirtualHost.create()) {
   vi.spyOn(Docker.prototype, 'createContainer').mockResolvedValue(container)
   vi.spyOn(container, 'start').mockResolvedValue(undefined)
   vi.spyOn(container, 'remove').mockResolvedValue(undefined)
-  const running = await createContainer({image: {tagName: 'test'}})
+  const running = await docker.createContainer({image: {tagName: 'test'}})
 
   return {
     container,
     sandbox: new SystemSandbox(host, running),
   }
 }
 
+test('copy and download use host archives while preserving renaming and exclusions', async () => {
+  const sourcePath = path.resolve('archive-source')
+  const host = VirtualHost.create({
+    [sourcePath]: {
+      'example.txt': 'Example',
+      nested: {'included.txt': 'Included', 'ignored.txt': 'Ignored on download'},
+      excluded: {'secret.txt': 'Excluded on copy'},
+    },
+  })
+  const containerHost = VirtualHost.create()
+  const {container, sandbox} = await createSandbox(host)
+  await using resource = sandbox
+  vi.spyOn(docker, 'exec').mockResolvedValue({
+    stdout: '',
+    stderr: '',
+    exitCode: 0,
+  })
+  vi.spyOn(container, 'putArchive').mockImplementation(async stream => {
+    if (typeof stream === 'string' || Buffer.isBuffer(stream)) {
+      throw new TypeError('Expected an archive stream')
+    }
+    await pipeline(
+      stream,
+      containerHost.archive.extract('/container', {
+        map(header) {
+          return {...header, uid: process.getuid?.() ?? header.uid, gid: process.getgid?.() ?? header.gid}
+        },
+      }),
+    )
+    return new PassThrough()
+  })
+  vi.spyOn(container, 'getArchive').mockImplementation(async () => {
+    return containerHost.archive.pack('/container', {
+      entries: ['renamed'],
+      map(header) {
+        return {...header, uid: process.getuid?.() ?? header.uid, gid: process.getgid?.() ?? header.gid}
+      },
+    })
+  })
+
+  await resource.copy(sourcePath, 'renamed', {exclude: ['excluded']})
+  await resource.download('renamed', '/downloaded', {
+    ignore(filepath) {
+      return path.basename(filepath) === 'ignored.txt'
+    },
+  })
+
+  await expect(containerHost.fs.promises.readFile('/container/renamed/example.txt', 'utf8')).resolves.toBe('Example')
+  expect(containerHost.fs.existsSync('/container/renamed/excluded')).toBe(false)
+  await expect(host.fs.promises.readFile('/downloaded/example.txt', 'utf8')).resolves.toBe('Example')
+  await expect(host.fs.promises.readFile('/downloaded/nested/included.txt', 'utf8')).resolves.toBe('Included')
+  expect(host.fs.existsSync('/downloaded/nested/ignored.txt')).toBe(false)
+  expect(host.fs.existsSync('/downloaded/excluded')).toBe(false)
+})
+
 test.each([
   {label: 'empty include', include: [], expected: []},
   {label: 'file paths', include: ['README.md'], expected: ['README.md']},

packages/agent-eval/src/sandbox/system.ts

@@ -3,7 +3,6 @@ import path from 'node:path'
 import {Readable, Writable} from 'node:stream'
 import {pipeline} from 'node:stream/promises'
 import {glob} from 'glob'
-import tarFs from 'tar-fs'
 import type {Headers} from 'tar-fs'
 import tarStream from 'tar-stream'
 import * as z from 'zod/mini'
@@ -98,7 +97,7 @@ class SystemSandbox implements Sandbox {
     const sourceDirectory = path.dirname(source)
     const sourceName = path.basename(source)
     const excludedPaths = new Set(options.exclude?.map(filepath => normalizeExcludedPath(filepath, source)))
-    const archive = tarFs.pack(sourceDirectory, {
+    const archive = this.#host.archive.pack(sourceDirectory, {
       entries: [sourceName],
       ignore(name) {
         const absolutePath = path.isAbsolute(name) ? name : path.resolve(sourceDirectory, name)
@@ -129,7 +128,7 @@ class SystemSandbox implements Sandbox {
 
     await pipeline(
       archive,
-      tarFs.extract(hostDestinationPath, {
+      this.#host.archive.extract(hostDestinationPath, {
         readable: true,
         writable: true,
         map(header) {

packages/agent-eval/src/scenario/image.test.ts

@@ -1,5 +1,6 @@
 import path from 'node:path'
-import tarFs from 'tar-fs'
+import {Readable} from 'node:stream'
+import {pipeline} from 'node:stream/promises'
 import {afterEach, expect, test, vi} from 'vitest'
 import {buildImage} from '../docker'
 import {VirtualHost} from '../host'
@@ -11,6 +12,7 @@ vi.mock('../docker', async importOriginal => {
   return {
     ...original,
     buildImage: vi.fn<typeof original.buildImage>(async (_context, options) => {
+      await Readable.from(_context).toArray()
       return {tagName: options.t}
     }),
   }
@@ -21,16 +23,24 @@ afterEach(() => {
   vi.clearAllMocks()
 })
 
-test('the default scenario Dockerfile uses the shared sandbox image', async () => {
-  const context = tarFs.pack('.', {entries: [], finalize: false})
-  const entry = vi.spyOn(context, 'entry')
-  vi.spyOn(tarFs, 'pack').mockImplementation((_directory, options) => {
-    options?.finish?.(context)
-    return context
+test('scenario image archives use host files and exclude ignored paths', async () => {
+  const host = VirtualHost.create({
+    '/scenarios/example': {
+      'package.json': '{"name":"example"}',
+      'scenario.config.ts': 'Scenario configuration',
+      '.dockerignore': 'Original ignore rules',
+      Dockerfile: 'Original Dockerfile',
+      src: {'index.ts': 'export const value = 1'},
+      node_modules: {'dependency.js': 'Dependency'},
+    },
+  })
+  vi.mocked(buildImage).mockImplementationOnce(async (context, options) => {
+    await pipeline(context, host.archive.extract('/image-context'))
+    return {tagName: options.t}
   })
 
   await buildScenarioImage({
-    host: VirtualHost.create(),
+    host,
     scenario: {
       id: 'example',
       directory: '/scenarios/example',
@@ -42,10 +52,20 @@ test('the default scenario Dockerfile uses the shared sandbox image', async () =
     },
   })
 
-  expect(entry).toHaveBeenCalledWith(
-    expect.objectContaining({name: 'Dockerfile'}),
-    expect.stringContaining(`FROM ${DEFAULT_DOCKER_IMAGE}\n`),
+  await expect(host.fs.promises.readFile('/image-context/Dockerfile', 'utf8')).resolves.toContain(
+    `FROM ${DEFAULT_DOCKER_IMAGE}\n`,
   )
+  await expect(host.fs.promises.readFile('/image-context/.dockerignore', 'utf8')).resolves.toBe(
+    'Dockerfile\n.dockerignore\n',
+  )
+  await expect(host.fs.promises.readFile('/image-context/package.json', 'utf8')).resolves.toBe('{"name":"example"}')
+  await expect(host.fs.promises.readFile('/image-context/src/index.ts', 'utf8')).resolves.toBe('export const value = 1')
+  expect((await host.fs.promises.readdir('/image-context')).sort()).toEqual([
+    '.dockerignore',
+    'Dockerfile',
+    'package.json',
+    'src',
+  ])
 })
 
 test.each(['Default', 'Reference', 'Build'] as const)(
@@ -55,10 +75,6 @@ test.each(['Default', 'Reference', 'Build'] as const)(
       '/first/example/Dockerfile': 'FROM node:26-slim\nCOPY . .\n',
       '/second/example/Dockerfile': 'FROM node:26-slim\nCOPY . .\n',
     })
-    const createArchive = tarFs.pack
-    const pack = vi.spyOn(tarFs, 'pack').mockImplementation(() => {
-      return createArchive('.', {entries: []})
-    })
     const createScenario = (directory: string): Scenario => {
       return {
         id: 'example',
@@ -85,8 +101,6 @@ test.each(['Default', 'Reference', 'Build'] as const)(
     expect(first.tagName).toMatch(/^agent-eval\/scenarios\/example:[a-f0-9]{16}$/)
     expect(first.tagName).not.toBe(second.tagName)
     expect(repeated.tagName).toBe(first.tagName)
-    expect(pack).toHaveBeenCalledWith('/first/example', expect.any(Object))
-    expect(pack).toHaveBeenCalledWith('/second/example', expect.any(Object))
     expect(buildImage).toHaveBeenCalledTimes(3)
 
     if (type === 'Build') {

packages/agent-eval/src/scenario/scenario.ts

@@ -1,5 +1,4 @@
 import path from 'node:path'
-import tarFs from 'tar-fs'
 import * as z from 'zod/mini'
 import {CheckSchema, type Check} from '../check'
 import {JudgeSchema, type Judge} from '../judge'
@@ -161,7 +160,7 @@ async function buildScenarioImage({host = DefaultHost, scenario}: BuildScenarioI
         return ignoreFile.filepath
       }),
     )
-    const context = tarFs.pack(contextDirectory, {
+    const context = host.archive.pack(contextDirectory, {
       finalize: false,
       ignore(filepath) {
         return ignored.has(filepath)