feat(powersync-db-collection): support numeric collection keys

.changeset/powersync-numeric-keys.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/powersync-db-collection': patch
+---
+
+Support numeric collection keys. A `deserializationSchema` may convert `id` to a number, for a backend with integer primary keys; the collection key is then the deserialized number. Sync, on-demand reloads, eviction, and local updates and deletes all use that key.

docs/collections/powersync-collection.md

@@ -399,6 +399,33 @@ const task = tasksCollection.get("task-1")
 task.due_date.getTime() // OK - TypeScript knows this is a Date
 ```
 
+### Integer Primary Keys
+
+SQLite stores every PowerSync `id` as text. If your backend uses integer primary keys, convert `id` to a number in `deserializationSchema`. The collection key is then a number, so integer foreign keys match it, for example in `eq(part.id, line.partId)`.
+
+```typescript
+const productSchema = z.object({
+  id: z.coerce.number(), // SQLite TEXT '5' to the key 5
+  name: z.string().nullable(),
+})
+
+const products = createCollection(
+  powerSyncCollectionOptions({
+    database: db,
+    table: APP_SCHEMA.props.products,
+    schema: productSchema,
+    deserializationSchema: productSchema,
+    onDeserializationError: (error) => {
+      // Present fatal error
+    },
+  })
+)
+
+products.get(5)
+```
+
+Updates and deletes write the key back as text. The PowerSync service converts integer `id` columns to text the same way when it syncs them.
+
 ### Optimistic Updates
 
 Updates to the collection are applied optimistically to the local state first, then synchronized with PowerSync and the backend. If an error occurs during sync, the changes are automatically rolled back.

packages/powersync-db-collection/src/PowerSyncTransactor.ts

@@ -2,7 +2,7 @@ import { LogLevels, sanitizeSQL } from '@powersync/common'
 import { LoadSubsetOperationAbortedError, isFailedStatus } from '@tanstack/db'
 import DebugModule from 'debug'
 import { PendingOperationStore } from './PendingOperationStore'
-import { asPowerSyncRecord, mapOperationToPowerSync } from './helpers'
+import { mapOperationToPowerSync } from './helpers'
 import type { CommonPowerSyncDatabase, LockContext } from '@powersync/common'
 import type { PendingMutation, Transaction } from '@tanstack/db'
 import type { PendingOperation } from './PendingOperationStore'
@@ -247,7 +247,7 @@ export class PowerSyncTransactor {
         SET ${keys.map((key) => `${key} = ?`).join(`, `)}
         WHERE id = ?
         `,
-          [...queryParameters, asPowerSyncRecord(mutation.original).id],
+          [...queryParameters, String(mutation.key)],
         )
 
         return
@@ -278,14 +278,14 @@ export class PowerSyncTransactor {
             `
             UPDATE ${tableName} SET _deleted = TRUE, _metadata = ? WHERE id = ?
             `,
-            [metadataValue, asPowerSyncRecord(mutation.original).id],
+            [metadataValue, String(mutation.key)],
           )
         } else {
           await context.execute(
             `
             DELETE FROM ${tableName} WHERE id = ?
             `,
-            [asPowerSyncRecord(mutation.original).id],
+            [String(mutation.key)],
           )
         }
       },

packages/powersync-db-collection/src/definitions.ts

@@ -26,6 +26,16 @@ export type InferPowerSyncOutputType<
   ? ExtractedTable<TTable>
   : InferSchemaOutput<TSchema>
 
+/**
+ * The collection key: the type of the output row's `id`. It is a string
+ * unless a schema converts `id`, for example to a number.
+ */
+export type InferPowerSyncKey<TOutput> = TOutput extends { id: infer TId }
+  ? TId extends number
+    ? number
+    : string
+  : string
+
 /**
  * A mapping type for custom serialization of object properties to SQLite-compatible values.
  *
@@ -208,7 +218,7 @@ export type BasePowerSyncCollectionConfig<
 > = Omit<
   BaseCollectionConfig<
     InferPowerSyncOutputType<TTable, TSchema>,
-    string,
+    InferPowerSyncKey<InferPowerSyncOutputType<TTable, TSchema>>,
     TSchema
   >,
   `onInsert` | `onUpdate` | `onDelete` | `getKey` | `syncMode`
@@ -306,7 +316,7 @@ export type EnhancedPowerSyncCollectionConfig<
   TSchema extends StandardSchemaV1 = never,
 > = CollectionConfig<
   OutputType,
-  string,
+  InferPowerSyncKey<OutputType>,
   TSchema,
   PowerSyncCollectionUtils<TTable>
 > & {

packages/powersync-db-collection/src/helpers.ts

@@ -2,10 +2,12 @@ import { DiffTriggerOperation } from '@powersync/common'
 import type { ExtractColumnValueType, Table } from '@powersync/common'
 
 /**
- * All PowerSync table records include a UUID `id` column.
+ * All PowerSync table records include an `id` column. SQLite stores it as
+ * text; a deserialization schema may convert it to a number, as for a
+ * backend with integer primary keys.
  */
 export type PowerSyncRecord = {
-  id: string
+  id: string | number
   [key: string]: unknown
 }
 
@@ -64,14 +66,7 @@ export type OptionalExtractedTable<TTable extends Table> = OptionalIfUndefined<{
  */
 export type AnyTableColumnType<TTable extends Table> = {
   [K in keyof ExtractedTableColumns<TTable>]: unknown
-} & { id: string }
-
-export function asPowerSyncRecord(record: any): PowerSyncRecord {
-  if (typeof record.id !== `string`) {
-    throw new Error(`Record must have a string id field`)
-  }
-  return record as PowerSyncRecord
-}
+} & { id: string | number }
 
 /**
  * Maps {@link DiffTriggerOperation} to TanstackDB operations

packages/powersync-db-collection/src/powersync.ts

@@ -4,7 +4,7 @@ import { compileSQLite } from './sqlite-compiler'
 import { PendingOperationStore } from './PendingOperationStore'
 import { PowerSyncTransactor } from './PowerSyncTransactor'
 import { DEFAULT_BATCH_SIZE } from './definitions'
-import { asPowerSyncRecord, mapOperation } from './helpers'
+import { mapOperation } from './helpers'
 import { convertTableToSchema } from './schema'
 import { serializeForSQLite } from './serialization'
 import type {
@@ -25,6 +25,7 @@ import type {
   ConfigWithSQLiteInputType,
   ConfigWithSQLiteTypes,
   EnhancedPowerSyncCollectionConfig,
+  InferPowerSyncKey,
   InferPowerSyncOutputType,
   PowerSyncCollectionConfig,
   PowerSyncCollectionUtils,
@@ -267,6 +268,7 @@ function createPowerSyncCollectionConfig<
 
   // The collection output type
   type OutputType = InferPowerSyncOutputType<TTable, TSchema>
+  type PowerSyncKey = InferPowerSyncKey<OutputType>
 
   const { viewName, trackMetadata: metadataIsTracked } = table
 
@@ -323,7 +325,7 @@ function createPowerSyncCollectionConfig<
    * Notice that this describes the Sync between the local SQLite table
    * and the in-memory tanstack-db collection.
    */
-  const sync: SyncConfig<OutputType, string> = {
+  const sync: SyncConfig<OutputType, PowerSyncKey> = {
     sync: (params) => {
       const {
         begin,
@@ -428,7 +430,7 @@ function createPowerSyncCollectionConfig<
           // holds.
           snapshot?: boolean
           when: Record<DiffTriggerOperation, string>
-          writeType: (rowId: string) => OperationType
+          writeType: (key: PowerSyncKey) => OperationType
           batchQuery: (
             lockContext: LockContext,
             batchSize: number,
@@ -450,16 +452,22 @@ function createPowerSyncCollectionConfig<
               // A load of every row is one snapshot transaction: during a
               // reload it replaces the kept rows, so a row the table no
               // longer has goes. A subset load commits each batch. A
-              // transaction begins only once its rows are read, so a batch
-              // that fails leaves none open.
+              // transaction begins only once its rows are read and
+              // deserialized, so a batch that fails leaves none open.
               const batches: Array<Array<TableType>> = []
               const apply = () => {
-                begin({ snapshot })
-                for (const row of batches.splice(0).flat()) {
-                  write({
-                    type: writeType(row.id),
-                    value: deserializeSyncRow(row),
+                // The key comes from the deserialized row: a schema may
+                // convert SQLite's text id to a number.
+                const entries = batches
+                  .splice(0)
+                  .flat()
+                  .map((row) => {
+                    const value = deserializeSyncRow(row)
+                    return [getKey(value), value] as const
                   })
+                begin({ snapshot })
+                for (const [key, value] of entries) {
+                  write({ type: writeType(key), value })
                 }
                 appliedReceipts.push(commit())
               }
@@ -633,7 +641,7 @@ function createPowerSyncCollectionConfig<
                 [DiffTriggerOperation.UPDATE]: `TRUE`,
                 [DiffTriggerOperation.DELETE]: `TRUE`,
               },
-              writeType: (_rowId: string) => `insert`,
+              writeType: () => `insert`,
               batchQuery: (
                 lockContext: LockContext,
                 batchSize: number,
@@ -814,22 +822,19 @@ function createPowerSyncCollectionConfig<
                       [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
                       [DiffTriggerOperation.DELETE]: oldDataWhenClause,
                     },
-                    writeType: (rowId: string) =>
-                      collection.has(rowId) ? `update` : `insert`,
+                    writeType: (key) => {
+                      missing?.delete(key)
+                      return collection.has(key) ? `update` : `insert`
+                    },
                     batchQuery: (
                       lockContext: LockContext,
                       batchSize: number,
                       cursor: number,
                     ) =>
-                      lockContext
-                        .getAll<TableType>(
-                          `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
-                          [batchSize, cursor],
-                        )
-                        .then((rows) => {
-                          for (const row of rows) missing?.delete(row.id)
-                          return rows
-                        }),
+                      lockContext.getAll<TableType>(
+                        `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
+                        [batchSize, cursor],
+                      ),
                   },
                   appliedReceipts,
                 )
@@ -1013,14 +1018,14 @@ function createPowerSyncCollectionConfig<
         ): Promise<void> => {
           const compiledDeparting = compileSQLite({ where: options.where })
           const departingWhereSQL = toInlinedWhereClause(compiledDeparting)
-          let rowsToEvict: Array<{ id: string }>
+          let rowsToEvict: Array<TableType>
           for (;;) {
             if (stopped) return
             const revision = trackingRevision
             const active = activeWhereExpressions()
             let evictionSQL: string
             if (active.length === 0) {
-              evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}`
+              evictionSQL = `SELECT * FROM ${viewName} WHERE ${departingWhereSQL}`
             } else {
               const combinedRemaining =
                 active.length === 1
@@ -1030,19 +1035,22 @@ function createPowerSyncCollectionConfig<
                 where: combinedRemaining,
               })
               const remainingWhereSQL = toInlinedWhereClause(compiledRemaining)
-              evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
+              evictionSQL = `SELECT * FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
             }
 
-            rowsToEvict = await database.getAll<{ id: string }>(evictionSQL)
+            rowsToEvict = await database.getAll<TableType>(evictionSQL)
             // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cleanup can run during the query
             if (stopped) return
             if (trackingRevision === revision) break
           }
           if (rowsToEvict.length > 0) {
+            // Derive every key before the transaction opens: a row that fails
+            // deserialization must not leave it open.
+            const keys = rowsToEvict.map((row) =>
+              getKey(deserializeSyncRow(row)),
+            )
             begin()
-            for (const { id } of rowsToEvict) {
-              write({ type: `delete`, key: id })
-            }
+            for (const key of keys) write({ type: `delete`, key })
             void commit()
           }
           await rebuildTracking()
@@ -1182,7 +1190,13 @@ function createPowerSyncCollectionConfig<
     getSyncMetadata: undefined,
   }
 
-  const getKey = (record: OutputType) => asPowerSyncRecord(record).id
+  const getKey = (record: OutputType): PowerSyncKey => {
+    const { id } = record as { id?: unknown }
+    if (typeof id !== `string` && typeof id !== `number`) {
+      throw new Error(`Record must have a string or number id field`)
+    }
+    return id as PowerSyncKey
+  }
 
   // Keep adapter hooks and serialization options out of the core config.
   const {