wip(agents-api): Initial migrations for postgres

Signed-off-by: Diwank Singh Tomer <[email protected]>

agents-api/docker-compose.yml

@@ -111,21 +111,3 @@ services:
           path: uv.lock
         - action: rebuild
           path: Dockerfile.worker
-
-  cozo-migrate:
-    image: julepai/cozo-migrate:${TAG:-dev}
-    container_name: cozo-migrate
-    build:
-      context: .
-      dockerfile: Dockerfile.migration
-    restart: "no" # Make sure to double quote this
-    environment:
-      <<: *shared-environment
-
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./migrations
-          target: /app/migrations
-        - action: rebuild
-          path: Dockerfile.migration

blob-store/docker-compose.yml

@@ -12,7 +12,7 @@ services:
     environment:
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
-      - DEBUG=${DEBUG:-true}
+      - DEBUG=${DEBUG:-false}
 
     ports:
       - 9333:9333  # master port

memory-store/docker-compose.yml

@@ -1,46 +1,21 @@
-name: julep-memory-store
-
+name: pgai
 services:
-  memory-store:
-    image: julepai/memory-store:${TAG:-dev}
-    environment:
-      - COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN}
-      - COZO_PORT=${COZO_PORT:-9070}
-      - COZO_MNT_DIR=${MNT_DIR:-/data}
-      - COZO_BACKUP_DIR=${COZO_BACKUP_DIR:-/backup}
-    volumes:
-      - cozo_data:/data
-      - cozo_backup:/backup
-    build:
-      context: .
-    ports:
-      - "9070:9070"
-
-    develop:
-      watch:
-        - action: sync+restart
-          path: ./options
-          target: /data/cozo.db/OPTIONS-000007
-        - action: rebuild
-          path: Dockerfile
-
-    labels:
-      ofelia.enabled: "true"
-      ofelia.job-exec.backupcron.schedule: "@every 3h"
-      ofelia.job-exec.backupcron.environment: '["COZO_PORT=${COZO_PORT}", "COZO_AUTH_TOKEN=${COZO_AUTH_TOKEN}", "COZO_BACKUP_DIR=${COZO_BACKUP_DIR}"]'
-      ofelia.job-exec.backupcron.command: bash /app/backup.sh
-
-  memory-store-backup-cron:
-    image: mcuadros/ofelia:latest
-    restart: unless-stopped
-    depends_on:
-      - memory-store
-    command: daemon --docker -f label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
+ db:
+   image: timescale/timescaledb-ha:pg17
+   environment:
+     - POSTGRES_PASSWORD=${MEMORY_STORE_PASSWORD:-postgres}
+     - VOYAGE_API_KEY=${VOYAGE_API_KEY}
+   ports:
+     - "5432:5432"
+   volumes:
+     - memory_store_data:/home/postgres/pgdata/data
+ vectorizer-worker:
+   image: timescale/pgai-vectorizer-worker:v0.3.0
+   environment:
+     - PGAI_VECTORIZER_WORKER_DB_URL=postgres://postgres:${MEMORY_STORE_PASSWORD:-postgres}@db:5432/postgres
+     - VOYAGE_API_KEY=${VOYAGE_API_KEY}
+   command: [ "--poll-interval", "5s" ]
 
 volumes:
-  cozo_data:
-    external: true
-  cozo_backup:
+  memory_store_data:
     external: true

memory-store/migrations/00001_initial.sql

@@ -0,0 +1,25 @@
+-- init timescaledb
+CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit CASCADE;
+
+-- add timescale's pgai extension
+CREATE EXTENSION IF NOT EXISTS vector CASCADE;
+CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;
+CREATE EXTENSION IF NOT EXISTS ai CASCADE;
+
+-- add misc extensions (for indexing etc)
+CREATE EXTENSION IF NOT EXISTS btree_gin CASCADE;
+CREATE EXTENSION IF NOT EXISTS btree_gist CASCADE;
+CREATE EXTENSION IF NOT EXISTS citext CASCADE;
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp" CASCADE;
+
+-- Create function to update the updated_at timestamp
+CREATE OR REPLACE FUNCTION update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = CURRENT_TIMESTAMP;
+    RETURN NEW;
+END;
+$$ language 'plpgsql';
+
+COMMENT ON FUNCTION update_updated_at_column() IS 'Trigger function to automatically update updated_at timestamp';

memory-store/migrations/00002_developers.sql

@@ -0,0 +1,33 @@
+-- Create developers table
+CREATE TABLE developers (
+    developer_id UUID NOT NULL,
+    email TEXT NOT NULL CONSTRAINT ct_developers_email_format CHECK (email ~* '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'),
+    active BOOLEAN NOT NULL DEFAULT true,
+    tags TEXT[] DEFAULT ARRAY[]::TEXT[],
+    settings JSONB NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT pk_developers PRIMARY KEY (developer_id),
+    CONSTRAINT uq_developers_email UNIQUE (email)
+);
+
+-- Create sorted index on developer_id (optimized for UUID v7)
+CREATE INDEX idx_developers_id_sorted ON developers (developer_id DESC);
+
+-- Create index on email
+CREATE INDEX idx_developers_email ON developers (email);
+
+-- Create GIN index for tags array
+CREATE INDEX idx_developers_tags ON developers USING GIN (tags);
+
+-- Create partial index for active developers
+CREATE INDEX idx_developers_active ON developers (developer_id) WHERE active = true;
+
+-- Create trigger to automatically update updated_at
+CREATE TRIGGER trg_developers_updated_at
+    BEFORE UPDATE ON developers
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+-- Add comment to table
+COMMENT ON TABLE developers IS 'Stores developer information including their settings and tags';

memory-store/migrations/00003_users.sql

@@ -0,0 +1,34 @@
+-- Create users table
+CREATE TABLE users (
+    developer_id UUID NOT NULL,
+    user_id UUID NOT NULL,
+    name TEXT NOT NULL,
+    about TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    metadata JSONB NOT NULL DEFAULT '{}'::JSONB,
+    CONSTRAINT pk_users PRIMARY KEY (developer_id, user_id)
+);
+
+-- Create sorted index on user_id (optimized for UUID v7)
+CREATE INDEX users_id_sorted_idx ON users (user_id DESC);
+
+-- Create foreign key constraint and index on developer_id
+ALTER TABLE users 
+    ADD CONSTRAINT users_developer_id_fkey 
+    FOREIGN KEY (developer_id) 
+    REFERENCES developers(developer_id);
+
+CREATE INDEX users_developer_id_idx ON users (developer_id);
+
+-- Create a GIN index on the entire metadata column
+CREATE INDEX users_metadata_gin_idx ON users USING GIN (metadata);
+
+-- Create trigger to automatically update updated_at
+CREATE TRIGGER update_users_updated_at
+    BEFORE UPDATE ON users
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+-- Add comment to table
+COMMENT ON TABLE users IS 'Stores user information linked to developers';

memory-store/migrations/00004_agents.sql

@@ -0,0 +1,40 @@
+-- Create agents table
+CREATE TABLE agents (
+    developer_id UUID NOT NULL,
+    agent_id UUID NOT NULL,
+    canonical_name citext NOT NULL CONSTRAINT ct_agents_canonical_name_length CHECK (length(canonical_name) >= 1 AND length(canonical_name) <= 255),
+    name TEXT NOT NULL CONSTRAINT ct_agents_name_length CHECK (length(name) >= 1 AND length(name) <= 255),
+    about TEXT CONSTRAINT ct_agents_about_length CHECK (about IS NULL OR length(about) <= 1000),
+    instructions TEXT[] DEFAULT ARRAY[]::TEXT[],
+    model TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    metadata JSONB NOT NULL DEFAULT '{}'::JSONB,
+    default_settings JSONB NOT NULL DEFAULT '{}'::JSONB,
+    CONSTRAINT pk_agents PRIMARY KEY (developer_id, agent_id),
+    CONSTRAINT uq_agents_canonical_name_unique UNIQUE (developer_id, canonical_name), -- per developer
+    CONSTRAINT ct_agents_canonical_name_valid_identifier CHECK (canonical_name ~ '^[a-zA-Z][a-zA-Z0-9_]*$')
+);
+
+-- Create sorted index on agent_id (optimized for UUID v7)
+CREATE INDEX idx_agents_id_sorted ON agents (agent_id DESC);
+
+-- Create foreign key constraint and index on developer_id
+ALTER TABLE agents 
+    ADD CONSTRAINT fk_agents_developer
+    FOREIGN KEY (developer_id) 
+    REFERENCES developers(developer_id);
+
+CREATE INDEX idx_agents_developer ON agents (developer_id);
+
+-- Create a GIN index on the entire metadata column
+CREATE INDEX idx_agents_metadata ON agents USING GIN (metadata);
+
+-- Create trigger to automatically update updated_at
+CREATE TRIGGER trg_agents_updated_at
+    BEFORE UPDATE ON agents
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+-- Add comment to table
+COMMENT ON TABLE agents IS 'Stores AI agent configurations and metadata for developers';