Fixing Chat Messages Overflow Issue (#84)

* Removed dependency on chat_flow property to track messages in conversation

chat_client/static/js/chat_utils.js

@@ -262,7 +262,7 @@ async function buildConversation(conversationData={}, skin = CONVERSATION_SKINS.
     const newConversationHTML = await buildConversationHTML(conversationData, skin);
     const conversationsBody = document.getElementById(conversationParentID);
     conversationsBody.insertAdjacentHTML('afterbegin', newConversationHTML);
-    initMessages(conversationData, skin);
+    await initMessages(conversationData, skin);
 
     const messageListContainer = getMessageListContainer(cid);
     const currentConversation = document.getElementById(cid);
@@ -363,18 +363,18 @@ async function buildConversation(conversationData={}, skin = CONVERSATION_SKINS.
 /**
  * Gets conversation data based on input string
  * @param input: input string text
- * @param firstMessageID: id of the the most recent message
+ * @param oldestMessageTS: creation timestamp of the oldest displayed message
  * @param skin: resolves by server for which data to return
  * @param maxResults: max number of messages to fetch
  * @param alertParent: parent of error alert (optional)
  * @returns {Promise<{}>} promise resolving conversation data returned
  */
-async function getConversationDataByInput(input="", skin=CONVERSATION_SKINS.BASE, firstMessageID=null, maxResults=20, alertParent=null){
+async function getConversationDataByInput(input="", skin=CONVERSATION_SKINS.BASE, oldestMessageTS=null, maxResults=20, alertParent=null){
     let conversationData = {};
-    if(input && typeof input === "string"){
-        let query_url = `chat_api/search/${input}?limit_chat_history=${maxResults}&skin=${skin}`;
-        if(firstMessageID){
-            query_url += `&first_message_id=${firstMessageID}`;
+    if(input){
+        let query_url = `chat_api/search/${input.toString()}?limit_chat_history=${maxResults}&skin=${skin}`;
+        if(oldestMessageTS){
+            query_url += `&creation_time_from=${oldestMessageTS}`;
         }
         await fetchServer(query_url)
             .then(response => {
@@ -443,7 +443,8 @@ async function addNewCID(cid, skin){
  * @param cid: conversation id to remove
  */
 async function removeConversation(cid){
-    return await getChatAlignmentTable().where({cid: cid}).delete();
+    return await Promise.all([DBGateway.getInstance(DB_TABLES.CHAT_ALIGNMENT).deleteItem(cid),
+                                     DBGateway.getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION).deleteItem(cid)]);
 }
 
 /**

chat_client/static/js/db.js

@@ -3,12 +3,14 @@ const DATABASES = {
 }
 const DB_TABLES = {
     CHAT_ALIGNMENT: 'chat_alignment',
-    MINIFY_SETTINGS: 'minify_settings'
+    MINIFY_SETTINGS: 'minify_settings',
+    CHAT_MESSAGES_PAGINATION: 'chat_messages_pagination'
 }
 const __db_instances = {}
 const __db_definitions = {
-    "chats": {
-        "chat_alignment": `cid, added_on, skin`
+    [DATABASES.CHATS]: {
+        [DB_TABLES.CHAT_ALIGNMENT]: `cid, added_on, skin`,
+        [DB_TABLES.CHAT_MESSAGES_PAGINATION]: `cid, oldest_created_on`
     }
 }
 
@@ -30,4 +32,46 @@ const getDb = (db, table) => {
         _instance = __db_instances[db];
     }
     return _instance[table];
-}
+}
+
+
+class DBGateway {
+    constructor(db, table) {
+        this.db = db;
+        this.table = table;
+
+        this._db_instance = getDb(this.db, this.table);
+        this._db_columns_definitions = __db_definitions[this.db][this.table]
+        this._db_key = this._db_columns_definitions.split(',')[0]
+    }
+
+    async getItem(key = "") {
+        return await this._db_instance.where( {[this._db_key]: key} ).first();
+    }
+
+    async listItems(orderBy="") {
+        let expression = this._db_instance;
+        if (orderBy !== ""){
+            expression = expression.orderBy(orderBy)
+        }
+        return await expression.toArray();
+    }
+
+    async putItem(data = {}){
+        return await this._db_instance.put(data, [data[this._db_key]])
+    }
+
+    updateItem(data = {}) {
+        const key = data[this._db_key]
+        delete data[this._db_key]
+        return this._db_instance.update(key, data);
+    }
+
+    async deleteItem(key = "") {
+        return await this._db_instance.where({[this._db_key]: key}).delete();
+    }
+
+    static getInstance(table){
+        return new DBGateway(DATABASES.CHATS, table);
+    }
+}

chat_client/static/js/message_utils.js

@@ -165,10 +165,10 @@ async function addOldMessages(cid, skin=CONVERSATION_SKINS.BASE) {
     if (messageContainer.children.length > 0) {
         for (let i = 0; i < messageContainer.children.length; i++) {
             const firstMessageItem = messageContainer.children[i];
-            const firstMessageID = getFirstMessageFromCID( firstMessageItem );
-            if (firstMessageID) {
+            const oldestMessageTS = await DBGateway.getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION).getItem(cid).then(res=> res?.oldest_created_on || null);
+            if (oldestMessageTS) {
                 const numMessages = await getCurrentSkin(cid) === CONVERSATION_SKINS.PROMPTS? 50: 20;
-                await getConversationDataByInput( cid, skin, firstMessageID, numMessages, null ).then( async conversationData => {
+                await getConversationDataByInput( cid, skin, oldestMessageTS, numMessages, null ).then( async conversationData => {
                     if (messageContainer) {
                         const userMessageList = getUserMessages( conversationData, null );
                         userMessageList.sort( (a, b) => {
@@ -183,7 +183,7 @@ async function addOldMessages(cid, skin=CONVERSATION_SKINS.BASE) {
                                 console.debug( `!!message_id=${message["message_id"]} is already displayed` )
                             }
                         }
-                        initMessages( conversationData, skin );
+                        await initMessages( conversationData, skin );
                     }
                 } ).then( _ => {
                     firstMessageItem.scrollIntoView( {behavior: "smooth"} );
@@ -293,7 +293,7 @@ function addProfileDisplay(cid, messageId, messageType='plain'){
 
 /**
  * Inits addProfileDisplay() on each message of provided conversation
- * @param conversationData: target conversation data
+ * @param conversationData - target conversation data
  */
 function initProfileDisplay(conversationData){
     getUserMessages(conversationData, null).forEach(message => {
@@ -302,9 +302,25 @@ function initProfileDisplay(conversationData){
 }
 
 
+/**
+ * Inits pagination based on the oldest message creation timestamp
+ * @param conversationData - target conversation data
+ */
+async function initPagination(conversationData) {
+    const userMessages = getUserMessages(conversationData, null);
+    if (userMessages.length > 0){
+        const oldestMessage = Math.min(...userMessages.map(msg => parseInt(msg.created_on)));
+        await DBGateway
+            .getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION)
+            .putItem({cid: conversationData['_id'],
+                           oldest_created_on: oldestMessage})
+    }
+}
+
+
 /**
  * Initializes messages based on provided conversation aata
- * @param conversationData: JS Object containing conversation data of type:
+ * @param conversationData - JS Object containing conversation data of type:
  * {
  *     '_id': 'id of conversation',
  *     'conversation_name': 'title of the conversation',
@@ -318,14 +334,15 @@ function initProfileDisplay(conversationData){
  *         'created_on': 'creation time of the message'
  *     }, ... (num of user messages returned)]
  * }
- * @param skin: target conversation skin to consider
+ * @param skin - target conversation skin to consider
  */
-function initMessages(conversationData, skin = CONVERSATION_SKINS.BASE){
+async function initMessages(conversationData, skin = CONVERSATION_SKINS.BASE){
     initProfileDisplay(conversationData);
     attachReplies(conversationData);
     addAttachments(conversationData);
     addCommunicationChannelTransformCallback(conversationData);
     initLoadOldMessages(conversationData, skin);
+    await initPagination(conversationData);
 }
 
 /**

chat_server/blueprints/chat.py

@@ -26,17 +26,18 @@
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import warnings
-from typing import Optional
 
 from time import time
-from fastapi import APIRouter, Request, Form
+from fastapi import APIRouter, Request, Form, Depends
 from fastapi.responses import JSONResponse
 
-from chat_server.constants.conversations import ConversationSkins
 from chat_server.server_utils.auth import login_required
 from chat_server.server_utils.conversation_utils import build_message_json
+from chat_server.server_utils.dependencies import CurrentUserDependency
+from chat_server.server_utils.models.chats import GetConversationModel
 from chat_server.services.popularity_counter import PopularityCounter
 from utils.common import generate_uuid
+from utils.database_utils.mongo_utils import MongoFilter, MongoLogicalOperators
 from utils.database_utils.mongo_utils.queries.mongo_queries import fetch_message_data
 from utils.database_utils.mongo_utils.queries.wrapper import MongoDocumentsAPI
 from utils.http_utils import respond
@@ -86,53 +87,51 @@ async def new_conversation(
         "created_on": int(time()),
     }
     MongoDocumentsAPI.CHATS.add_item(data=request_data_dict)
-    PopularityCounter.add_new_chat(cid=cid, name=conversation_name)
+    PopularityCounter.add_new_chat(cid=cid)
     return JSONResponse(content=request_data_dict)
 
 
 @router.get("/search/{search_str}")
-# @login_required
 async def get_matching_conversation(
-    request: Request,
-    search_str: str,
-    chat_history_from: int = 0,
-    first_message_id: Optional[str] = None,
-    limit_chat_history: int = 100,
-    skin: str = ConversationSkins.BASE,
+    current_user: CurrentUserDependency, model: GetConversationModel = Depends()
 ):
     """
     Gets conversation data matching search string
 
-    :param request: Starlette Request object
-    :param search_str: provided search string
-    :param chat_history_from: upper time bound for messages
-    :param first_message_id: id of the first message to start from
-    :param limit_chat_history: lower time bound for messages
-    :param skin: conversation skin type from ConversationSkins
+    :param current_user: current user data
+    :param model: request data model described in GetConversationModel
 
     :returns conversation data if found, 401 error code otherwise
     """
     conversation_data = MongoDocumentsAPI.CHATS.get_conversation_data(
-        search_str=search_str
+        search_str=model.search_str, requested_user_id=current_user.user_id
     )
 
     if not conversation_data:
-        return respond(f'No conversation matching = "{search_str}"', 404)
+        return respond(f'No conversation matching = "{model.search_str}"', 404)
+
+    if model.creation_time_from:
+        query_filter = MongoFilter(
+            key="created_on",
+            logical_operator=MongoLogicalOperators.LT,
+            value=int(model.creation_time_from),
+        )
+    else:
+        query_filter = None
 
     message_data = (
         fetch_message_data(
-            skin=skin,
+            skin=model.skin,
             conversation_data=conversation_data,
-            start_idx=chat_history_from,
-            limit=limit_chat_history,
-            start_message_id=first_message_id,
+            limit=model.limit_chat_history,
+            creation_time_filter=query_filter,
         )
         or []
     )
-    conversation_data["chat_flow"] = []
-    for i in range(len(message_data)):
-        message_record = build_message_json(raw_message=message_data[i], skin=skin)
-        conversation_data["chat_flow"].append(message_record)
+    conversation_data["chat_flow"] = [
+        build_message_json(raw_message=message_data[i], skin=model.skin)
+        for i in range(len(message_data))
+    ]
 
     return conversation_data
 

chat_server/server_utils/models/chats.py

@@ -0,0 +1,43 @@
+# NEON AI (TM) SOFTWARE, Software Development Kit & Application Framework
+# All trademark and other rights reserved by their respective owners
+# Copyright 2008-2022 Neongecko.com Inc.
+# Contributors: Daniel McKnight, Guy Daniels, Elon Gasper, Richard Leeds,
+# Regina Bloomstine, Casimiro Ferreira, Andrii Pernatii, Kirill Hrymailo
+# BSD-3 License
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS;  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from time import time
+
+from fastapi import Query, Path
+from pydantic import BaseModel, Field
+
+from chat_server.constants.conversations import ConversationSkins
+
+
+class GetConversationModel(BaseModel):
+    search_str: str = Field(Path(), examples=["1"])
+    limit_chat_history: int = (Field(Query(default=100), examples=[100]),)
+    creation_time_from: str | None = Field(Query(default=None), examples=[int(time())])
+    skin: str = Field(
+        Query(default=ConversationSkins.BASE), examples=[ConversationSkins.BASE]
+    )