Removed dependency on chat_flow property to track messages in convers…

…ation

chat_client/static/js/chat_utils.js

@@ -262,7 +262,7 @@ async function buildConversation(conversationData={}, skin = CONVERSATION_SKINS.
     const newConversationHTML = await buildConversationHTML(conversationData, skin);
     const conversationsBody = document.getElementById(conversationParentID);
     conversationsBody.insertAdjacentHTML('afterbegin', newConversationHTML);
-    initMessages(conversationData, skin);
+    await initMessages(conversationData, skin);
 
     const messageListContainer = getMessageListContainer(cid);
     const currentConversation = document.getElementById(cid);
@@ -363,18 +363,18 @@ async function buildConversation(conversationData={}, skin = CONVERSATION_SKINS.
 /**
  * Gets conversation data based on input string
  * @param input: input string text
- * @param firstMessageID: id of the the most recent message
+ * @param oldestMessageTS: creation timestamp of the oldest displayed message
  * @param skin: resolves by server for which data to return
  * @param maxResults: max number of messages to fetch
  * @param alertParent: parent of error alert (optional)
  * @returns {Promise<{}>} promise resolving conversation data returned
  */
-async function getConversationDataByInput(input="", skin=CONVERSATION_SKINS.BASE, firstMessageID=null, maxResults=20, alertParent=null){
+async function getConversationDataByInput(input="", skin=CONVERSATION_SKINS.BASE, oldestMessageTS=null, maxResults=20, alertParent=null){
     let conversationData = {};
-    if(input && typeof input === "string"){
-        let query_url = `chat_api/search/${input}?limit_chat_history=${maxResults}&skin=${skin}`;
-        if(firstMessageID){
-            query_url += `&first_message_id=${firstMessageID}`;
+    if(input){
+        let query_url = `chat_api/search/${input.toString()}?limit_chat_history=${maxResults}&skin=${skin}`;
+        if(oldestMessageTS){
+            query_url += `&creation_time_from=${oldestMessageTS}`;
         }
         await fetchServer(query_url)
             .then(response => {
@@ -443,7 +443,8 @@ async function addNewCID(cid, skin){
  * @param cid: conversation id to remove
  */
 async function removeConversation(cid){
-    return await getChatAlignmentTable().where({cid: cid}).delete();
+    return await Promise.all([DBGateway.getInstance(DB_TABLES.CHAT_ALIGNMENT).deleteItem(cid),
+                                     DBGateway.getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION).deleteItem(cid)]);
 }
 
 /**

chat_client/static/js/db.js

@@ -3,12 +3,14 @@ const DATABASES = {
 }
 const DB_TABLES = {
     CHAT_ALIGNMENT: 'chat_alignment',
-    MINIFY_SETTINGS: 'minify_settings'
+    MINIFY_SETTINGS: 'minify_settings',
+    CHAT_MESSAGES_PAGINATION: 'chat_messages_pagination'
 }
 const __db_instances = {}
 const __db_definitions = {
-    "chats": {
-        "chat_alignment": `cid, added_on, skin`
+    [DATABASES.CHATS]: {
+        [DB_TABLES.CHAT_ALIGNMENT]: `cid, added_on, skin`,
+        [DB_TABLES.CHAT_MESSAGES_PAGINATION]: `cid, oldest_created_on`
     }
 }
 
@@ -30,4 +32,46 @@ const getDb = (db, table) => {
         _instance = __db_instances[db];
     }
     return _instance[table];
-}
+}
+
+
+class DBGateway {
+    constructor(db, table) {
+        this.db = db;
+        this.table = table;
+
+        this._db_instance = getDb(this.db, this.table);
+        this._db_columns_definitions = __db_definitions[this.db][this.table]
+        this._db_key = this._db_columns_definitions.split(',')[0]
+    }
+
+    async getItem(key = "") {
+        return await this._db_instance.where( {[this._db_key]: key} ).first();
+    }
+
+    async listItems(orderBy="") {
+        let expression = this._db_instance;
+        if (orderBy !== ""){
+            expression = expression.orderBy(orderBy)
+        }
+        return await expression.toArray();
+    }
+
+    async putItem(data = {}){
+        return await this._db_instance.put(data, [data[this._db_key]])
+    }
+
+    updateItem(data = {}) {
+        const key = data[this._db_key]
+        delete data[this._db_key]
+        return this._db_instance.update(key, data);
+    }
+
+    async deleteItem(key = "") {
+        return await this._db_instance.where({[this._db_key]: key}).delete();
+    }
+
+    static getInstance(table){
+        return new DBGateway(DATABASES.CHATS, table);
+    }
+}

chat_client/static/js/message_utils.js

@@ -165,10 +165,10 @@ async function addOldMessages(cid, skin=CONVERSATION_SKINS.BASE) {
     if (messageContainer.children.length > 0) {
         for (let i = 0; i < messageContainer.children.length; i++) {
             const firstMessageItem = messageContainer.children[i];
-            const firstMessageID = getFirstMessageFromCID( firstMessageItem );
-            if (firstMessageID) {
+            const oldestMessageTS = await DBGateway.getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION).getItem(cid).then(res=> res?.oldest_created_on || null);
+            if (oldestMessageTS) {
                 const numMessages = await getCurrentSkin(cid) === CONVERSATION_SKINS.PROMPTS? 50: 20;
-                await getConversationDataByInput( cid, skin, firstMessageID, numMessages, null ).then( async conversationData => {
+                await getConversationDataByInput( cid, skin, oldestMessageTS, numMessages, null ).then( async conversationData => {
                     if (messageContainer) {
                         const userMessageList = getUserMessages( conversationData, null );
                         userMessageList.sort( (a, b) => {
@@ -183,7 +183,7 @@ async function addOldMessages(cid, skin=CONVERSATION_SKINS.BASE) {
                                 console.debug( `!!message_id=${message["message_id"]} is already displayed` )
                             }
                         }
-                        initMessages( conversationData, skin );
+                        await initMessages( conversationData, skin );
                     }
                 } ).then( _ => {
                     firstMessageItem.scrollIntoView( {behavior: "smooth"} );
@@ -293,7 +293,7 @@ function addProfileDisplay(cid, messageId, messageType='plain'){
 
 /**
  * Inits addProfileDisplay() on each message of provided conversation
- * @param conversationData: target conversation data
+ * @param conversationData - target conversation data
  */
 function initProfileDisplay(conversationData){
     getUserMessages(conversationData, null).forEach(message => {
@@ -302,9 +302,25 @@ function initProfileDisplay(conversationData){
 }
 
 
+/**
+ * Inits pagination based on the oldest message creation timestamp
+ * @param conversationData - target conversation data
+ */
+async function initPagination(conversationData) {
+    const userMessages = getUserMessages(conversationData, null);
+    if (userMessages.length > 0){
+        const oldestMessage = Math.min(...userMessages.map(msg => parseInt(msg.created_on)));
+        await DBGateway
+            .getInstance(DB_TABLES.CHAT_MESSAGES_PAGINATION)
+            .putItem({cid: conversationData['_id'],
+                           oldest_created_on: oldestMessage})
+    }
+}
+
+
 /**
  * Initializes messages based on provided conversation aata
- * @param conversationData: JS Object containing conversation data of type:
+ * @param conversationData - JS Object containing conversation data of type:
  * {
  *     '_id': 'id of conversation',
  *     'conversation_name': 'title of the conversation',
@@ -318,14 +334,15 @@ function initProfileDisplay(conversationData){
  *         'created_on': 'creation time of the message'
  *     }, ... (num of user messages returned)]
  * }
- * @param skin: target conversation skin to consider
+ * @param skin - target conversation skin to consider
  */
-function initMessages(conversationData, skin = CONVERSATION_SKINS.BASE){
+async function initMessages(conversationData, skin = CONVERSATION_SKINS.BASE){
     initProfileDisplay(conversationData);
     attachReplies(conversationData);
     addAttachments(conversationData);
     addCommunicationChannelTransformCallback(conversationData);
     initLoadOldMessages(conversationData, skin);
+    await initPagination(conversationData);
 }
 
 /**

chat_server/blueprints/chat.py

@@ -26,17 +26,18 @@
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import warnings
-from typing import Optional
 
 from time import time
-from fastapi import APIRouter, Request, Form
+from fastapi import APIRouter, Request, Form, Depends
 from fastapi.responses import JSONResponse
 
-from chat_server.constants.conversations import ConversationSkins
 from chat_server.server_utils.auth import login_required
 from chat_server.server_utils.conversation_utils import build_message_json
+from chat_server.server_utils.dependencies import CurrentUserDependency
+from chat_server.server_utils.models.chats import GetConversationModel
 from chat_server.services.popularity_counter import PopularityCounter
 from utils.common import generate_uuid
+from utils.database_utils.mongo_utils import MongoFilter, MongoLogicalOperators
 from utils.database_utils.mongo_utils.queries.mongo_queries import fetch_message_data
 from utils.database_utils.mongo_utils.queries.wrapper import MongoDocumentsAPI
 from utils.http_utils import respond
@@ -86,53 +87,51 @@ async def new_conversation(
         "created_on": int(time()),
     }
     MongoDocumentsAPI.CHATS.add_item(data=request_data_dict)
-    PopularityCounter.add_new_chat(cid=cid, name=conversation_name)
+    PopularityCounter.add_new_chat(cid=cid)
     return JSONResponse(content=request_data_dict)
 
 
 @router.get("/search/{search_str}")
-# @login_required
 async def get_matching_conversation(
-    request: Request,
-    search_str: str,
-    chat_history_from: int = 0,
-    first_message_id: Optional[str] = None,
-    limit_chat_history: int = 100,
-    skin: str = ConversationSkins.BASE,
+    current_user: CurrentUserDependency, model: GetConversationModel = Depends()
 ):
     """
     Gets conversation data matching search string
 
-    :param request: Starlette Request object
-    :param search_str: provided search string
-    :param chat_history_from: upper time bound for messages
-    :param first_message_id: id of the first message to start from
-    :param limit_chat_history: lower time bound for messages
-    :param skin: conversation skin type from ConversationSkins
+    :param current_user: current user data
+    :param model: request data model described in GetConversationModel
 
     :returns conversation data if found, 401 error code otherwise
     """
     conversation_data = MongoDocumentsAPI.CHATS.get_conversation_data(
-        search_str=search_str
+        search_str=model.search_str, requested_user_id=current_user.user_id
     )
 
     if not conversation_data:
-        return respond(f'No conversation matching = "{search_str}"', 404)
+        return respond(f'No conversation matching = "{model.search_str}"', 404)
+
+    if model.creation_time_from:
+        query_filter = MongoFilter(
+            key="created_on",
+            logical_operator=MongoLogicalOperators.LT,
+            value=int(model.creation_time_from),
+        )
+    else:
+        query_filter = None
 
     message_data = (
         fetch_message_data(
-            skin=skin,
+            skin=model.skin,
             conversation_data=conversation_data,
-            start_idx=chat_history_from,
-            limit=limit_chat_history,
-            start_message_id=first_message_id,
+            limit=model.limit_chat_history,
+            creation_time_filter=query_filter,
         )
         or []
     )
-    conversation_data["chat_flow"] = []
-    for i in range(len(message_data)):
-        message_record = build_message_json(raw_message=message_data[i], skin=skin)
-        conversation_data["chat_flow"].append(message_record)
+    conversation_data["chat_flow"] = [
+        build_message_json(raw_message=message_data[i], skin=model.skin)
+        for i in range(len(message_data))
+    ]
 
     return conversation_data
 

chat_server/server_utils/models/chats.py

@@ -0,0 +1,15 @@
+from time import time
+
+from fastapi import Query, Path
+from pydantic import BaseModel, Field
+
+from chat_server.constants.conversations import ConversationSkins
+
+
+class GetConversationModel(BaseModel):
+    search_str: str = Field(Path(), examples=["1"])
+    limit_chat_history: int = (Field(Query(default=100), examples=[100]),)
+    creation_time_from: str | None = Field(Query(default=None), examples=[int(time())])
+    skin: str = Field(
+        Query(default=ConversationSkins.BASE), examples=[ConversationSkins.BASE]
+    )

chat_server/services/popularity_counter.py

@@ -25,6 +25,7 @@
 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE,  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from collections import Counter
 from dataclasses import dataclass
 from time import time
 from typing import List
@@ -63,8 +64,9 @@ def get_data(cls):
         return cls.__DATA
 
     @classmethod
-    def add_new_chat(cls, cid, name, popularity: int = 0):
+    def add_new_chat(cls, cid, popularity: int = 0):
         """Adds new chat to the tracked chat popularity records"""
+        name = MongoDocumentsAPI.CHATS.get_item(item_id=cid).get("conversation_name")
         cls.__DATA.append(
             ChatPopularityRecord(cid=cid, name=name, popularity=popularity)
         )
@@ -78,29 +80,47 @@ def init_data(cls, actuality_days: int = 7):
         :param actuality_days: number of days for message to affect the chat popularity
         """
         curr_time = int(time())
-        chats = MongoDocumentsAPI.CHATS.list_items(include_private=False)
+        oldest_timestamp = curr_time - 3600 * 24 * actuality_days
+        chats = MongoDocumentsAPI.CHATS.list_items(
+            filters=[
+                MongoFilter(
+                    key="last_shout_ts",
+                    logical_operator=MongoLogicalOperators.GTE,
+                    value=oldest_timestamp,
+                )
+            ],
+            include_private=False,
+            result_as_cursor=False,
+        )
         relevant_shouts = MongoDocumentsAPI.SHOUTS.list_items(
             filters=[
                 MongoFilter(
                     key="created_on",
                     logical_operator=MongoLogicalOperators.GTE,
-                    value=curr_time - 3600 * 24 * actuality_days,
-                )
+                    value=oldest_timestamp,
+                ),
+                MongoFilter(
+                    key="cid",
+                    value=[chat["_id"] for chat in chats],
+                    logical_operator=MongoLogicalOperators.IN,
+                ),
             ]
         )
-        relevant_shouts = set(x["_id"] for x in relevant_shouts)
+        cids_popularity_counter = Counter()
+        for shout in relevant_shouts:
+            cids_popularity_counter[str(shout["cid"])] += 1
         formatted_chats = []
-        for chat in chats:
-            chat_flow = set(chat.get("chat_flow", []))
-            popularity = len(chat_flow.intersection(relevant_shouts))
-            if chat["_id"] is not None:
-                formatted_chats.append(
-                    ChatPopularityRecord(
-                        cid=str(chat["_id"]),
-                        name=chat["conversation_name"],
-                        popularity=popularity,
-                    )
+        for cid in cids_popularity_counter:
+            relevant_chat = [
+                chat for chat in chats if str(chat.get("_id", "")) == str(cid)
+            ][0]
+            formatted_chats.append(
+                ChatPopularityRecord(
+                    cid=cid,
+                    name=relevant_chat["conversation_name"],
+                    popularity=cids_popularity_counter[cid],
                 )
+            )
         cls.last_updated_ts = int(time())
         cls.__DATA = sorted(formatted_chats, key=lambda x: x.popularity, reverse=True)
 
@@ -111,7 +131,8 @@ def increment_cid_popularity(cls, cid):
             matching_item = [item for item in cls.get_data() if item.cid == cid][0]
             matching_item.popularity += 1
         except IndexError:
-            LOG.error(f"No cid matching = {cid}")
+            LOG.debug(f"No cid matching = {cid}")
+            cls.add_new_chat(cid=cid, popularity=1)
 
     @classmethod
     def get_first_n_items(cls, search_str, exclude_items: list = None, limit: int = 10):