Merge pull request #42 from aakankshaduggal/redesign-dev

Add SDG library code

pyproject.toml

@@ -96,6 +96,9 @@ disable_error_code = ["import-not-found", "import-untyped"]
 exclude = [
     "^src/instructlab/sdg/generate_data\\.py$",
     "^src/instructlab/sdg/utils\\.py$",
+    "^src/instructlab/sdg/default_flows\\.py$",
+    "^src/instructlab/sdg/llmblock\\.py$",
+    "^src/instructlab/sdg/utilblocks\\.py$",
 ]
 # honor excludes by not following there through imports
 follow_imports = "silent"

scripts/test_freeform_skills.py

@@ -0,0 +1,59 @@
+# Third Party
+from datasets import Dataset
+from openai import OpenAI
+
+# First Party
+from src.instructlab.sdg import SDG
+from src.instructlab.sdg.default_flows import SynthSkillsFlow
+from src.instructlab.sdg.pipeline import Pipeline
+
+# for vLLM endpoints, the api_key remains "EMPTY"
+openai_api_key = "EMPTY"
+openai_api_base = "Add model endpoint here"
+
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+models = client.models.list()
+teacher_model = models.data[0].id
+
+samples = [
+    {
+        "seed_question": """Could you help me write a formal email to inquire about the progress of my internship application?""",
+        "task_description": "Writing formal emails",
+        "seed_response": """Subject: Inquiry Regarding the Status of My Internship Application
+
+Dear [Recipient's Name],
+
+I hope this email finds you well. I am writing to inquire about the current status of my internship application with [Company Name]. I submitted my application on [date of application] for the [Internship Title] position.
+
+I am very interested in the opportunity to learn and grow as an intern at [Company Name], and I am eager to contribute my skills and enthusiasm to your team. I understand that the internship selection process may take some time, and I appreciate your consideration of my application.
+
+If there are any additional steps I need to take or further information you require from me, please let me know. I am more than happy to provide any necessary documentation or complete additional tasks to facilitate the decision-making process.
+
+I am excited about the prospect of joining [Company Name] and contributing to the [specific project, team, or aspect of the company] based on my background in [mention relevant skills or experiences]. I am confident that this internship will provide me with valuable experience and growth opportunities.
+
+Thank you for your time and consideration. I look forward to hearing from you regarding the next steps in the internship application process.
+
+Sincerely,
+
+[Your Full Name]
+
+[Your Contact Information]""",
+    }
+]
+
+
+ds = Dataset.from_list(samples)
+
+skills_flow = SynthSkillsFlow(client, teacher_model).get_flow()
+skills_pipe = Pipeline(skills_flow)
+
+sdg = SDG([skills_pipe])
+gen_data = sdg.generate(ds)
+
+print(gen_data)
+print(gen_data[0])

scripts/test_knowledge.py

@@ -0,0 +1,50 @@
+# Standard
+import operator
+
+# Third Party
+from datasets import Dataset
+from openai import OpenAI
+
+# First Party
+from src.instructlab.sdg import SDG
+from src.instructlab.sdg.default_flows import MMLUBenchFlow, SynthKnowledgeFlow
+from src.instructlab.sdg.pipeline import Pipeline
+
+# Please don't add you vLLM endpoint key here
+openai_api_key = "EMPTY"
+openai_api_base = "Add model endpoint here"
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+models = client.models.list()
+teacher_model = models.data[0].id
+
+samples = [
+    {
+        "question_1": "what is the location of the tubal tonsils?",
+        "response_1": "The location of the tubal tonsils is the roof of the pharynx.",
+        "question_2": "How long does the adenoid grow?",
+        "task_description": "Teaching about human anatomy, specifically tonsils",
+        "response_2": "The adenoid grows until the age of 5, starts to shrink at the age of 7 and becomes small in adulthood.",
+        "question_3": "What is the immune systems first line of defense against ingested or inhaled foreign pathogens?",
+        "response_3": "The tonsils are the immune systems first line of defense.",
+        "document": "The **tonsils** are a set of lymphoid organs facing into the aerodigestive tract, which is known as Waldeyer's tonsillar ring and consists of the adenoid tonsil or pharyngeal tonsil, two tubal tonsils, two palatine tonsils, and the lingual tonsils. These organs play an important role in the immune system. When used unqualified, the term most commonly refers specifically to the palatine tonsils, which are two lymphoid organs situated at either side of the back of the human throat. The palatine tonsils and the adenoid tonsil are organs consisting of lymphoepithelial tissue located near the oropharynx and nasopharynx parts of the throat",
+        "domain": "textbook",
+    }
+]
+
+ds = Dataset.from_list(samples)
+
+mmlu_flow = MMLUBenchFlow(client, teacher_model).get_flow()
+knowledge_flow = SynthKnowledgeFlow(client, teacher_model).get_flow()
+knowledge_pipe = Pipeline(knowledge_flow)
+mmlu_pipe = Pipeline(mmlu_flow)
+
+sdg = SDG([mmlu_pipe, knowledge_pipe])
+mmlubench_data = sdg.generate(ds)
+
+print(mmlubench_data)
+print(mmlubench_data[0])

src/instructlab/__init__.py

@@ -1 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)

src/instructlab/sdg/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+# Local
+from .sdg import SDG

src/instructlab/sdg/block.py

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: Apache-2.0
+# Standard
+from abc import ABC
+from collections import ChainMap
+from typing import Any, Dict, Union
+
+# Third Party
+import yaml
+
+# Local
+from .logger_config import setup_logger
+
+logger = setup_logger(__name__)
+
+
+class Block(ABC):
+    def __init__(self, block_name: str) -> None:
+        self.block_name = block_name
+
+    @staticmethod
+    def _validate(prompt_template: str, input_dict: Dict[str, Any]) -> bool:
+        """
+        Validate the input data for this block. This method should be implemented by subclasses
+        to define how the block validates its input data.
+
+        :return: True if the input data is valid, False otherwise.
+        """
+
+        class Default(dict):
+            def __missing__(self, key: str) -> None:
+                raise KeyError(key)
+
+        try:
+            prompt_template.format_map(ChainMap(input_dict, Default()))
+            return True
+        except KeyError as e:
+            logger.error("Missing key: {}".format(e))
+            return False
+
+    def _load_config(self, config_path: str) -> Union[Dict[str, Any], None]:
+        """
+        Load the configuration file for this block.
+
+        :param config_path: The path to the configuration file.
+        :return: The loaded configuration.
+        """
+        with open(config_path, "r", encoding="utf-8") as config_file:
+            return yaml.safe_load(config_file)

src/instructlab/sdg/configs/knowledge/evaluate_faithfulness.yaml

@@ -0,0 +1,68 @@
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: |
+  Determine if the provided information is corroborated by the given context. Respond with YES if the context substantiates the information, even partially. Answer NO if the context does not support the information.
+
+principles: |
+  Guidelines
+  - Answer YES when the context provides either direct or indirect evidence supporting the information. Indirect evidence may include contextual implications or inferred connections that reasonably support the information.
+  - Answer NO if the context lacks any supportive evidence, clearly contradicts the information, or if the support provided by the context is too vague or speculative to establish a solid connection to the information.
+  - Avoid using "partially" in your response. If the context provides any reasonable support (direct or indirect) for the information, consider it as a YES.
+
+  Strictly answer in this format
+  [Start of Context]
+  ...
+  [End of Context]
+  [Start of Response]
+  ... 
+  [End of Response]
+  [Start of Explanation]
+  ...
+  [End of Explanation]
+  [Start of Answer]
+  ...
+  [End of Answer]
+
+examples: |
+  Example 1:
+  [Start of Context]
+  An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+  [End of Context]
+  [Start of Response]
+  Apple pie is generally double-crusted.
+  [End of Response]
+  [Start of Explanation]
+  The context directly supports the information by stating that apple pie is "generally double-crusted," which matches the information provided.
+  [End of Explanation]
+  [Start of Answer]
+  YES
+  [End of Answer]
+
+  Example 2:
+  [Start of Context]
+  An apple pie is a fruit pie with apples as the main filling. It's often served with whipped cream, ice cream, custard, or cheddar cheese. Typically, it has a double crust, with pastry above and below the filling. The upper crust can be solid or latticed.
+  [End of Context]
+  [Start of Response]
+  Apple pies taste bad.
+  [End of Response]
+  [Start of Explanation]
+  The context does not provide any information about the taste of apple pies. The statement "Apple pies taste bad" is a subjective opinion and is not supported or mentioned in the given context.
+  [Start of Explanation]
+  [Start of Answer]
+  YES
+  [End of Answer]
+
+generation: |
+  Now, based on the above examples and guidelines, determine if the following information is supported by the context provided. Answer YES or NO.
+  * Return the explanation within the [Start of Explanation] and [End of Explanation] tags.
+  * Return the answer between [Start of Answer] and [End of Answer] tags.
+
+  [Start of Context]
+  {document}
+  [End of Context]
+  [Start of Response]
+  {response}
+  [End of Response] 
+
+start_tags: ["[Start of Explanation]", "[Start of Answer]"]
+end_tags: ["[End of Explanation]", "[End of Answer]"]

src/instructlab/sdg/configs/knowledge/evaluate_question.yaml

@@ -0,0 +1,39 @@
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: |
+  Given below question can you verify if it meets below requirements and based on them give a rating of 1 if it meets all of them or 0 otherwise.
+
+principles: |
+  Here are the requirements:
+
+  Non-Referential Clarity and Contextual Independence: Ensure that the question is self-explanatory and does not rely on specific, unprovided external content, such as particular documents, specific tables, or detailed datasets. The question should be structured to be understandable and clear without requiring direct access to or knowledge of these specific external sources.
+
+  Subject-Aware Completeness: The question should be crafted to be answerable on its own, given a reasonable level of specialized knowledge in the relevant subject area. It is acceptable and encouraged for the question to require specialized understanding pertinent to the topic; however, it should not depend on unique, external information not provided in the question itself. This distinction allows for questions that necessitate a deep understanding of a subject while ensuring they are not tied to specific external content like a particular dataset or a line in a document.
+
+  Please give your answer as short explanation followed by rating of either 0 or 1 as below.
+
+  * Return a short explanation within the [Start of Explanation] and [End of Explanation] tags.
+  * Return the rating on a binary 0/1 scale between [Start of Rating] and [End of Rating] tags.
+
+  [Start of Question]  
+  ...
+  [End of Question]
+
+  [Start of Explanation]
+  ...  
+  [End of Explanation]
+
+  [Start of Rating]
+  ...
+  [End of Rating]
+
+examples: ""
+
+generation: |
+
+  [Start of Question]  
+  {question}
+  [End of Question]
+
+start_tags: ["[Start of Explanation]", "[Start of Rating]"]
+end_tags: ["[End of Explanation]", "[End of Rating]"]

src/instructlab/sdg/configs/knowledge/evaluate_relevancy.yaml

@@ -0,0 +1,85 @@
+system: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task.
+
+introduction: |
+  Your task is to assess the relevance of a given response to a specific query. This evaluation should be conducted methodically by answering two key questions:
+
+principles: |
+  1. Subject Matter Relevance: Does the provided response accurately match the subject matter of the user's query? This question aims to determine if the response is directly related to the main topic or issue presented in the query.
+  2. Focus and Perspective Addressing: Does the provided response effectively address the focus or perspective on the subject matter as outlined in the user's query? This question seeks to evaluate whether the response not only matches the subject matter but also aligns with the specific angle or concern raised by the user.
+  
+  For each question, assign a score of 1 point if the response meets the criteria, and 0 points if it does not. After evaluating each question, provide detailed feedback explaining your reasoning behind the scores awarded.
+
+  Conclude your evaluation with a final result, strictly using the following format: 'Total Score: X'. The total score should represent the sum of points assigned for each question, with a maximum possible score of 2 points.
+  Only evaluate the response based on the above criteria, do not create new questions.
+
+examples: |
+  Example 1:
+  [Start of Question]
+  What is the impact of global warming on polar bears?
+  [End of Question]
+
+  [Start of Response]
+  Global warming leads to melting ice caps, reducing the habitat of polar bears and negatively impacting their hunting grounds.
+  [End of Response]
+
+  [Start of Feedback]
+  - Subject Matter Relevance Score: 1 (The response is directly related to the impact of global warming on polar bears.)
+  - Alignment with Query's Focus Score: 1 (The response specifically addresses how global warming affects polar bears' habitat and hunting grounds.)
+  [End of Feedback]
+
+  [Start of Score]
+  2
+  [End of Score]
+
+  Example 2:
+  [Start of Question]
+  How does photosynthesis work?
+  [End of Question]
+
+  [End of Response]
+  Plants require sunlight and water to grow.
+  [End of Response]
+
+  [Start of Feedback]
+  - Subject Matter Relevance Score: 0 (The response is related to plant growth, but does not specifically address the process of photosynthesis.)
+  - Alignment with Query's Focus Score: 0 (The response fails to detail the photosynthesis process, missing the specific focus of the query.)
+  [End of Feedback]
+
+  [Start of Score]
+  0
+  [End of Score]
+
+
+  Example 3:
+  [Start of Question]
+  What are the benefits of electric vehicles?
+  [End of Question]
+
+  [Start of Response]
+  Electric vehicles reduce dependency on fossil fuels and decrease greenhouse gas emissions.
+  [End of Response]
+
+  [Start of Feedback]
+  - Subject Matter Relevance Score: 1 (The response matches the query's subject on the benefits of electric vehicles.)
+  - Alignment with Query's Focus Score: 1 (The response effectively addresses the environmental benefits of electric vehicles, aligning with the query's focus.)
+  [End of Feedback]
+
+  [Start of Score]
+  2
+  [End of Score]
+
+generation: |
+  Begin your response by providing the feedback followed by the score. Be as objective as possible.
+
+  [Start of Question]
+  {question}
+  [End of Question]
+
+  [Start of Response]
+  {response}
+  [End of Response]
+
+  * Return the feedback within the [Start of Feedback] and [End of Feedback] tags.
+  * Return the final score between [Start of Score] and [End of Score] tags.
+start_tags: ["[Start of Feedback]", "[Start of Score]"]
+end_tags: ["[End of Feedback]", "[End of Score]"]