Merge pull request #41 from IDEA-Research/dev

Release v0.3.0

.github/workflows/tool-ci.yaml

@@ -97,13 +97,13 @@ jobs:
         run: |
           pip install -r requirements-dev.txt
           pytest --cov=deepdataspace -s --cov-report=xml
-      - name: Upload Test Report
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          fail_ci_if_error: true
-          verbose: true
-          flags: backend
+      # - name: Upload Test Report
+      #   uses: codecov/codecov-action@v3
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     fail_ci_if_error: true
+      #     verbose: true
+      #     flags: backend
   test_frontend:
     if: github.repository_owner == 'IDEA-Research'
     runs-on: self-hosted
@@ -118,10 +118,10 @@ jobs:
         run: pnpm run dev:build
       - name: Run tests and collect coverage
         run: pnpm run test
-      - name: Upload Test Report
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          fail_ci_if_error: true
-          verbose: true
-          flags: frontend
+      # - name: Upload Test Report
+      #   uses: codecov/codecov-action@v3
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     fail_ci_if_error: true
+      #     verbose: true
+      #     flags: frontend

README.md

@@ -43,9 +43,10 @@ Deep Data Space (DDS) is an open-source dataset tool with these features out-of-
 
 ### 1.1 Prerequisites
 DeepDataSpace(DDS) requires **Python 3.8 - 3.10** and runs on the following platforms:
-- Mac, x86/arm64
-- Windows 10, x86/x64
-- Ubuntu 18.04/20.04/22.04, x86/x64
+- Mac OS: ✅ x86/x64, ✅ arm64
+- Windows 10: ✅ x86/x64, ❌ arm64
+- Ubuntu LTS since 18.04: ✅ x86/x64, ❌ arm64
+- Docker: ✅ x86/x64, ❌ arm64
 
 ### 1.2 Installing from PyPI
 
@@ -133,12 +134,12 @@ docker run -it --name dds --rm \
     dds --quickstart -V
 ```
 
-If everything goes well, you can start visit DDS at [http://127.0.0.1:8765](http://127.0.0.1:8765)
+If everything goes well, you can start visiting DDS at [http://127.0.0.1:8765](http://127.0.0.1:8765)
 
 #### Step 3: Mount your dataset directory(**Optional**)
 
-If you start DDS in `quickstart` mode, DDS will try to download some sample datasets and import them for you.  
-But most frequently, you want DDS to import your local dataset files. This is possible by mouting your local dataset directory to `/dds/datasets` inside container.
+If you start DDS in `quickstart` mode, DDS will try to download the sample datasets and import them for you.  
+But most frequently, you want DDS to import your local dataset files. This is possible by mounting your local dataset directory to `/dds/datasets` inside container.
 
 ```
 # assume $PWD/datasets is your local dataset directory

deepdataspace/constants.py

@@ -261,6 +261,7 @@ class LabelProjectStatus:
     Reviewing = "reviewing"  #: Manager has finished the label project, waiting for owner to review.
     Rejected = "rejected"  #: Owner rejected the project.
     Accepted = "accepted"  #: Owner accepted the project.
+    Exported = "exported"  #: Owner has exported the project back to datasets.
 
 
 class LabelProjectRoles:

deepdataspace/environs.py

@@ -124,6 +124,6 @@
 
 # celery configurations
 CELERY_BROKER = f"redis://:{REDIS_PASS}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DBNAME}"
-CELERY_WORKER_POOL = os.environ.get("DDS_CELERY_WORKERPOOL", "prefork")
+CELERY_WORKER_POOL = os.environ.get("DDS_CELERY_WORKER_POOL", "prefork")
 
 SENTRY_DSN: str = os.environ.get("DDS_SENTRY_DSN", None)

deepdataspace/model/dataset.py

@@ -4,6 +4,7 @@
 The dataset model.
 """
 
+import importlib
 import json
 import logging
 import os
@@ -54,6 +55,9 @@ def get_collection(cls):
     object_types: list = []  # what kind of objects this dataset contains
     num_images: int = 0
     files: dict = {}  # the relevant files of this dataset
+    cover_url: str = None  # the cover image url
+    description: str = None  # the dataset description
+    description_func: str = None  # a function to generate description
     group_id: str = None
     group_name: str = None
 
@@ -67,6 +71,8 @@ def create_dataset(cls,
                        type: str = None,
                        path: str = None,
                        files: dict = None,
+                       description: str = None,
+                       description_func: str = None,
                        ) -> "DataSet":
         """
         Create a dataset.
@@ -78,8 +84,13 @@ def create_dataset(cls,
         :param type: the optional dataset type, can be "tsv", "coco2017".
         :param path: the optional dataset directory path.
         :param files: the optional dataset relevant files. The key is the file info, the value is the file path.
+        :param description: the optional dataset description.
+        :param description_func: an import path of a function to generate description.
+            The function takes the dataset instance as the only argument and returns a string.
+            If this is provided, it proceeds the description str.
         :return: the dataset object.
         """
+
         if id_:
             dataset = DataSet.find_one({"id": id_})
             if dataset is not None:
@@ -93,7 +104,9 @@ def create_dataset(cls,
             id_ = uuid.uuid4().hex
 
         files = files or {}
-        dataset = cls(name=name, id=id_, type=type, path=path, files=files, status=DatasetStatus.Ready)
+        dataset = cls(name=name, id=id_, type=type, path=path,
+                      files=files, status=DatasetStatus.Ready,
+                      description=description, description_func=description_func)
         dataset.post_init()
         dataset.save()
         return dataset
@@ -129,6 +142,19 @@ def get_importing_dataset(cls,
         dataset.save()
         return dataset
 
+    def _add_cover(self, force_update: bool = False):
+        has_cover = bool(self.cover_url)
+        if has_cover and not force_update:
+            return
+
+        IModel = Image(self.id)
+        images = list(IModel.find_many({}, sort=[("idx", 1)], size=1))
+        if not images:
+            return
+
+        self.cover_url = images[0].url.strip()
+        self.save()
+
     def add_image(self,
                   uri: str,
                   thumb_uri: str = None,
@@ -195,7 +221,7 @@ def add_image(self,
 
         image.save()
         self.num_images = Model.count_num({})
-        self.save()
+        self._add_cover()
 
         # save whitelist to redis
         whitelist_dirs = set()
@@ -347,3 +373,23 @@ def finish_batch_add_image(self):
         This saves all images in the buffer queue to database.
         """
         self._save_image_batch()
+        self._add_cover()
+
+    def eval_description(self):
+        """
+        Evaluate the description function and return the description.
+        """
+
+        if self.description_func is not None:
+            try:
+                module, func = self.description_func.rsplit(".", 1)
+                description_module = importlib.import_module(module)
+                description_func = getattr(description_module, func)
+                return description_func(self)
+            except (ImportError, AttributeError):
+                msg = f"Cannot import description_func[{self.description_func}] for dataset[{self.id}]"
+                logger.warning(msg)
+                return self.description or self.path
+            except:
+                logger.warning(f"Failed to eval description_func[{self.description_func}] for dataset[{self.id}]")
+                return self.description or self.path

deepdataspace/model/image.py

@@ -265,6 +265,35 @@ def _update_dataset(self, bbox, segmentation, alpha_uri, coco_keypoints):
         if modified:
             self.dataset.save()
 
+    def _add_annotation(self,
+                        category: str,
+                        label: str = LabelName.GroundTruth,
+                        label_type: Literal["GT", "Pred", "User"] = "GT",
+                        conf: float = 1.0,
+                        is_group: bool = False,
+                        bbox: Tuple[int, int, int, int] = None,
+                        segmentation: List[List[int]] = None,
+                        alpha_uri: str = None,
+                        coco_keypoints: List[Union[float, int]] = None,
+                        confirm_type: int = 0,
+                        ):
+        if bbox:
+            if not self.width or not self.height:
+                raise ValueError("image width and height must be set before setting bbox")
+
+        label_obj = self._add_label(label, label_type)
+        category_obj = self._add_category(category)
+        bounding_box = self._format_bbox(self.width, self.height, bbox)
+        segmentation = self._format_segmentation(segmentation)
+        points, colors, lines, names = self._format_coco_keypoints(coco_keypoints)
+        anno_obj = Object(label_name=label, label_type=label_type, label_id=label_obj.id,
+                          category_name=category, category_id=category_obj.id,
+                          bounding_box=bounding_box, segmentation=segmentation, alpha=alpha_uri,
+                          points=points, lines=lines, point_colors=colors, point_names=names,
+                          conf=conf, is_group=is_group, confirm_type=confirm_type)
+        anno_obj.post_init()
+        self.objects.append(anno_obj)
+
     def add_annotation(self,
                        category: str,
                        label: str = LabelName.GroundTruth,
@@ -294,22 +323,9 @@ def add_annotation(self,
         :param confirm_type: the confirm_type of the annotation, 0 = not confirmed, 1 = gt may be fn, 2 = pred may be fp
         """
 
-        if bbox:
-            if not self.width or not self.height:
-                raise ValueError("image width and height must be set before setting bbox")
-
-        label_obj = self._add_label(label, label_type)
-        category_obj = self._add_category(category)
-        bounding_box = self._format_bbox(self.width, self.height, bbox)
-        segmentation = self._format_segmentation(segmentation)
-        points, colors, lines, names = self._format_coco_keypoints(coco_keypoints)
-        anno_obj = Object(label_name=label, label_type=label_type, label_id=label_obj.id,
-                          category_name=category, category_id=category_obj.id,
-                          bounding_box=bounding_box, segmentation=segmentation, alpha=alpha_uri,
-                          points=points, lines=lines, point_colors=colors, point_names=names,
-                          conf=conf, is_group=is_group, confirm_type=confirm_type)
-        anno_obj.post_init()
-        self.objects.append(anno_obj)
+        self._add_annotation(category, label, label_type, conf,
+                             is_group, bbox, segmentation, alpha_uri,
+                             coco_keypoints, confirm_type)
 
         self.save()
         self._update_dataset(bbox, segmentation, alpha_uri, coco_keypoints)