Public API (#813)

* specify public API and map shallow to deep imports

* fix docs

* replaced deep with shallow imports

* fix doc

* see if building docs under 3.11 fixes things

* remove things from root namespace that should be used as `config.*`

* remove types from public api that should be used as `types.*`

* allow `__api__` to subset override `__all__` for api pattern in config

* add autogenerated public API

* install repo for Python to locate bsb

* fix empty line diff

.github/devops/generate_public_api.py

@@ -0,0 +1,55 @@
+import difflib
+import sys
+from pathlib import Path
+
+from bsb import _get_public_api_map
+
+
+def public_annotations():
+    annotations = []
+    for api, module in _get_public_api_map().items():
+        annotation = f'"bsb.{module}.{api}"'
+        if api[0].isupper():
+            annotation = f"typing.Type[{annotation}]"
+        annotations.append(f"{api}: {annotation}")
+
+    lines = [
+        "if typing.TYPE_CHECKING:",
+        *sorted(f"  import bsb.{module}" for module in {*_get_public_api_map().values()}),
+        "",
+        *sorted(annotations),
+        "",
+    ]
+
+    return lines
+
+
+if __name__ == "__main__":
+    import bsb
+
+    path = Path(bsb.__path__[0]) / "__init__.py"
+    text = path.read_text()
+    find = (
+        "# Do not modify: autogenerated public API type annotations of the `bsb` module\n"
+        "# fmt: off\n"
+        "# isort: off\n"
+    )
+    idx = text.find(find)
+    annotation_lines = public_annotations()
+    if idx == -1:
+        print("__init__.py file is missing the replacement tag", file=sys.stderr)
+        exit(1)
+    if "--check" in sys.argv:
+        diff = "\n".join(
+            l
+            for l in difflib.ndiff(
+                text[idx + len(find) :].split("\n"),
+                annotation_lines,
+            )
+            if l[0] != " "
+        )
+        print(diff, file=sys.stderr, end="")
+        exit(bool(diff))
+    else:
+        text = text[:idx] + find + "\n".join(annotation_lines)
+        path.write_text(text)

.github/workflows/api.yml

@@ -0,0 +1,21 @@
+name: Check public API
+
+on: [push, pull_request]
+
+jobs:
+  check-documentation:
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.11
+    - name: Setup Python environment
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+    - name: Check public API
+      run: python .github/devops/generate_public_api.py --check

.github/workflows/docs.yml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.9
+    - name: Set up Python 3.11
       uses: actions/setup-python@v1
       with:
-        python-version: 3.9
+        python-version: 3.11
     - name: Setup software environment
       run: |
         sudo apt update

bsb/_package_spec.py

@@ -1,7 +1,7 @@
 from exceptiongroup import ExceptionGroup
 
-from bsb.exceptions import PackageRequirementWarning
-from bsb.reporting import warn
+from .exceptions import PackageRequirementWarning
+from .reporting import warn
 
 
 class MissingRequirementErrors(ExceptionGroup):

bsb/cell_types.py

@@ -148,3 +148,6 @@ def morphologies(self, value):
             "`cell_type.morphologies` is a readonly attribute. Did you mean"
             " `cell_type.spatial.morphologies`?"
         )
+
+
+__all__ = ["CellType", "PlacementIndications", "Plotting"]

bsb/cli/__init__.py

@@ -3,7 +3,7 @@
 import sys
 
 from .._contexts import get_cli_context, reset_cli_context
-from ..exceptions import *
+from ..exceptions import CommandError, DryrunError
 from .commands import load_root_command
 
 
@@ -39,3 +39,6 @@ def _can_dryrun(handler, namespace):
         return bool(inspect.signature(handler).bind(namespace, dryrun=True))
     except TypeError:
         return False
+
+
+__all__ = ["handle_cli", "handle_command"]

bsb/cli/commands/__init__.py

@@ -149,3 +149,6 @@ def load_root_command():
     # class using the `__init_subclass__` function.
     discover("commands")
     return RootCommand()
+
+
+__all__ = ["BaseCommand", "BaseParser", "BsbCommand", "RootCommand", "load_root_command"]

bsb/cli/commands/_commands.py

@@ -56,12 +56,6 @@ class Output(BsbOption, name="output", cli=("output", "o"), env=("BSB_OUTPUT_FIL
     pass
 
 
-class Plot(
-    BsbOption, name="plot", cli=("plot", "p"), env=("BSB_PLOT_NETWORK",), flag=True
-):
-    pass
-
-
 class SkipPlacement(
     BsbOption,
     name="skip_placement",
@@ -102,6 +96,16 @@ class SkipAfterConnectivity(
     pass
 
 
+class IgnoreErrors(
+    BsbOption,
+    name="ignore_errors",
+    cli=("ignore", "ignore-errors"),
+    env=("BSB_IGNORE_ERRORS",),
+    flag=True,
+):
+    pass
+
+
 def _flatten_arr_args(arr):
     if arr is None:
         return arr
@@ -147,13 +151,9 @@ def handler(self, context):
             force=context.force,
             append=context.append,
             redo=context.redo,
+            fail_fast=not context.ignore_errors,
         )
 
-        if context.plot:
-            from bsb.plotting import plot_network
-
-            plot_network(network)
-
     def get_options(self):
         return {
             "x": XScale(),
@@ -169,8 +169,8 @@ def get_options(self):
             "append": Append(),
             "redo": Redo(),
             "clear": Clear(),
-            "plot": Plot(),
             "output": Output(),
+            "ignore_errors": IgnoreErrors(),
         }
 
     def add_parser_arguments(self, parser):

bsb/cli/commands/_projects.py

@@ -67,9 +67,9 @@ def handler(self, context):
         conn_path = root / "connectome.py"
         if not place_path.exists():
             with open(place_path, "w") as f:
-                f.write("from bsb.placement import PlacementStrategy\n")
+                f.write("from bsb import PlacementStrategy\n")
         if not conn_path.exists():
             with open(conn_path, "w") as f:
-                f.write("from bsb.connectivity import ConnectionStrategy\n")
+                f.write("from bsb import ConnectionStrategy\n")
 
         report(f"Created '{name}' project structure.", level=1)

bsb/config/__init__.py

@@ -291,3 +291,64 @@ def parser_method(self, file=None, data=None, path=None):
 
 ConfigurationModule.__all__ = sorted(ConfigurationModule.__all__)
 sys.modules[__name__] = ConfigurationModule(__name__)
+
+# Static public API
+__all__ = [
+    "Configuration",
+    "ConfigurationAttribute",
+    "Distribution",
+    "after",
+    "attr",
+    "before",
+    "catch_all",
+    "compose_nodes",
+    "copy_template",
+    "dict",
+    "dynamic",
+    "file",
+    "format_content",
+    "from_content",
+    "from_file",
+    "from_json",
+    "get_config_attributes",
+    "get_config_path",
+    "get_parser",
+    "has_hook",
+    "list",
+    "make_config_diagram",
+    "node",
+    "on",
+    "parsers",
+    "pluggable",
+    "property",
+    "provide",
+    "ref",
+    "reflist",
+    "root",
+    "run_hook",
+    "slot",
+    "types",
+    "unset",
+    "walk_node_attributes",
+    "walk_nodes",
+]
+__api__ = [
+    "Configuration",
+    "ConfigurationAttribute",
+    "Distribution",
+    "compose_nodes",
+    "copy_template",
+    "format_content",
+    "from_content",
+    "from_file",
+    "from_json",
+    "get_config_attributes",
+    "get_config_path",
+    "get_parser",
+    "make_config_diagram",
+    "parsers",
+    "refs",
+    "types",
+    "walk_node_attributes",
+    "walk_nodes",
+]

bsb/config/_make.py

@@ -364,7 +364,7 @@ def _bubble_up_warnings(log):
 
 
 def _bootstrap_components(components, file_store=None):
-    from bsb.storage._files import CodeDependencyNode
+    from ..storage._files import CodeDependencyNode
 
     for component in components:
         component_node = CodeDependencyNode(component)

bsb/config/parsers/__init__.py

@@ -9,3 +9,6 @@ def get_parser_classes():
 
 def get_parser(parser):
     return get_parser_classes()[parser]()
+
+
+__all__ = ["Parser", "get_parser", "get_parser_classes"]

bsb/config/refs.py

@@ -101,7 +101,7 @@ def is_ref(self, value):
 
 class SimCellModelReference(Reference):
     def __call__(self, root, here):
-        from bsb.simulation.simulation import Simulation
+        from ..simulation.simulation import Simulation
 
         sim = self.up(here, Simulation)
         return sim.cell_models
@@ -121,4 +121,15 @@ def is_ref(self, value):
 region_ref = RegionReference()
 sim_cell_model_ref = SimCellModelReference()
 
-__all__ = [k for k in vars().keys() if k.endswith("_ref") or k.endswith("__")]
+__all__ = [
+    "Reference",
+    "cell_type_ref",
+    "conn_type_ref",
+    "partition_ref",
+    "placement_ref",
+    "connectivity_ref",
+    "regional_ref",
+    "region_ref",
+    "sim_cell_model_ref",
+]
+__api__ = ["Reference"]

bsb/config/types.py

@@ -781,3 +781,37 @@ def __inv__(self, value):
 
     def __hint__(self):
         return "numpy==1.24.0"
+
+
+__all__ = [
+    "PackageRequirement",
+    "TypeHandler",
+    "WeakInverter",
+    "any_",
+    "class_",
+    "deg_to_radian",
+    "dict",
+    "distribution",
+    "evaluation",
+    "float",
+    "fraction",
+    "function_",
+    "in_",
+    "in_classmap",
+    "int",
+    "key",
+    "list",
+    "list_or_scalar",
+    "method",
+    "method_shortcut",
+    "mut_excl",
+    "ndarray",
+    "number",
+    "object_",
+    "or_",
+    "scalar_expand",
+    "shortform",
+    "str",
+    "voxel_size",
+]
+__api__ = ["PackageRequirement", "TypeHandler", "WeakInverter"]

bsb/connectivity/__init__.py

@@ -1,5 +1,9 @@
+# isort: off
+# Load module before others to prevent partially initialized modules
+from .strategy import ConnectionStrategy
+
+# isort: on
 from .detailed import *
 from .detailed.fiber_intersection import FiberTransform, QuiverTransform
 from .general import *
 from .import_ import CsvImportConnectivity
-from .strategy import ConnectionStrategy

bsb/connectivity/detailed/__init__.py

@@ -1,19 +1,3 @@
-import inspect
-import os
-from glob import glob
-from importlib import import_module
-
-from ..strategy import ConnectionStrategy
-
-# Scan the whole directory for python files, then import any ConnectionStrategies into
-# this module.
-src_files = glob(os.path.join(os.path.dirname(__file__), "*.py"))
-exclude_src = ["__init__"]
-for src_file in src_files:
-    module_name = os.path.basename(src_file).split(".")[0]
-    if module_name in exclude_src:
-        continue
-    module = import_module("." + module_name, __package__)
-    for name, obj in module.__dict__.items():
-        if inspect.isclass(obj) and issubclass(obj, ConnectionStrategy):
-            globals()[name] = obj
+from .fiber_intersection import FiberIntersection
+from .touch_detection import TouchDetector
+from .voxel_intersection import VoxelIntersection

bsb/connectivity/detailed/fiber_intersection.py

@@ -5,7 +5,11 @@
 
 from ... import config
 from ...config import types
-from ...exceptions import *
+from ...exceptions import (
+    ConfigurationError,
+    IncompleteMorphologyError,
+    QuiverFieldWarning,
+)
 from ...reporting import warn
 from ..strategy import ConnectionStrategy
 from .shared import Intersectional
@@ -426,3 +430,6 @@ def get_branch_direction(self, branch):
             # Normalize branch_dir vector
             branch_dir = branch_dir / np.linalg.norm(branch_dir)
             return branch_dir
+
+
+__all__ = ["FiberIntersection", "FiberTransform", "QuiverTransform"]