feat: Update release workflow and manifest for improved dependency management and layer handling

.github/workflows/release.yml

@@ -3,6 +3,7 @@ name: Release Version
 on:
   push:
     tags:
+      # Publish on any tag starting with a `v`, e.g., v0.1.0
       - 'v*'
 
 jobs:
@@ -42,14 +43,17 @@ jobs:
         uses: actions/checkout@v4
         with:
           lfs: "true"
-      - name: Set up Python
+      - name: Install uv
-        uses: actions/setup-python@v4
+        uses: astral-sh/setup-uv@v7
-        with:
+      - name: Install Dependencies
-          python-version: 3.11
+        run: |
+          uv python install 3.11
+          uv python pin 3.11
+          uv sync
       - name: Build Extension
         run: | # build.py need tomlkit to edit blender_manifest.toml
-          pip install tomlkit
+          uv run build.py ${{ matrix.platform }} 3.11
-          python build.py ${{ matrix.platform }}
+          uv run build.py ${{ matrix.platform }} 3.13
           cd src/bioxelnodes
           zip -r ../../package.zip .
       - name: Upload Extension
@@ -60,5 +64,5 @@ jobs:
         with:
           upload_url: ${{ needs.draft_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
           asset_path: ./package.zip
-          asset_name: BioxelNodes_${{ needs.draft_release.outputs.version }}_${{ matrix.platform }}.zip
+          asset_name: BioxelNodes.${{ needs.draft_release.outputs.version }}.${{ matrix.platform }}.zip
           asset_content_type: application/zip

build.py

@@ -12,10 +12,10 @@ class Platform:
     blender_tag: str
 
 
-required_packages = ["SimpleITK==2.3.1",
+required_packages = ["SimpleITK==2.5.5",
                      "pyometiff==1.0.0",
                      "mrcfile==1.5.1",
-                     "h5py==3.11.0",
+                     "h5py==3.16.0",
                      "transforms3d==0.4.2",
                      "tifffile==2024.7.24",
                      "matplotlib==3.10.7",
@@ -42,7 +42,7 @@ def run_python(args: str):
     subprocess.run([python] + args.split(" "))
 
 
-def build_extension(platform: Platform, python_version="3.11") -> None:
+def build_extension(platform: Platform, python_version: str) -> None:
     wheel_dirpath = Path("./src/bioxelnodes/wheels")
     toml_filepath = Path("./src/bioxelnodes/blender_manifest.toml")
     scipy_ndimage_dirpath = Path("./scipy_ndimage", platform.blender_tag)
@@ -52,26 +52,28 @@ def build_extension(platform: Platform, python_version="3.11") -> None:
         f"-m pip download {' '.join(required_packages)} --dest {wheel_dirpath.as_posix()} --only-binary=:all: --python-version={python_version} --platform={platform.pypi_suffix}"
     )
 
-    for f in wheel_dirpath.glob('*.whl'):
+    for f in wheel_dirpath.glob("*.whl"):
         if any([package in f.name for package in packages_to_remove]):
             f.unlink(missing_ok=True)
 
-        elif platform.blender_tag == "macos-arm64" and \
+        elif (
-                "lxml" in f.name and "universal2" in f.name:
+            platform.blender_tag == "macos-arm64"
-            f.rename(Path(f.parent,
+            and "lxml" in f.name
-                          f.name.replace("universal2", "arm64")))
+            and "universal2" in f.name
+        ):
+            f.rename(Path(f.parent, f.name.replace("universal2", "arm64")))
 
     for ndimage_filepath in scipy_ndimage_dirpath.iterdir():
         to_filepath = Path("./src/bioxelnodes/bioxel/scipy", ndimage_filepath.name)
         shutil.copy(ndimage_filepath, to_filepath)
-
+        
     # Load the TOML file
     with toml_filepath.open("r") as file:
         manifest = tomlkit.parse(file.read())
 
     manifest["platforms"] = [platform.blender_tag]
-    manifest["wheels"] = [f"./wheels/{f.name}"
+    manifest["wheels"] = [
-                          for f in wheel_dirpath.glob('*.whl')]
+        f"./wheels/{f.name}" for f in wheel_dirpath.glob("*.whl")]
 
     # build.append('generated', generated)
     # manifest.append('build', build)
@@ -84,8 +86,9 @@ def build_extension(platform: Platform, python_version="3.11") -> None:
 
 def main():
     platform_name = sys.argv[1]
+    python_version = sys.argv[2] or "3.11"
     platform = platforms[platform_name]
-    build_extension(platform)
+    build_extension(platform, python_version)
 
 
 if __name__ == "__main__":

src/bioxelnodes/blender_manifest.toml

@@ -14,25 +14,7 @@ license = ['SPDX:GPL-3.0-or-later']
 copyright = ["2024 OmooLab"]
 platforms = ["windows-x64"]
 
-wheels = [
+wheels = ["./wheels/contourpy-1.3.3-cp311-cp311-win_amd64.whl", "./wheels/contourpy-1.3.3-cp313-cp313-win_amd64.whl", "./wheels/cycler-0.12.1-py3-none-any.whl", "./wheels/fonttools-4.63.0-cp311-cp311-win_amd64.whl", "./wheels/fonttools-4.63.0-cp313-cp313-win_amd64.whl", "./wheels/h5py-3.16.0-cp311-cp311-win_amd64.whl", "./wheels/h5py-3.16.0-cp313-cp313-win_amd64.whl", "./wheels/kiwisolver-1.5.0-cp311-cp311-win_amd64.whl", "./wheels/kiwisolver-1.5.0-cp313-cp313-win_amd64.whl", "./wheels/lxml-6.1.1-cp311-cp311-win_amd64.whl", "./wheels/lxml-6.1.1-cp313-cp313-win_amd64.whl", "./wheels/matplotlib-3.10.7-cp311-cp311-win_amd64.whl", "./wheels/matplotlib-3.10.7-cp313-cp313-win_amd64.whl", "./wheels/mrcfile-1.5.1-py2.py3-none-any.whl", "./wheels/packaging-26.2-py3-none-any.whl", "./wheels/pillow-11.2.1-cp311-cp311-win_amd64.whl", "./wheels/pillow-11.2.1-cp313-cp313-win_amd64.whl", "./wheels/pyometiff-1.0.0-py3-none-any.whl", "./wheels/pyparsing-3.3.2-py3-none-any.whl", "./wheels/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", "./wheels/simpleitk-2.5.5-cp311-abi3-win_amd64.whl", "./wheels/six-1.17.0-py2.py3-none-any.whl", "./wheels/tifffile-2024.7.24-py3-none-any.whl", "./wheels/transforms3d-0.4.2-py3-none-any.whl"]
-    "./wheels/contourpy-1.3.3-cp311-cp311-win_amd64.whl",
-    "./wheels/cycler-0.12.1-py3-none-any.whl",
-    "./wheels/fonttools-4.60.1-cp311-cp311-win_amd64.whl",
-    "./wheels/h5py-3.11.0-cp311-cp311-win_amd64.whl",
-    "./wheels/kiwisolver-1.4.9-cp311-cp311-win_amd64.whl",
-    "./wheels/lxml-6.0.2-cp311-cp311-win_amd64.whl",
-    "./wheels/matplotlib-3.10.7-cp311-cp311-win_amd64.whl",
-    "./wheels/mrcfile-1.5.1-py2.py3-none-any.whl",
-    "./wheels/packaging-25.0-py3-none-any.whl",
-    "./wheels/pillow-11.2.1-cp311-cp311-win_amd64.whl",
-    "./wheels/pyometiff-1.0.0-py3-none-any.whl",
-    "./wheels/pyparsing-3.2.5-py3-none-any.whl",
-    "./wheels/python_dateutil-2.9.0.post0-py2.py3-none-any.whl",
-    "./wheels/SimpleITK-2.3.1-cp311-cp311-win_amd64.whl",
-    "./wheels/six-1.17.0-py2.py3-none-any.whl",
-    "./wheels/tifffile-2024.7.24-py3-none-any.whl",
-    "./wheels/transforms3d-0.4.2-py3-none-any.whl",
-]
 
 [permissions]
 files = "Import/export volume data from/to disk"

src/bioxelnodes/constants.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 NODE_LIB_DIRPATH = Path(Path(__file__).parent,
-                        "assets/O Bioxel").resolve()
+                        "assets/O_Bioxel").resolve()
 
 LATEST_NODE_LIB_PATH = Path(NODE_LIB_DIRPATH,
                             "BioxelNodes.blend").resolve()

src/bioxelnodes/operators/io.py

@@ -12,20 +12,37 @@ import transforms3d
 from ..props import BIOXEL_Series
 from ..utils import get_layer_obj, wrapped_label
 
-from ..bioxel import (
+from ..bioxel.layer import Layer
-    DICOM_EXTS,
+from ..bioxel.parse import DICOM_EXTS, SUPPORT_EXTS, get_ext, parse_volumetric_data
-    SUPPORT_EXTS,
-    read,
-    read_meta,
-    calc_layer_shape,
-    calc_layer_size,
-    get_ext,
-)
 
 from ..utils import get_cache_dir, progress_update, progress_bar
 from ..layer import get_layer_caches, save_layers_to_json
 
 
+def get_layer_shape(bioxel_size: float, orig_shape: tuple, orig_spacing: tuple):
+    shape = (
+        int(orig_shape[0] / bioxel_size * orig_spacing[0]),
+        int(orig_shape[1] / bioxel_size * orig_spacing[1]),
+        int(orig_shape[2] / bioxel_size * orig_spacing[2]),
+    )
+
+    return (
+        shape[0] if shape[0] > 0 else 1,
+        shape[1] if shape[1] > 0 else 1,
+        shape[2] if shape[2] > 0 else 1,
+    )
+
+
+def get_layer_size(shape: tuple, bioxel_size: float, scale: float = 1.0):
+    size = (
+        float(shape[0] * bioxel_size * scale),
+        float(shape[1] * bioxel_size * scale),
+        float(shape[2] * bioxel_size * scale),
+    )
+
+    return size
+
+
 """
 ImportData    -> ParseVolumetricData -> ImportDataDialog
     start import                 parse data              execute import
@@ -194,7 +211,11 @@ class ParseVolumetricData(bpy.types.Operator):
 
             try:
                 series_id = self.series_id if self.series_id != "empty" else ""
-                data = read_meta(self.filepath, series_id=series_id)
+                data, meta = parse_volumetric_data(
+                    data_file=self.filepath,
+                    series_id=series_id,
+                    progress_callback=progress_callback,
+                )
             except KeyboardInterrupt:
                 return
             except Exception as e:
@@ -204,8 +225,8 @@ class ParseVolumetricData(bpy.types.Operator):
             if cancel():
                 return
 
-            self.meta = data.meta
+            self.meta = meta
-            self.label_count = int(np.max(data.data))
+            self.label_count = int(np.max(data))
             self.dtype = data.dtype
 
         # Init cancel flag
@@ -313,8 +334,8 @@ class ParseVolumetricData(bpy.types.Operator):
 
         bioxel_size = max(min(*orig_spacing), 1.0)
 
-        layer_shape = calc_layer_shape(bioxel_size, orig_shape, orig_spacing)
+        layer_shape = get_layer_shape(bioxel_size, orig_shape, orig_spacing)
-        layer_size = calc_layer_size(layer_shape, bioxel_size, 0.01)
+        layer_size = get_layer_size(layer_shape, bioxel_size, 0.01)
         min_log10 = math.floor(math.log10(min(*layer_size)))
         max_log10 = math.floor(math.log10(max(*layer_size)))
 
@@ -526,23 +547,33 @@ class ImportDataDialog(bpy.types.Operator):
 
     def execute(self, context):
         def import_volumetric_data_func(self, context, cancel):
+            progress_update(context, 0.0, "Parsing Volumetirc Data...")
+
             def progress_callback(factor, text):
                 if cancel():
                     raise KeyboardInterrupt("Cancelled by user")
                 progress_update(context, factor * 0.2, text)
 
-            try:
+            def progress_callback_factory(layer_name, progress, progress_step):
-                data = read(self.filepath, self.series_id, progress_callback)
+                def progress_callback(frame, total):
+                    if cancel():
+                        raise KeyboardInterrupt("Cancelled by user")
+                    sub_progress_step = progress_step / total
+                    sub_progress = progress + frame * sub_progress_step
+                    progress_update(
+                        context,
+                        sub_progress,
+                        f"Processing {layer_name} Frame {frame+1}...",
+                    )
+                    print(f"Processing {layer_name} Frame {frame+1}...")
 
-                self.layers = data.to_layers(
+                return progress_callback
-                    kind=self.read_as.lower(),
+
-                    layer_name=self.layer_name,
+            try:
-                    bioxel_size=self.bioxel_size,
+                data, meta = parse_volumetric_data(
-                    smooth=self.smooth,
+                    data_file=self.filepath,
-                    remap=self.remap,
+                    series_id=self.series_id,
-                    split_channel=self.split_channel,
+                    progress_callback=progress_callback,
-                    frame_source=self.frame_source,
-                    progress_callback=lambda f, t: progress_callback(0.2 + f * 0.7, t),
                 )
 
             except KeyboardInterrupt:
@@ -554,6 +585,203 @@ class ImportDataDialog(bpy.types.Operator):
             if cancel():
                 return
 
+            shape = get_layer_shape(
+                self.bioxel_size, self.orig_shape, self.orig_spacing
+            )
+
+            mat_scale = transforms3d.zooms.zfdir2aff(self.bioxel_size)
+            affine = np.dot(meta["affine"], mat_scale)
+            kind = self.read_as.lower()
+
+            if cancel():
+                return
+
+            # change shape as sequence or not
+            if self.frame_source == "-1":
+                data = data[0:1, :, :, :, :]
+            elif self.frame_source == "0":
+                # frame as frame
+                pass
+            elif self.frame_source == "1":
+                # X as frame
+                data = data.transpose(1, 0, 2, 3, 4)
+                shape = (1, shape[1], shape[2])
+            elif self.frame_source == "2":
+                # Y as frame
+                data = data.transpose(2, 1, 0, 3, 4)
+                shape = (shape[0], 1, shape[2])
+            elif self.frame_source == "3":
+                # Z as frame
+                data = data.transpose(3, 1, 2, 0, 4)
+                shape = (shape[0], shape[1], 1)
+            else:
+                # channel as frame
+                data = data.transpose(4, 1, 2, 3, 0)
+
+            layers = []
+            if kind == "label":
+                name = self.layer_name or "Label"
+                data = data.astype(int)
+                label_count = int(np.max(data))
+                progress_step = 0.7 / label_count
+
+                for i in range(label_count):
+                    if cancel():
+                        return
+
+                    name_i = f"{name}_{i+1}"
+                    progress = 0.2 + i * progress_step
+                    progress_update(context, progress, f"Processing {name_i}...")
+
+                    progress_callback = progress_callback_factory(
+                        name_i, progress, progress_step
+                    )
+                    label_data = data == np.full_like(data, i + 1)
+                    # label_data = label_data.astype(np.float32)
+                    try:
+                        layer = Layer(data=label_data, name=name_i, kind=kind)
+
+                        layer.resize(
+                            shape=shape,
+                            smooth=self.smooth,
+                            progress_callback=progress_callback,
+                        )
+
+                        layer.affine = affine
+
+                        layers.append(layer)
+                    except KeyboardInterrupt:
+                        return
+                    except Exception as e:
+                        self.has_error = e
+                        return
+
+            if kind == "color":
+                if np.issubdtype(np.uint8, data.dtype):
+                    data = np.multiply(data, 1.0 / 256, dtype=np.float32)
+                elif data.dtype.kind in ["u", "i"]:
+                    # Convert the normalized array to float dtype
+                    data = data.astype(np.float32)
+
+                    min_val = data.min()
+                    max_val = data.max()
+                    # Avoid division by zero if all values are the same
+                    if max_val != min_val:
+                        # Normalize the array to the range (0,1)
+                        data = (data - min_val) / (max_val - min_val)
+                    else:
+                        # If all values are the same, the normalized array will be all zeros
+                        data = np.zeros_like(data, dtype=np.float32)
+
+                else:
+                    data = data.astype(np.float32)
+
+                # Gamma Correct
+                # data = data ** 2.2
+
+                name = self.layer_name or "Color"
+                if data.shape[4] == 1:
+                    data = np.repeat(data, repeats=3, axis=4)
+                elif data.shape[4] == 2:
+                    d_shape = list(data.shape)
+                    d_shape = d_shape[:4] + [1]
+                    zore = np.zeros(tuple(d_shape), dtype=np.float32)
+                    data = np.concatenate((data, zore), axis=-1)
+                elif data.shape[4] > 3:
+                    data = data[:, :, :, :, :3]
+
+                if cancel():
+                    return
+
+                progress_update(context, 0.2, f"Processing {name}...")
+                progress_callback = progress_callback_factory(name, 0.2, 0.7)
+
+                try:
+                    layer = Layer(data=data, name=name, kind=kind)
+
+                    layer.resize(shape=shape, progress_callback=progress_callback)
+
+                    layer.affine = affine
+
+                    layers.append(layer)
+                except KeyboardInterrupt:
+                    return
+                except Exception as e:
+                    self.has_error = e
+                    return
+
+            elif kind == "scalar":
+                name = self.layer_name or "Scalar"
+
+                if self.remap:
+                    # Convert the normalized array to float dtype
+                    data = data.astype(np.float32)
+
+                    min_val = data.min()
+                    max_val = data.max()
+                    # Avoid division by zero if all values are the same
+                    if max_val != min_val:
+                        # Normalize the array to the range (0,1)
+                        data = (data - min_val) / (max_val - min_val)
+                    else:
+                        # If all values are the same, the normalized array will be all zeros
+                        data = np.zeros_like(data, dtype=np.float32)
+
+                if self.split_channel:
+                    progress_step = 0.7 / self.channel_count
+
+                    for i in range(self.channel_count):
+                        if cancel():
+                            return
+
+                        name_i = f"{name}_{i+1}"
+                        progress = 0.2 + i * progress_step
+                        progress_update(context, progress, f"Processing {name_i}...")
+                        progress_callback = progress_callback_factory(
+                            name_i, progress, progress_step
+                        )
+                        try:
+                            layer = Layer(
+                                data=data[:, :, :, :, i : i + 1], name=name_i, kind=kind
+                            )
+
+                            layer.resize(
+                                shape=shape, progress_callback=progress_callback
+                            )
+
+                            layer.affine = affine
+
+                            layers.append(layer)
+                        except KeyboardInterrupt:
+                            return
+                        except Exception as e:
+                            self.has_error = e
+                            return
+                else:
+                    if cancel():
+                        return
+
+                    progress_update(context, 0.2, f"Processing {name}...")
+                    progress_callback = progress_callback_factory(name, 0.2, 0.7)
+
+                    try:
+                        layer = Layer(data=data, name=name, kind=kind)
+
+                        layer.resize(shape=shape, progress_callback=progress_callback)
+
+                        layer.affine = affine
+
+                        layers.append(layer)
+                    except KeyboardInterrupt:
+                        return
+                    except Exception as e:
+                        self.has_error = e
+                        return
+
+            if cancel():
+                return
+
+            self.layers = layers
             progress_update(context, 0.9, "Creating Layers...")
 
         self.is_cancelled = False
@@ -623,7 +851,7 @@ class ImportDataDialog(bpy.types.Operator):
         return {"RUNNING_MODAL"}
 
     def draw(self, context):
-        layer_shape = calc_layer_shape(
+        layer_shape = get_layer_shape(
             self.bioxel_size, self.orig_shape, self.orig_spacing
         )