diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 51c6b57..bb3ee98 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,7 +26,7 @@ jobs: set -e cd /opt/tono git pull --ff-only - .venv/bin/pip install -e . --quiet + .venv/bin/pip install -e ".[server]" --quiet cd frontend && npm ci --ignore-scripts && npm run build && cd .. sudo systemctl restart tono REMOTE diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 95d5ae7..2fedfa9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: cache: npm - name: Install Python dependencies - run: pip install -e ".[dev]" + run: pip install -e ".[server,dev]" - name: Install Node dependencies run: npm install --ignore-scripts diff --git a/README.md b/README.md index cb6941a..6accb62 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Install a local binary from the Releases section, or run locally: ```bash # Installation python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate -pip install -e ".[dev]" +pip install -e ".[server,dev]" npm install # Running the servers @@ -29,29 +29,46 @@ npm run dev # terminal 2 — Vite dev server, open the URL it prints ```bash git clone https://github.com/VIPQualityPost/tono.git && cd tono python -m venv .venv && source .venv/bin/activate -pip install -e . +pip install -e ".[server]" cd frontend && npm ci && npm run build && cd .. TONO_HOST=0.0.0.0 python -m backend.main ``` See [Self-Hosting](docs/self-hosting.md) for reverse proxy setup, environment variables, and configuration. +## Python library + +tono's processing nodes can also be used as a standalone Python library — no server needed: + +```bash +pip install -e . +``` + +```python +import tono + +fields = tono.load("scan.gwy") +leveled = tono.apply("PlaneLevelField", fields[0]) +filtered = tono.apply("GaussianFilter", leveled, sigma=2.0) +``` + +See [Library Usage](docs/library.md) for the full API and more examples. + ## Docs - [Building](docs/building.md) — setup, dev mode, web deployment, and native desktop builds for macOS, Linux, and Windows - [Self-Hosting](docs/self-hosting.md) — deploying tono on a server +- [Library Usage](docs/library.md) — using tono as a Python signal processing library - [Plugins](docs/plugins.md) — writing and uploading custom node plugins - [Testing](docs/testing.md) — running tests and writing new ones -## Project layout +## Project plans -```text -tono/ - backend/ Python server, execution engine, nodes - frontend/ React/Vite app - plugins/ User plugin files (.py) - tests/ Python tests - docs/ Documentation - desktop.py Desktop launcher - scripts/ Build scripts (macOS, Linux, Windows) -``` +- Please help with providing demo files for validating importers! +- Please try making weird workflows to see what breaks or does not flow nicely + +- Adding support for force curves +- Adding general support for spectroscopic data +- Adding general support for spectroscopic volumes + +- Adding more generic numerical operations and visualisations \ No newline at end of file diff --git a/backend/api.py b/backend/api.py new file mode 100644 index 0000000..1395e9a --- /dev/null +++ b/backend/api.py @@ -0,0 +1,233 @@ +""" +Standalone library API for tono signal processing nodes. + +Usage:: + + import tono + + # Load an SPM file + fields = tono.load("scan.gwy") + field = fields[0] + + # Apply a processing node + result = tono.apply("GaussianFilter", field, sigma=3.0) + + # Chain operations + result = tono.apply("PlaneLevelField", field, mode="subtract_mean") + result = tono.apply("GaussianFilter", result, sigma=2.0) + + # Get available nodes + print(tono.nodes()) + + # Direct node access + node = tono.get_node("GaussianFilter") + (filtered,) = node.process(field=field, sigma=3.0) +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from backend.data_types import DataField, LineData, RecordTable, DataTable, MeshModel + + +def _ensure_registry() -> None: + """Import all node modules so @register_node decorators fire.""" + from backend.node_registry import NODE_CLASS_MAPPINGS + if NODE_CLASS_MAPPINGS: + return + import backend.nodes # noqa: F401 — triggers auto-discovery + + +def load(path: str | Path) -> list[DataField]: + """Load an SPM data file and return a list of DataField channels. + + Supported formats: .gwy, .sxm, .ibw, .hdf5, .png, .tif, .jpg, and more. + + Parameters + ---------- + path : str or Path + Path to the data file. + + Returns + ------- + list[DataField] + One DataField per channel in the file. + + Raises + ------ + ValueError + If the file extension is not supported. + """ + from backend.importers import get_importer + + p = Path(path) + ext = p.suffix.lower() + importer = get_importer(ext) + if importer is None: + from backend.importers import all_extensions + supported = ", ".join(sorted(all_extensions())) + raise ValueError( + f"Unsupported file extension {ext!r}. Supported: {supported}" + ) + return importer.load(str(p)) + + +def channel_names(path: str | Path) -> list[str]: + """Return the channel names in a data file without loading the full data.""" + from backend.importers import get_importer + + p = Path(path) + importer = get_importer(p.suffix.lower()) + if importer is None: + raise ValueError(f"Unsupported file extension: {p.suffix!r}") + return importer.channel_names(str(p)) + + +def supported_formats() -> frozenset[str]: + """Return all supported file extensions (e.g. {'.gwy', '.sxm', ...}).""" + from backend.importers import all_extensions + return all_extensions() + + +def nodes() -> list[str]: + """Return a sorted list of all registered node class names.""" + from backend.node_registry import NODE_CLASS_MAPPINGS + _ensure_registry() + return sorted(NODE_CLASS_MAPPINGS.keys()) + + +def get_node(class_name: str) -> Any: + """Return a fresh instance of the named node class. + + Parameters + ---------- + class_name : str + The node class name, e.g. ``"GaussianFilter"``. + + Returns + ------- + object + A node instance. Call its ``process(**kwargs)`` method to run it. + + Raises + ------ + KeyError + If no node with that name is registered. + """ + from backend.node_registry import NODE_CLASS_MAPPINGS + _ensure_registry() + if class_name not in NODE_CLASS_MAPPINGS: + raise KeyError( + f"Unknown node {class_name!r}. " + f"Use tono.nodes() to list available nodes." + ) + return NODE_CLASS_MAPPINGS[class_name]() + + +def apply(class_name: str, *args: Any, **kwargs: Any) -> Any: + """Run a node's process function and return the result. + + Positional arguments are mapped to the node's required inputs in order. + Keyword arguments are passed directly. + + If the node returns a single output, it is unwrapped from the tuple. + If the node returns multiple outputs, the full tuple is returned. + + Parameters + ---------- + class_name : str + The node class name, e.g. ``"GaussianFilter"``. + *args + Positional arguments mapped to required inputs in declaration order. + **kwargs + Keyword arguments passed directly to the node's process function. + + Returns + ------- + DataField or tuple + Single output unwrapped, or tuple of outputs if multiple. + + Examples + -------- + >>> result = tono.apply("GaussianFilter", field, sigma=3.0) + >>> leveled, mask = tono.apply("PlaneLevelField", field, mode="subtract_mean") + """ + from backend.node_registry import NODE_CLASS_MAPPINGS + from backend.execution_context import execution_callbacks, active_node + + _ensure_registry() + if class_name not in NODE_CLASS_MAPPINGS: + raise KeyError( + f"Unknown node {class_name!r}. " + f"Use tono.nodes() to list available nodes." + ) + + cls = NODE_CLASS_MAPPINGS[class_name] + instance = cls() + + # Map positional args to required input names in declaration order + if args: + input_types = cls.INPUT_TYPES() + required_names = list(input_types.get("required", {}).keys()) + for i, arg in enumerate(args): + if i >= len(required_names): + raise TypeError( + f"{class_name} has {len(required_names)} required inputs, " + f"but {len(args)} positional arguments were given" + ) + name = required_names[i] + if name in kwargs: + raise TypeError( + f"{class_name}: input {name!r} specified both as positional " + f"argument and keyword argument" + ) + kwargs[name] = arg + + func = getattr(instance, cls.FUNCTION) + + # Run inside an execution context so emit_* calls are no-ops + with execution_callbacks(), active_node("api"): + result = func(**kwargs) + + if not isinstance(result, tuple): + result = (result,) + + return result[0] if len(result) == 1 else result + + +def field(data: Any, xreal: float = 1e-6, yreal: float = 1e-6, + si_unit_xy: str = "m", si_unit_z: str = "m", **kwargs: Any) -> DataField: + """Create a DataField from a 2D array. + + A convenience wrapper around ``DataField(...)`` with sensible defaults. + + Parameters + ---------- + data : array_like + 2D numpy array or anything convertible to one. + xreal : float + Physical width in metres (default 1 um). + yreal : float + Physical height in metres (default 1 um). + si_unit_xy : str + Lateral unit string (default "m"). + si_unit_z : str + Height/value unit string (default "m"). + **kwargs + Additional DataField keyword arguments. + + Returns + ------- + DataField + """ + import numpy as np + return DataField( + data=np.asarray(data, dtype=np.float64), + xreal=xreal, + yreal=yreal, + si_unit_xy=si_unit_xy, + si_unit_z=si_unit_z, + **kwargs, + ) diff --git a/docs/building.md b/docs/building.md index 8105610..4e655ab 100644 --- a/docs/building.md +++ b/docs/building.md @@ -8,17 +8,17 @@ ### First-time setup ```bash -# Install Python dependencies -pip install -e . +# Install Python dependencies (server extra required for the web app) +pip install -e ".[server]" # Install frontend dependencies npm install # For running tests -pip install -e ".[dev]" +pip install -e ".[server,dev]" # For building desktop executables -pip install -e ".[desktop]" +pip install -e ".[server,desktop]" ``` Using a virtual environment is recommended: @@ -26,7 +26,7 @@ Using a virtual environment is recommended: ```bash python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate -pip install -e ".[dev,desktop]" +pip install -e ".[server,dev,desktop]" ``` --- diff --git a/docs/library.md b/docs/library.md new file mode 100644 index 0000000..d257d4e --- /dev/null +++ b/docs/library.md @@ -0,0 +1,212 @@ +# Using tono as a Python library + +tono's processing nodes can be used as a standalone Python library for scripting, batch processing, and integration into other tools — no web server required. + +## Installation + +```bash +pip install -e . +``` + +This installs the core library with all signal processing dependencies (numpy, scipy, scikit-image, etc.) but **not** the web server (aiohttp). To install the full app with the server, use `pip install -e ".[server]"`. + +## Quick start + +```python +import tono + +# Load an SPM data file +fields = tono.load("scan.gwy") +height = fields[0] + +# Apply a processing node +leveled = tono.apply("PlaneLevelField", height) +filtered = tono.apply("GaussianFilter", leveled, sigma=2.0) + +# Access the raw numpy array +print(filtered.data.shape) # (256, 256) +print(filtered.data.mean()) # height in metres +``` + +## API reference + +### Loading data + +#### `tono.load(path) -> list[DataField]` + +Load an SPM data file. Returns one `DataField` per channel. + +```python +fields = tono.load("scan.gwy") # Gwyddion +fields = tono.load("image.sxm") # Nanonis +fields = tono.load("data.ibw") # Igor Binary Wave +fields = tono.load("scan.hdf5") # HDF5 +fields = tono.load("photo.png") # Standard images +``` + +#### `tono.channel_names(path) -> list[str]` + +Get channel names without loading the full data. + +```python +names = tono.channel_names("scan.gwy") +# ['Height', 'Phase', 'Amplitude'] +``` + +#### `tono.supported_formats() -> frozenset[str]` + +List all supported file extensions. + +### Processing + +#### `tono.apply(node_name, *args, **kwargs)` + +Run a processing node. Positional arguments are mapped to required inputs in declaration order. Returns a single output if the node has one output, or a tuple if it has multiple. + +```python +# Positional: first required input is `field` +result = tono.apply("GaussianFilter", my_field, sigma=3.0) + +# All keyword arguments +result = tono.apply("GaussianFilter", field=my_field, sigma=3.0) + +# Nodes with multiple outputs return a tuple +field_out, mask_out = tono.apply("ThresholdMask", my_field, method="otsu") +``` + +#### `tono.get_node(name) -> node_instance` + +Get a node instance for direct use. This gives full control over the node's `process()` method. + +```python +gauss = tono.get_node("GaussianFilter") +(result,) = gauss.process(field=my_field, sigma=3.0) +``` + +#### `tono.nodes() -> list[str]` + +List all available node class names. + +```python +for name in tono.nodes(): + print(name) +``` + +### Creating data + +#### `tono.field(data, xreal=1e-6, yreal=1e-6, ...) -> DataField` + +Create a `DataField` from a numpy array. + +```python +import numpy as np + +# From a numpy array +f = tono.field(np.random.randn(256, 256)) + +# With physical dimensions (10 um x 10 um scan) +f = tono.field(data, xreal=10e-6, yreal=10e-6, si_unit_z="V") +``` + +### Data types + +#### `DataField` + +The core 2D data container, analogous to Gwyddion's `GwyDataField`. + +| Attribute | Type | Description | +|---|---|---| +| `data` | `np.ndarray` | 2D float64 array (yres x xres) | +| `xres`, `yres` | `int` | Pixel dimensions | +| `xreal`, `yreal` | `float` | Physical dimensions in metres | +| `dx`, `dy` | `float` | Pixel size in metres (property) | +| `si_unit_xy` | `str` | Lateral unit (e.g. `"m"`) | +| `si_unit_z` | `str` | Value unit (e.g. `"m"`, `"V"`, `"A"`) | +| `domain` | `str` | `"spatial"` or `"frequency"` | + +Key methods: +- `field.copy()` — deep copy +- `field.replace(data=..., si_unit_z=...)` — copy with selected fields replaced + +#### Other types + +- `LineData` — 1D data with optional x-axis and units +- `RecordTable` — list of `{quantity, value, unit}` dicts (scalar measurements) +- `DataTable` — list of row dicts (tabular data like grain statistics) + +## Batch processing example + +```python +import tono +from pathlib import Path + +input_dir = Path("scans/") +results = {} + +for path in input_dir.glob("*.gwy"): + fields = tono.load(path) + height = fields[0] + + # Standard processing pipeline + leveled = tono.apply("PlaneLevelField", height) + filtered = tono.apply("GaussianFilter", leveled, sigma=1.5) + + # Extract statistics + stats_node = tono.get_node("Statistics") + (table,) = stats_node.process(field=filtered) + results[path.name] = table + + print(f"{path.name}: processed {height.xres}x{height.yres} " + f"({height.xreal*1e6:.1f} x {height.yreal*1e6:.1f} um)") +``` + +## Integration with matplotlib + +```python +import tono +import matplotlib.pyplot as plt +import numpy as np + +fields = tono.load("scan.gwy") +field = fields[0] + +# Convert physical coordinates for axis labels +extent = [0, field.xreal * 1e6, 0, field.yreal * 1e6] # in micrometres + +fig, axes = plt.subplots(1, 3, figsize=(15, 4)) + +axes[0].imshow(field.data * 1e9, extent=extent, cmap="afmhot") +axes[0].set_title("Raw") +axes[0].set_xlabel("x (um)") +axes[0].set_ylabel("y (um)") + +leveled = tono.apply("PlaneLevelField", field) +axes[1].imshow(leveled.data * 1e9, extent=extent, cmap="afmhot") +axes[1].set_title("Leveled") + +filtered = tono.apply("GaussianFilter", leveled, sigma=2.0) +axes[2].imshow(filtered.data * 1e9, extent=extent, cmap="afmhot") +axes[2].set_title("Filtered") + +for ax in axes: + ax.set_xlabel("x (um)") + +plt.tight_layout() +plt.savefig("pipeline.png", dpi=150) +``` + +## Available nodes + +Use `tono.nodes()` to list all nodes. Major categories include: + +| Category | Example nodes | +|---|---| +| **Level & Correct** | PlaneLevelField, PolyLevelField, FacetLevelField, LineCorrection, DriftCorrection | +| **Filter** | GaussianFilter, MedianFilter, KuwaharaFilter, WaveletDenoise, EdgeDetect | +| **Spectral** | FFT2D, FFT2DInverse, FFTFilter, PSDF, ACF2D, CrossCorrelate | +| **Measure** | Statistics, Histogram, CrossSection, Curvature, FractalDimension | +| **Detect** | FeatureDetection, HoughTransform, TemplateMatch, PixelClassification | +| **Mask** | ThresholdMask, GrainMark, DrawMask, MaskMorphology | +| **Grains** | GrainAnalysis, WatershedSegmentation, GrainDistributions | +| **Geometry** | CropResizeField, RotateField, Resample, AffineCorrection | +| **Tip** | TipModel, TipDeconvolution, BlindTipEstimate | diff --git a/docs/self-hosting.md b/docs/self-hosting.md index de7c92d..969c6ca 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -7,7 +7,7 @@ tono can be self-hosted on any server with Python 3.10+ and Node.js 18+. ```bash git clone https://github.com/VIPQualityPost/tono.git && cd tono python -m venv .venv && source .venv/bin/activate -pip install -e . +pip install -e ".[server]" cd frontend && npm ci && npm run build && cd .. TONO_HOST=0.0.0.0 python -m backend.main ``` @@ -31,7 +31,7 @@ sudo chown tono:tono /var/lib/tono ```bash sudo git clone https://github.com/VIPQualityPost/tono.git /opt/tono sudo chown -R tono:tono /opt/tono -sudo -u tono bash -c 'cd /opt/tono && python3 -m venv .venv && source .venv/bin/activate && pip install -e .' +sudo -u tono bash -c 'cd /opt/tono && python3 -m venv .venv && source .venv/bin/activate && pip install -e ".[server]"' sudo -u tono bash -c 'cd /opt/tono/frontend && npm ci && npm run build' ``` diff --git a/examples/library_usage.py b/examples/library_usage.py new file mode 100644 index 0000000..3e9b89b --- /dev/null +++ b/examples/library_usage.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Example: using tono as a standalone signal processing library. + +Run from the repo root: + python examples/library_usage.py +""" + +import numpy as np +import tono + +# ── 1. Generate a synthetic surface ────────────────────────────────── + +print("Generating synthetic surface...") +surface = tono.apply( + "SyntheticSurface", + pattern="fbm", + xres=256, + yres=256, + xreal=10e-6, # 10 um + yreal=10e-6, + amplitude=50e-9, # 50 nm height range + seed=42, +) +print(f" Shape: {surface.data.shape}") +print(f" Physical size: {surface.xreal*1e6:.1f} x {surface.yreal*1e6:.1f} um") +print(f" Height range: {np.ptp(surface.data)*1e9:.1f} nm") + +# ── 2. Level the surface ───────────────────────────────────────────── + +print("\nLeveling...") +leveled = tono.apply("PlaneLevelField", surface) +print(f" Mean after leveling: {leveled.data.mean()*1e9:.4f} nm") + +# ── 3. Apply a Gaussian filter ─────────────────────────────────────── + +print("\nFiltering...") +filtered = tono.apply("GaussianFilter", leveled, sigma=2.0) +print(f" Height range after filtering: {np.ptp(filtered.data)*1e9:.1f} nm") + +# ── 4. Compute statistics ──────────────────────────────────────────── + +print("\nStatistics:") +stats_node = tono.get_node("Statistics") +(table,) = stats_node.process(field=filtered) +for row in table: + print(f" {row['quantity']}: {row['value']:.6g} {row.get('unit', '')}") + +# ── 5. Edge detection ──────────────────────────────────────────────── + +print("\nEdge detection...") +edges = tono.apply("EdgeDetect", filtered, method="sobel", sigma=1.0) +print(f" Edge map range: [{edges.data.min():.2f}, {edges.data.max():.2f}]") + +# ── 6. Create a DataField from a numpy array ───────────────────────── + +print("\nCreating field from numpy array...") +data = np.sin(np.linspace(0, 4 * np.pi, 128).reshape(1, -1)) * \ + np.cos(np.linspace(0, 6 * np.pi, 128).reshape(-1, 1)) +custom_field = tono.field(data, xreal=5e-6, yreal=5e-6, si_unit_z="V") +print(f" Shape: {custom_field.data.shape}") +print(f" Unit: {custom_field.si_unit_z}") + +# ── 7. FFT analysis ────────────────────────────────────────────────── + +print("\nFFT of the filtered surface...") +fft_log_mag, fft_mag, fft_phase, fft_psdf = tono.apply("FFT2D", filtered, windowing="hann", level="mean") +print(f" FFT shape: {fft_mag.data.shape}") +print(f" Domain: {fft_mag.domain}") + +# ── 8. List available nodes ────────────────────────────────────────── + +all_nodes = tono.nodes() +print(f"\nTotal available nodes: {len(all_nodes)}") +print("First 10:", ", ".join(all_nodes[:10])) + +print("\nDone!") diff --git a/pyproject.toml b/pyproject.toml index 30b66d8..da85fda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ description = "topographical node-based image analysis." readme = "README.md" requires-python = ">=3.10" dependencies = [ - "aiohttp>=3.9,<4", "gwyfile>=0.2", "h5py>=3.10,<4", "igor>=0.3", @@ -22,6 +21,9 @@ dependencies = [ ] [project.optional-dependencies] +server = [ + "aiohttp>=3.9,<4", +] dev = [ "pytest>=8,<9", "pytest-cov>=7,<8", @@ -38,6 +40,9 @@ icons = [ [project.scripts] tono = "backend.main:main" +[tool.setuptools] +py-modules = ["tono"] + [tool.setuptools.packages.find] include = ["backend*"] diff --git a/scripts/build-linux.sh b/scripts/build-linux.sh index 387a088..3095d29 100755 --- a/scripts/build-linux.sh +++ b/scripts/build-linux.sh @@ -28,7 +28,7 @@ echo "Building frontend bundle..." npm run build echo "Installing desktop build dependencies..." -uv pip install -e ".[desktop]" +uv pip install -e ".[server,desktop]" echo "Packaging desktop app with PyInstaller..." $PYTHON -m PyInstaller \ diff --git a/scripts/build-mac.sh b/scripts/build-mac.sh index e325a4d..6f59ad6 100755 --- a/scripts/build-mac.sh +++ b/scripts/build-mac.sh @@ -28,7 +28,7 @@ echo "Building frontend bundle..." npm run build echo "Installing desktop build dependencies..." -uv pip install -e ".[desktop]" +uv pip install -e ".[server,desktop]" echo "Packaging desktop app with PyInstaller..." $PYTHON -m PyInstaller \ diff --git a/scripts/build-windows.ps1 b/scripts/build-windows.ps1 index c9b8524..0152e85 100644 --- a/scripts/build-windows.ps1 +++ b/scripts/build-windows.ps1 @@ -46,7 +46,7 @@ npm run build Assert-LastExitCode "Frontend build" Write-Host "Installing desktop build dependencies..." -& uv pip install -e ".[desktop]" +& uv pip install -e ".[server,desktop]" Assert-LastExitCode "Desktop dependency installation" $pyInstallerArgs = @( diff --git a/tono.py b/tono.py new file mode 100644 index 0000000..f923404 --- /dev/null +++ b/tono.py @@ -0,0 +1,40 @@ +""" +tono — topographical signal processing library. + +This module provides a convenient ``import tono`` entry point for using +tono's processing nodes as a standalone Python library, without running +the web server. + +Quick start:: + + import tono + + # Load SPM data + fields = tono.load("scan.gwy") + + # Process + result = tono.apply("GaussianFilter", fields[0], sigma=3.0) + + # Create a field from a numpy array + import numpy as np + f = tono.field(np.random.randn(256, 256)) + +See ``backend.api`` for the full API documentation. +""" + +from backend.api import ( # noqa: F401 + apply, + channel_names, + field, + get_node, + load, + nodes, + supported_formats, +) +from backend.data_types import ( # noqa: F401 + DataField, + DataTable, + LineData, + MeshModel, + RecordTable, +)