Add runtime environment readiness checks

This commit is contained in:
2026-06-30 14:28:49 +08:00
parent 442b521705
commit d9ea249ff0
12 changed files with 603 additions and 18 deletions

View File

@@ -17,6 +17,7 @@ core, then adds:
Seg_Data_Server_Net/ Seg_Data_Server_Net/
backend/ FastAPI API, job runner, module wrappers backend/ FastAPI API, job runner, module wrappers
frontend/ React + Vite operator UI frontend/ React + Vite operator UI
envs/ conda environment specs for task and MMSeg runtimes
scripts/ helper scripts for running services and syncing weights scripts/ helper scripts for running services and syncing weights
weights/ copied model weights and manifest.json weights/ copied model weights and manifest.json
``` ```
@@ -27,6 +28,9 @@ Seg_Data_Server_Net/
cd Seg_Data_Server_Net cd Seg_Data_Server_Net
cp .env.example .env cp .env.example .env
# Create or repair the two runtime environments, then verify imports.
scripts/bootstrap_conda_envs.sh
# Backend. The deployment env is seg_smp so the API and most task wrappers # Backend. The deployment env is seg_smp so the API and most task wrappers
# share the same segmentation dependency stack. MMSeg jobs default to the # share the same segmentation dependency stack. MMSeg jobs default to the
# separate SEG_MMSEG_CONDA_ENV because full mmcv wheels must match torch/CUDA. # separate SEG_MMSEG_CONDA_ENV because full mmcv wheels must match torch/CUDA.
@@ -59,6 +63,10 @@ The coverage panel calls `GET /api/coverage` and verifies that the user-facing
scripts from the existing `Seg/` workspace are mapped to web jobs. MMSeg scripts from the existing `Seg/` workspace are mapped to web jobs. MMSeg
vendored internals, docs, build outputs, converters, and config templates are vendored internals, docs, build outputs, converters, and config templates are
classified as supporting artifacts rather than direct web actions. classified as supporting artifacts rather than direct web actions.
The runtime panel calls `GET /api/system/readiness` and verifies the conda
imports required for the backend/task environment and the full MMSeg/mmcv
environment. Command-line verification is available with
`PYTHONPATH=backend conda run -n seg_smp python scripts/verify_runtime_envs.py --refresh`.
The same panel can run `POST /api/acceptance/smoke`, a lightweight live smoke The same panel can run `POST /api/acceptance/smoke`, a lightweight live smoke
that creates an upload dataset, uploads a label, downloads it through the that creates an upload dataset, uploads a label, downloads it through the
@@ -81,14 +89,13 @@ Current `seg_smp` uses `mmcv-lite` because no `torch 2.6/cu124` full `mmcv`
wheel is available on this machine and `nvcc` is not installed for source wheel is available on this machine and `nvcc` is not installed for source
builds. A dedicated `seg_mmcv` environment is used for MMSeg tasks and has builds. A dedicated `seg_mmcv` environment is used for MMSeg tasks and has
`torch 2.1.2+cu121`, `mmcv 2.1.0`, `mmsegmentation 1.2.2`, and NumPy 1.26. `torch 2.1.2+cu121`, `mmcv 2.1.0`, `mmsegmentation 1.2.2`, and NumPy 1.26.
If rebuilding the environment, keep these versions aligned: The reproducible specs live in `envs/seg_smp.yml` and `envs/seg_mmcv.yml`;
the bootstrap script uses the same pinned package sources:
```bash ```bash
conda create -n seg_mmcv python=3.10 -y scripts/bootstrap_conda_envs.sh all
conda run -n seg_mmcv python -m pip install -U pip scripts/bootstrap_conda_envs.sh task
conda run -n seg_mmcv python -m pip install torch==2.1.2 torchvision==0.16.2 --index-url https://download.pytorch.org/whl/cu121 scripts/bootstrap_conda_envs.sh mmseg
conda run -n seg_mmcv python -m pip install mmengine==0.10.7 mmsegmentation==1.2.2 'mmcv==2.1.0' -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
conda run -n seg_mmcv python -m pip install 'numpy<2' 'opencv-python<4.12' ftfy regex matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
``` ```
## Weight Sync ## Weight Sync
@@ -149,8 +156,9 @@ PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build
``` ```
The validation agent checks catalog coverage, the `seg_smp` task env, the The validation agent checks catalog coverage, the `seg_smp` task env, the
`seg_mmcv` MMSeg env, GPU visibility, no-weight Git safety, backend tests, `seg_mmcv` MMSeg env, runtime import readiness, GPU visibility, no-weight Git
frontend build, and live backend/frontend endpoints when the services are safety, backend tests, frontend build, and live backend/frontend endpoints
running. With live validation enabled it also runs the lightweight acceptance when the services are running. With live validation enabled it also runs the
smoke above. By default it also runs the deep training acceptance; set lightweight acceptance smoke above. By default it also runs the deep training
`SEG_VALIDATE_DEEP=0` when a quick non-training validation pass is needed. acceptance; set `SEG_VALIDATE_DEEP=0` when a quick non-training validation
pass is needed.

View File

@@ -43,9 +43,13 @@ def evaluate_project() -> dict:
"dataset_quality_ui": "DatasetQuality" in frontend_text and "generateSelectedYoloYaml" in frontend_text, "dataset_quality_ui": "DatasetQuality" in frontend_text and "generateSelectedYoloYaml" in frontend_text,
"loss_result_ui": "loss" in frontend_text.lower() and "heatmap" in frontend_text.lower() and "CurvePanel" in frontend_text, "loss_result_ui": "loss" in frontend_text.lower() and "heatmap" in frontend_text.lower() and "CurvePanel" in frontend_text,
"job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text, "job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text,
"runtime_readiness_ui": "runtimeReadiness" in frontend_text and "环境就绪" in frontend_text,
"dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text, "dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text,
"dataset_quality_api": "/api/datasets/{dataset_name}/validate" in backend_text and "/api/datasets/{dataset_name}/yolo-yaml" in backend_text, "dataset_quality_api": "/api/datasets/{dataset_name}/validate" in backend_text and "/api/datasets/{dataset_name}/yolo-yaml" in backend_text,
"job_progress_api": "progress_from_log_path" in backend_text and '"progress"' in backend_text, "job_progress_api": "progress_from_log_path" in backend_text and '"progress"' in backend_text,
"runtime_readiness_api": "/api/system/readiness" in backend_text,
"runtime_bootstrap_scripts": (settings.project_root / "scripts" / "bootstrap_conda_envs.sh").exists()
and (settings.project_root / "scripts" / "verify_runtime_envs.py").exists(),
"curve_api": "/api/results/curves" in backend_text, "curve_api": "/api/results/curves" in backend_text,
"deep_acceptance_api": "/api/acceptance/deep" in backend_text, "deep_acceptance_api": "/api/acceptance/deep" in backend_text,
"deep_acceptance_ui": "runDeepAcceptance" in frontend_text and "深度训练" in frontend_text, "deep_acceptance_ui": "runDeepAcceptance" in frontend_text and "深度训练" in frontend_text,

View File

@@ -13,7 +13,7 @@ from ..catalog import get_catalog
from ..config import settings from ..config import settings
from ..coverage import get_coverage_report from ..coverage import get_coverage_report
from ..modules.results.service import scan_training_curves from ..modules.results.service import scan_training_curves
from ..modules.system.service import get_conda_envs, get_gpus from ..modules.system.service import get_conda_envs, get_gpus, get_runtime_readiness
from ..modules.weights.service import load_manifest from ..modules.weights.service import load_manifest
from ..progress import parse_progress from ..progress import parse_progress
@@ -64,6 +64,8 @@ def validate_project(run_build: bool = False) -> dict:
env_names = [item["name"] for item in get_conda_envs().get("envs", [])] env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}}) checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}}) checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
runtime_readiness = get_runtime_readiness(force=True)
checks.append({"name": "runtime_env_readiness", "passed": runtime_readiness["passed"], "detail": runtime_readiness})
smoke = _run( smoke = _run(
[ [
@@ -102,6 +104,7 @@ def validate_project(run_build: bool = False) -> dict:
health = _fetch(f"{backend_url}/api/health") health = _fetch(f"{backend_url}/api/health")
datasets = _fetch(f"{backend_url}/api/datasets") datasets = _fetch(f"{backend_url}/api/datasets")
live_jobs = _fetch(f"{backend_url}/api/jobs") live_jobs = _fetch(f"{backend_url}/api/jobs")
live_readiness = _fetch(f"{backend_url}/api/system/readiness")
live_coverage = _fetch(f"{backend_url}/api/coverage") live_coverage = _fetch(f"{backend_url}/api/coverage")
live_curves = _fetch(f"{backend_url}/api/results/curves") live_curves = _fetch(f"{backend_url}/api/results/curves")
frontend = _fetch(frontend_url) frontend = _fetch(frontend_url)
@@ -116,6 +119,11 @@ def validate_project(run_build: bool = False) -> dict:
"passed": live_jobs["passed"] and isinstance(live_job_items, list) and (not live_job_items or "progress" in live_job_items[0]), "passed": live_jobs["passed"] and isinstance(live_job_items, list) and (not live_job_items or "progress" in live_job_items[0]),
"detail": live_jobs, "detail": live_jobs,
}) })
checks.append({
"name": "live_runtime_readiness_api",
"passed": live_readiness["passed"] and '"passed":true' in live_readiness.get("body", "").replace(" ", ""),
"detail": live_readiness,
})
checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage}) checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves}) checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend}) checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})

View File

@@ -15,7 +15,7 @@ from .config import settings
from .coverage import get_coverage_report from .coverage import get_coverage_report
from .jobs import cancel_job, create_job from .jobs import cancel_job, create_job
from .modules.results.service import scan_results, scan_training_curves from .modules.results.service import scan_results, scan_training_curves
from .modules.system.service import disk_usage, get_conda_envs, get_gpus from .modules.system.service import disk_usage, get_conda_envs, get_gpus, get_runtime_readiness
from .modules.dataset.service import create_dataset, generate_yolo_dataset_yaml, list_uploaded_datasets, save_upload, validate_dataset from .modules.dataset.service import create_dataset, generate_yolo_dataset_yaml, list_uploaded_datasets, save_upload, validate_dataset
from .modules.weights.service import load_manifest, sync_weights, verify_weights from .modules.weights.service import load_manifest, sync_weights, verify_weights
from .agents.evaluation_agent import evaluate_project from .agents.evaluation_agent import evaluate_project
@@ -71,6 +71,11 @@ def api_envs() -> dict:
return get_conda_envs() return get_conda_envs()
@app.get("/api/system/readiness")
def api_runtime_readiness(refresh: bool = False) -> dict:
return get_runtime_readiness(force=refresh)
@app.get("/api/catalog") @app.get("/api/catalog")
def api_catalog() -> dict: def api_catalog() -> dict:
return get_catalog() return get_catalog()

View File

@@ -1,11 +1,70 @@
from __future__ import annotations from __future__ import annotations
import json
import shutil import shutil
import subprocess import subprocess
import threading
import time
from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import Any
import fcntl
from ...config import settings from ...config import settings
READINESS_CACHE_SECONDS = 300
_readiness_cache: tuple[float, dict[str, Any]] | None = None
_readiness_thread_lock = threading.Lock()
@contextmanager
def readiness_probe_lock():
settings.project_root.joinpath("var").mkdir(parents=True, exist_ok=True)
lock_path = settings.project_root / "var" / "runtime_readiness.lock"
with _readiness_thread_lock, lock_path.open("w") as lock_file:
fcntl.flock(lock_file, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock_file, fcntl.LOCK_UN)
def runtime_environment_specs() -> list[dict[str, Any]]:
return [
{
"role": "backend_task",
"name": settings.task_conda_env,
"label": "Backend, SegModel, YOLO",
"env_file": "envs/seg_smp.yml",
"required_imports": [
{"module": "fastapi", "package": "fastapi"},
{"module": "uvicorn", "package": "uvicorn"},
{"module": "torch", "package": "torch"},
{"module": "cv2", "package": "opencv-python"},
{"module": "segmentation_models_pytorch", "package": "segmentation-models-pytorch"},
{"module": "ultralytics", "package": "ultralytics"},
{"module": "albumentations", "package": "albumentations"},
{"module": "mmengine", "package": "mmengine"},
{"module": "mmseg", "package": "mmsegmentation"},
],
},
{
"role": "mmseg_full",
"name": settings.mmseg_conda_env,
"label": "MMSeg full mmcv runtime",
"env_file": "envs/seg_mmcv.yml",
"required_imports": [
{"module": "torch", "package": "torch"},
{"module": "cv2", "package": "opencv-python"},
{"module": "mmcv", "package": "mmcv"},
{"module": "mmcv._ext", "package": "mmcv"},
{"module": "mmengine", "package": "mmengine"},
{"module": "mmseg", "package": "mmsegmentation"},
],
},
]
def parse_nvidia_smi_csv(output: str) -> list[dict]: def parse_nvidia_smi_csv(output: str) -> list[dict]:
gpus: list[dict] = [] gpus: list[dict] = []
@@ -63,6 +122,152 @@ def get_conda_envs() -> dict:
return {"available": True, "envs": envs, "task_default": settings.task_conda_env, "mmseg_default": settings.mmseg_conda_env} return {"available": True, "envs": envs, "task_default": settings.task_conda_env, "mmseg_default": settings.mmseg_conda_env}
def probe_code(required_imports: list[dict[str, str]]) -> str:
imports_json = json.dumps(required_imports, ensure_ascii=False)
return f"""
import importlib
import importlib.metadata as metadata
import json
import platform
import sys
required = {imports_json}
checks = []
for item in required:
module_name = item["module"]
package_name = item.get("package") or module_name
try:
module = importlib.import_module(module_name)
version = getattr(module, "__version__", None)
if version is None:
try:
version = metadata.version(package_name)
except Exception:
version = None
checks.append({{
"module": module_name,
"package": package_name,
"passed": True,
"version": str(version) if version is not None else None,
}})
except Exception as exc:
checks.append({{
"module": module_name,
"package": package_name,
"passed": False,
"error": f"{{type(exc).__name__}}: {{exc}}",
}})
extra = {{"python": sys.version.split()[0], "platform": platform.platform()}}
try:
import torch
extra["torch_cuda_available"] = bool(torch.cuda.is_available())
extra["torch_cuda"] = getattr(torch.version, "cuda", None)
except Exception:
pass
print(json.dumps({{"checks": checks, "extra": extra}}, ensure_ascii=False))
"""
def parse_probe_stdout(stdout: str) -> dict[str, Any]:
for line in reversed(stdout.splitlines()):
text = line.strip()
if not text or not text.startswith("{"):
continue
return json.loads(text)
raise ValueError("probe did not emit JSON")
def inspect_conda_env(env_name: str, required_imports: list[dict[str, str]], timeout: int = 45, retries: int = 1) -> dict[str, Any]:
command = ["conda", "run", "-n", env_name, "python", "-c", probe_code(required_imports)]
attempts = []
result: subprocess.CompletedProcess[str] | None = None
for attempt in range(retries + 1):
result = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
attempts.append({"attempt": attempt + 1, "returncode": result.returncode})
if result.returncode == 0 or result.returncode not in {139, -11} or attempt >= retries:
break
time.sleep(2)
if result is None:
raise RuntimeError("probe did not run")
detail = {
"command": command[:4] + ["..."],
"returncode": result.returncode,
"attempts": attempts,
"stdout_tail": result.stdout[-2000:],
"stderr_tail": result.stderr[-2000:],
}
if result.returncode != 0:
return {"passed": False, "checks": [], "extra": {}, "detail": detail}
try:
parsed = parse_probe_stdout(result.stdout)
except Exception as exc:
return {"passed": False, "checks": [], "extra": {}, "detail": {**detail, "error": str(exc)}}
checks = parsed.get("checks", [])
return {
"passed": bool(checks) and all(item.get("passed") for item in checks),
"checks": checks,
"extra": parsed.get("extra", {}),
"detail": detail,
}
def get_runtime_readiness(force: bool = False) -> dict[str, Any]:
global _readiness_cache
now = time.time()
if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
cached = dict(_readiness_cache[1])
cached["cached"] = True
return cached
with readiness_probe_lock():
now = time.time()
if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
cached = dict(_readiness_cache[1])
cached["cached"] = True
return cached
conda = get_conda_envs()
env_paths = {item["name"]: item["path"] for item in conda.get("envs", [])}
envs: list[dict[str, Any]] = []
for spec in runtime_environment_specs():
env_name = spec["name"]
exists = env_name in env_paths
env_report: dict[str, Any] = {
"role": spec["role"],
"name": env_name,
"label": spec["label"],
"env_file": spec["env_file"],
"path": env_paths.get(env_name),
"exists": exists,
"passed": False,
"checks": [],
"extra": {},
}
if exists:
probe = inspect_conda_env(env_name, spec["required_imports"])
env_report.update(probe)
envs.append(env_report)
payload = {
"available": bool(conda.get("available")),
"passed": bool(conda.get("available")) and all(item["passed"] for item in envs),
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)),
"cache_seconds": READINESS_CACHE_SECONDS,
"cached": False,
"envs": envs,
"specs": {
"bootstrap_script": "scripts/bootstrap_conda_envs.sh",
"verify_script": "scripts/verify_runtime_envs.py",
"env_files": [spec["env_file"] for spec in runtime_environment_specs()],
"task_default": settings.task_conda_env,
"mmseg_default": settings.mmseg_conda_env,
},
}
_readiness_cache = (now, payload)
return payload
def disk_usage() -> dict: def disk_usage() -> dict:
usage = shutil.disk_usage(settings.source_root) usage = shutil.disk_usage(settings.source_root)
return { return {

View File

@@ -1,4 +1,5 @@
from app.modules.system.service import parse_nvidia_smi_csv from app.modules.system import service
from app.modules.system.service import get_runtime_readiness, parse_nvidia_smi_csv, parse_probe_stdout
def test_parse_nvidia_smi_csv(): def test_parse_nvidia_smi_csv():
@@ -16,3 +17,57 @@ def test_parse_nvidia_smi_csv():
} }
] ]
def test_parse_probe_stdout_uses_last_json_line():
parsed = parse_probe_stdout(
"warning before json\n"
'{"checks":[{"module":"torch","passed":true,"version":"2.6.0"}],"extra":{"python":"3.11"}}\n'
)
assert parsed["checks"][0]["module"] == "torch"
assert parsed["extra"]["python"] == "3.11"
def test_runtime_readiness_marks_missing_env(monkeypatch):
monkeypatch.setattr(service, "_readiness_cache", None)
monkeypatch.setattr(service, "get_conda_envs", lambda: {"available": True, "envs": []})
readiness = get_runtime_readiness(force=True)
assert readiness["passed"] is False
assert all(not item["exists"] for item in readiness["envs"])
def test_runtime_readiness_aggregates_probe_results(monkeypatch):
monkeypatch.setattr(service, "_readiness_cache", None)
specs = [
{"role": "task", "name": "seg_smp", "label": "task", "env_file": "envs/seg_smp.yml", "required_imports": []},
{"role": "mmseg", "name": "seg_mmcv", "label": "mmseg", "env_file": "envs/seg_mmcv.yml", "required_imports": []},
]
monkeypatch.setattr(service, "runtime_environment_specs", lambda: specs)
monkeypatch.setattr(
service,
"get_conda_envs",
lambda: {
"available": True,
"envs": [
{"name": "seg_smp", "path": "/envs/seg_smp"},
{"name": "seg_mmcv", "path": "/envs/seg_mmcv"},
],
},
)
monkeypatch.setattr(
service,
"inspect_conda_env",
lambda name, imports: {
"passed": True,
"checks": [{"module": "torch", "passed": True}],
"extra": {"python": "3.11"},
},
)
readiness = get_runtime_readiness(force=True)
assert readiness["passed"] is True
assert readiness["envs"][0]["path"] == "/envs/seg_smp"
assert readiness["specs"]["env_files"] == ["envs/seg_smp.yml", "envs/seg_mmcv.yml"]

27
envs/seg_mmcv.yml Normal file
View File

@@ -0,0 +1,27 @@
name: seg_mmcv
channels:
- conda-forge
- defaults
dependencies:
- python=3.10
- pip
- pip:
- --index-url https://download.pytorch.org/whl/cu121
- torch==2.1.2
- torchvision==0.16.2
- --extra-index-url https://pypi.org/simple
- mmengine==0.10.7
- mmsegmentation==1.2.2
- -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
- mmcv==2.1.0
- numpy<2
- opencv-python<4.12
- ftfy
- regex
- matplotlib
- pandas
- scikit-learn
- scipy
- seaborn
- tqdm
- tensorboard

31
envs/seg_smp.yml Normal file
View File

@@ -0,0 +1,31 @@
name: seg_smp
channels:
- conda-forge
- defaults
dependencies:
- python=3.11
- pip
- pip:
- --extra-index-url https://download.pytorch.org/whl/cu124
- fastapi>=0.110
- uvicorn[standard]>=0.27
- pydantic>=2
- python-multipart>=0.0.9
- pytest>=8
- torch==2.6.0
- torchvision==0.21.0
- opencv-python<4.12
- numpy<2
- albumentations
- segmentation-models-pytorch
- ultralytics
- mmengine
- mmsegmentation==1.2.2
- mmcv-lite
- matplotlib
- pandas
- scikit-learn
- scipy
- seaborn
- tqdm
- tensorboard

View File

@@ -157,6 +157,42 @@ type GpuPayload = {
}>; }>;
}; };
type RuntimeCheck = {
module: string;
package?: string;
passed: boolean;
version?: string | null;
error?: string;
};
type RuntimeEnv = {
role: string;
name: string;
label: string;
env_file: string;
path?: string;
exists: boolean;
passed: boolean;
checks: RuntimeCheck[];
extra: Record<string, unknown>;
};
type RuntimeReadinessPayload = {
available: boolean;
passed: boolean;
generated_at: string;
cached: boolean;
cache_seconds: number;
envs: RuntimeEnv[];
specs: {
bootstrap_script: string;
verify_script: string;
env_files: string[];
task_default: string;
mmseg_default: string;
};
};
async function api<T>(path: string, init?: RequestInit): Promise<T> { async function api<T>(path: string, init?: RequestInit): Promise<T> {
const res = await fetch(`${API_BASE}${path}`, { const res = await fetch(`${API_BASE}${path}`, {
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
@@ -229,13 +265,15 @@ function useData() {
const [coverage, setCoverage] = useState<CoveragePayload | null>(null); const [coverage, setCoverage] = useState<CoveragePayload | null>(null);
const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null); const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null);
const [deepAcceptance, setDeepAcceptance] = useState<DeepAcceptancePayload | null>(null); const [deepAcceptance, setDeepAcceptance] = useState<DeepAcceptancePayload | null>(null);
const [runtimeReadiness, setRuntimeReadiness] = useState<RuntimeReadinessPayload | null>(null);
const [error, setError] = useState<string>(""); const [error, setError] = useState<string>("");
async function refresh() { async function refresh() {
try { try {
const [catalogNext, gpusNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([ const [catalogNext, gpusNext, readinessNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([
api<Catalog>("/api/catalog"), api<Catalog>("/api/catalog"),
api<GpuPayload>("/api/system/gpus"), api<GpuPayload>("/api/system/gpus"),
api<RuntimeReadinessPayload>("/api/system/readiness"),
api<Job[]>("/api/jobs"), api<Job[]>("/api/jobs"),
api<ResultItem[]>("/api/results"), api<ResultItem[]>("/api/results"),
api<TrainingCurve[]>("/api/results/curves"), api<TrainingCurve[]>("/api/results/curves"),
@@ -246,6 +284,7 @@ function useData() {
]); ]);
setCatalog(catalogNext); setCatalog(catalogNext);
setGpus(gpusNext); setGpus(gpusNext);
setRuntimeReadiness(readinessNext);
setJobs(jobsNext); setJobs(jobsNext);
setResults(resultsNext.slice(0, 80)); setResults(resultsNext.slice(0, 80));
setCurves(curvesNext.slice(0, 12)); setCurves(curvesNext.slice(0, 12));
@@ -277,7 +316,7 @@ function useData() {
return () => window.clearInterval(timer); return () => window.clearInterval(timer);
}, []); }, []);
return { catalog, gpus, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh }; return { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh };
} }
function StatusPill({ status }: { status: string }) { function StatusPill({ status }: { status: string }) {
@@ -300,7 +339,7 @@ function JobProgressBar({ progress }: { progress?: JobProgress }) {
} }
function App() { function App() {
const { catalog, gpus, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData(); const { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData();
const [taskType, setTaskType] = useState("mock.echo"); const [taskType, setTaskType] = useState("mock.echo");
const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2)); const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2));
const [selectedJob, setSelectedJob] = useState<Job | null>(null); const [selectedJob, setSelectedJob] = useState<Job | null>(null);
@@ -492,6 +531,7 @@ function App() {
<a href="#jobs"><Terminal size={18} /></a> <a href="#jobs"><Terminal size={18} /></a>
<a href="#datasets"><Boxes size={18} /></a> <a href="#datasets"><Boxes size={18} /></a>
<a href="#gpu"><Cpu size={18} />GPU</a> <a href="#gpu"><Cpu size={18} />GPU</a>
<a href="#runtime"><ShieldCheck size={18} /></a>
<a href="#coverage"><ClipboardCheck size={18} /></a> <a href="#coverage"><ClipboardCheck size={18} /></a>
<a href="#weights"><HardDrive size={18} /></a> <a href="#weights"><HardDrive size={18} /></a>
<a href="#results"><BarChart3 size={18} /></a> <a href="#results"><BarChart3 size={18} /></a>
@@ -768,7 +808,7 @@ function App() {
</div> </div>
</section> </section>
<section className="grid three"> <section className="grid four">
<div className="panel" id="gpu"> <div className="panel" id="gpu">
<div className="panelHead"> <div className="panelHead">
<div> <div>
@@ -789,6 +829,37 @@ function App() {
))} ))}
</div> </div>
<div className="panel" id="runtime">
<div className="panelHead">
<div>
<p className="eyebrow">Runtime</p>
<h2></h2>
</div>
<ShieldCheck size={22} />
</div>
<div className="envList">
{(runtimeReadiness?.envs ?? []).map((env) => (
<div key={env.role} className={`envCard ${env.passed ? "ok" : "bad"}`}>
<div className="envHead">
<div>
<strong>{env.name}</strong>
<small>{env.label}</small>
</div>
<span>{env.passed ? "READY" : env.exists ? "CHECK" : "MISSING"}</span>
</div>
<div className="envChecks">
{env.checks.slice(0, 8).map((check) => (
<span key={check.module} className={check.passed ? "ok" : "bad"} title={check.error ?? check.package}>
{check.module}{check.version ? ` ${check.version}` : ""}
</span>
))}
</div>
</div>
))}
</div>
<p className="muted">{runtimeReadiness?.passed ? "runtime imports ready" : "run scripts/bootstrap_conda_envs.sh"} · {runtimeReadiness?.generated_at ?? "not checked"}</p>
</div>
<div className="panel" id="weights"> <div className="panel" id="weights">
<div className="panelHead"> <div className="panelHead">
<div> <div>

View File

@@ -219,6 +219,11 @@ h2 {
margin-bottom: 16px; margin-bottom: 16px;
} }
.grid.four {
grid-template-columns: repeat(4, minmax(0, 1fr));
margin-bottom: 16px;
}
.panel { .panel {
padding: 18px; padding: 18px;
min-width: 0; min-width: 0;
@@ -748,6 +753,87 @@ meter {
accent-color: var(--green); accent-color: var(--green);
} }
.envList {
display: grid;
gap: 10px;
}
.envCard {
min-width: 0;
display: grid;
gap: 8px;
padding: 10px;
border-radius: 7px;
border: 1px solid var(--line);
background: #101310;
}
.envCard.ok {
border-color: rgba(157, 226, 111, 0.32);
}
.envCard.bad {
border-color: rgba(240, 113, 103, 0.55);
}
.envHead {
min-width: 0;
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 8px;
align-items: start;
}
.envHead strong,
.envHead small {
display: block;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.envHead small {
color: var(--muted);
margin-top: 2px;
}
.envHead > span {
color: var(--green);
font-size: 11px;
font-weight: 760;
}
.envCard.bad .envHead > span {
color: var(--red);
}
.envChecks {
display: flex;
flex-wrap: wrap;
gap: 5px;
}
.envChecks span {
max-width: 100%;
padding: 4px 6px;
border-radius: 5px;
border: 1px solid rgba(238, 242, 232, 0.1);
color: var(--muted);
background: #080a08;
font-size: 11px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.envChecks span.ok {
color: var(--green);
}
.envChecks span.bad {
color: var(--red);
}
.bigNumber { .bigNumber {
font-size: 54px; font-size: 54px;
font-weight: 760; font-weight: 760;

58
scripts/bootstrap_conda_envs.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
TASK_ENV="${SEG_TASK_CONDA_ENV:-seg_smp}"
MMSEG_ENV="${SEG_MMSEG_CONDA_ENV:-seg_mmcv}"
env_exists() {
conda env list | awk '{print $1}' | grep -Fxq "$1"
}
create_task_env() {
if ! env_exists "${TASK_ENV}"; then
conda create -n "${TASK_ENV}" python=3.11 -y
fi
conda run -n "${TASK_ENV}" python -m pip install -U pip
conda run -n "${TASK_ENV}" python -m pip install -r "${ROOT_DIR}/backend/requirements.txt"
conda run -n "${TASK_ENV}" python -m pip install \
torch==2.6.0 torchvision==0.21.0 \
'numpy<2' 'opencv-python<4.12' albumentations segmentation-models-pytorch ultralytics \
mmengine mmsegmentation==1.2.2 mmcv-lite \
matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
}
create_mmseg_env() {
if ! env_exists "${MMSEG_ENV}"; then
conda create -n "${MMSEG_ENV}" python=3.10 -y
fi
conda run -n "${MMSEG_ENV}" python -m pip install -U pip
conda run -n "${MMSEG_ENV}" python -m pip install \
torch==2.1.2 torchvision==0.16.2 \
--index-url https://download.pytorch.org/whl/cu121
conda run -n "${MMSEG_ENV}" python -m pip install \
mmengine==0.10.7 mmsegmentation==1.2.2 'mmcv==2.1.0' \
-f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
conda run -n "${MMSEG_ENV}" python -m pip install \
'numpy<2' 'opencv-python<4.12' ftfy regex matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
}
case "${1:-all}" in
all)
create_task_env
create_mmseg_env
PYTHONPATH="${ROOT_DIR}/backend" conda run -n "${TASK_ENV}" python "${ROOT_DIR}/scripts/verify_runtime_envs.py" --refresh
;;
task)
create_task_env
echo "Created or repaired ${TASK_ENV}. Run '$0 all' for full runtime verification."
;;
mmseg)
create_mmseg_env
echo "Created or repaired ${MMSEG_ENV}. Run '$0 all' for full runtime verification."
;;
*)
echo "usage: $0 [all|task|mmseg]" >&2
exit 2
;;
esac

27
scripts/verify_runtime_envs.py Executable file
View File

@@ -0,0 +1,27 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from app.modules.system.service import get_runtime_readiness # noqa: E402
def main() -> None:
parser = argparse.ArgumentParser(description="Verify Seg Data Server runtime conda environments.")
parser.add_argument("--refresh", action="store_true", help="ignore the backend readiness cache")
args = parser.parse_args()
report = get_runtime_readiness(force=args.refresh)
print(json.dumps(report, ensure_ascii=False, indent=2))
if not report.get("passed"):
raise SystemExit(1)
if __name__ == "__main__":
main()