Add runtime environment readiness checks
This commit is contained in:
30
README.md
30
README.md
@@ -17,6 +17,7 @@ core, then adds:
|
||||
Seg_Data_Server_Net/
|
||||
backend/ FastAPI API, job runner, module wrappers
|
||||
frontend/ React + Vite operator UI
|
||||
envs/ conda environment specs for task and MMSeg runtimes
|
||||
scripts/ helper scripts for running services and syncing weights
|
||||
weights/ copied model weights and manifest.json
|
||||
```
|
||||
@@ -27,6 +28,9 @@ Seg_Data_Server_Net/
|
||||
cd Seg_Data_Server_Net
|
||||
cp .env.example .env
|
||||
|
||||
# Create or repair the two runtime environments, then verify imports.
|
||||
scripts/bootstrap_conda_envs.sh
|
||||
|
||||
# Backend. The deployment env is seg_smp so the API and most task wrappers
|
||||
# share the same segmentation dependency stack. MMSeg jobs default to the
|
||||
# separate SEG_MMSEG_CONDA_ENV because full mmcv wheels must match torch/CUDA.
|
||||
@@ -59,6 +63,10 @@ The coverage panel calls `GET /api/coverage` and verifies that the user-facing
|
||||
scripts from the existing `Seg/` workspace are mapped to web jobs. MMSeg
|
||||
vendored internals, docs, build outputs, converters, and config templates are
|
||||
classified as supporting artifacts rather than direct web actions.
|
||||
The runtime panel calls `GET /api/system/readiness` and verifies the conda
|
||||
imports required for the backend/task environment and the full MMSeg/mmcv
|
||||
environment. Command-line verification is available with
|
||||
`PYTHONPATH=backend conda run -n seg_smp python scripts/verify_runtime_envs.py --refresh`.
|
||||
|
||||
The same panel can run `POST /api/acceptance/smoke`, a lightweight live smoke
|
||||
that creates an upload dataset, uploads a label, downloads it through the
|
||||
@@ -81,14 +89,13 @@ Current `seg_smp` uses `mmcv-lite` because no `torch 2.6/cu124` full `mmcv`
|
||||
wheel is available on this machine and `nvcc` is not installed for source
|
||||
builds. A dedicated `seg_mmcv` environment is used for MMSeg tasks and has
|
||||
`torch 2.1.2+cu121`, `mmcv 2.1.0`, `mmsegmentation 1.2.2`, and NumPy 1.26.
|
||||
If rebuilding the environment, keep these versions aligned:
|
||||
The reproducible specs live in `envs/seg_smp.yml` and `envs/seg_mmcv.yml`;
|
||||
the bootstrap script uses the same pinned package sources:
|
||||
|
||||
```bash
|
||||
conda create -n seg_mmcv python=3.10 -y
|
||||
conda run -n seg_mmcv python -m pip install -U pip
|
||||
conda run -n seg_mmcv python -m pip install torch==2.1.2 torchvision==0.16.2 --index-url https://download.pytorch.org/whl/cu121
|
||||
conda run -n seg_mmcv python -m pip install mmengine==0.10.7 mmsegmentation==1.2.2 'mmcv==2.1.0' -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
|
||||
conda run -n seg_mmcv python -m pip install 'numpy<2' 'opencv-python<4.12' ftfy regex matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
|
||||
scripts/bootstrap_conda_envs.sh all
|
||||
scripts/bootstrap_conda_envs.sh task
|
||||
scripts/bootstrap_conda_envs.sh mmseg
|
||||
```
|
||||
|
||||
## Weight Sync
|
||||
@@ -149,8 +156,9 @@ PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build
|
||||
```
|
||||
|
||||
The validation agent checks catalog coverage, the `seg_smp` task env, the
|
||||
`seg_mmcv` MMSeg env, GPU visibility, no-weight Git safety, backend tests,
|
||||
frontend build, and live backend/frontend endpoints when the services are
|
||||
running. With live validation enabled it also runs the lightweight acceptance
|
||||
smoke above. By default it also runs the deep training acceptance; set
|
||||
`SEG_VALIDATE_DEEP=0` when a quick non-training validation pass is needed.
|
||||
`seg_mmcv` MMSeg env, runtime import readiness, GPU visibility, no-weight Git
|
||||
safety, backend tests, frontend build, and live backend/frontend endpoints
|
||||
when the services are running. With live validation enabled it also runs the
|
||||
lightweight acceptance smoke above. By default it also runs the deep training
|
||||
acceptance; set `SEG_VALIDATE_DEEP=0` when a quick non-training validation
|
||||
pass is needed.
|
||||
|
||||
@@ -43,9 +43,13 @@ def evaluate_project() -> dict:
|
||||
"dataset_quality_ui": "DatasetQuality" in frontend_text and "generateSelectedYoloYaml" in frontend_text,
|
||||
"loss_result_ui": "loss" in frontend_text.lower() and "heatmap" in frontend_text.lower() and "CurvePanel" in frontend_text,
|
||||
"job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text,
|
||||
"runtime_readiness_ui": "runtimeReadiness" in frontend_text and "环境就绪" in frontend_text,
|
||||
"dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text,
|
||||
"dataset_quality_api": "/api/datasets/{dataset_name}/validate" in backend_text and "/api/datasets/{dataset_name}/yolo-yaml" in backend_text,
|
||||
"job_progress_api": "progress_from_log_path" in backend_text and '"progress"' in backend_text,
|
||||
"runtime_readiness_api": "/api/system/readiness" in backend_text,
|
||||
"runtime_bootstrap_scripts": (settings.project_root / "scripts" / "bootstrap_conda_envs.sh").exists()
|
||||
and (settings.project_root / "scripts" / "verify_runtime_envs.py").exists(),
|
||||
"curve_api": "/api/results/curves" in backend_text,
|
||||
"deep_acceptance_api": "/api/acceptance/deep" in backend_text,
|
||||
"deep_acceptance_ui": "runDeepAcceptance" in frontend_text and "深度训练" in frontend_text,
|
||||
|
||||
@@ -13,7 +13,7 @@ from ..catalog import get_catalog
|
||||
from ..config import settings
|
||||
from ..coverage import get_coverage_report
|
||||
from ..modules.results.service import scan_training_curves
|
||||
from ..modules.system.service import get_conda_envs, get_gpus
|
||||
from ..modules.system.service import get_conda_envs, get_gpus, get_runtime_readiness
|
||||
from ..modules.weights.service import load_manifest
|
||||
from ..progress import parse_progress
|
||||
|
||||
@@ -64,6 +64,8 @@ def validate_project(run_build: bool = False) -> dict:
|
||||
env_names = [item["name"] for item in get_conda_envs().get("envs", [])]
|
||||
checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}})
|
||||
checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}})
|
||||
runtime_readiness = get_runtime_readiness(force=True)
|
||||
checks.append({"name": "runtime_env_readiness", "passed": runtime_readiness["passed"], "detail": runtime_readiness})
|
||||
|
||||
smoke = _run(
|
||||
[
|
||||
@@ -102,6 +104,7 @@ def validate_project(run_build: bool = False) -> dict:
|
||||
health = _fetch(f"{backend_url}/api/health")
|
||||
datasets = _fetch(f"{backend_url}/api/datasets")
|
||||
live_jobs = _fetch(f"{backend_url}/api/jobs")
|
||||
live_readiness = _fetch(f"{backend_url}/api/system/readiness")
|
||||
live_coverage = _fetch(f"{backend_url}/api/coverage")
|
||||
live_curves = _fetch(f"{backend_url}/api/results/curves")
|
||||
frontend = _fetch(frontend_url)
|
||||
@@ -116,6 +119,11 @@ def validate_project(run_build: bool = False) -> dict:
|
||||
"passed": live_jobs["passed"] and isinstance(live_job_items, list) and (not live_job_items or "progress" in live_job_items[0]),
|
||||
"detail": live_jobs,
|
||||
})
|
||||
checks.append({
|
||||
"name": "live_runtime_readiness_api",
|
||||
"passed": live_readiness["passed"] and '"passed":true' in live_readiness.get("body", "").replace(" ", ""),
|
||||
"detail": live_readiness,
|
||||
})
|
||||
checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
|
||||
checks.append({"name": "live_training_curves_api", "passed": live_curves["passed"] and live_curves.get("body", "").lstrip().startswith("["), "detail": live_curves})
|
||||
checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})
|
||||
|
||||
@@ -15,7 +15,7 @@ from .config import settings
|
||||
from .coverage import get_coverage_report
|
||||
from .jobs import cancel_job, create_job
|
||||
from .modules.results.service import scan_results, scan_training_curves
|
||||
from .modules.system.service import disk_usage, get_conda_envs, get_gpus
|
||||
from .modules.system.service import disk_usage, get_conda_envs, get_gpus, get_runtime_readiness
|
||||
from .modules.dataset.service import create_dataset, generate_yolo_dataset_yaml, list_uploaded_datasets, save_upload, validate_dataset
|
||||
from .modules.weights.service import load_manifest, sync_weights, verify_weights
|
||||
from .agents.evaluation_agent import evaluate_project
|
||||
@@ -71,6 +71,11 @@ def api_envs() -> dict:
|
||||
return get_conda_envs()
|
||||
|
||||
|
||||
@app.get("/api/system/readiness")
|
||||
def api_runtime_readiness(refresh: bool = False) -> dict:
|
||||
return get_runtime_readiness(force=refresh)
|
||||
|
||||
|
||||
@app.get("/api/catalog")
|
||||
def api_catalog() -> dict:
|
||||
return get_catalog()
|
||||
|
||||
@@ -1,11 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import fcntl
|
||||
|
||||
from ...config import settings
|
||||
|
||||
READINESS_CACHE_SECONDS = 300
|
||||
_readiness_cache: tuple[float, dict[str, Any]] | None = None
|
||||
_readiness_thread_lock = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def readiness_probe_lock():
|
||||
settings.project_root.joinpath("var").mkdir(parents=True, exist_ok=True)
|
||||
lock_path = settings.project_root / "var" / "runtime_readiness.lock"
|
||||
with _readiness_thread_lock, lock_path.open("w") as lock_file:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def runtime_environment_specs() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"role": "backend_task",
|
||||
"name": settings.task_conda_env,
|
||||
"label": "Backend, SegModel, YOLO",
|
||||
"env_file": "envs/seg_smp.yml",
|
||||
"required_imports": [
|
||||
{"module": "fastapi", "package": "fastapi"},
|
||||
{"module": "uvicorn", "package": "uvicorn"},
|
||||
{"module": "torch", "package": "torch"},
|
||||
{"module": "cv2", "package": "opencv-python"},
|
||||
{"module": "segmentation_models_pytorch", "package": "segmentation-models-pytorch"},
|
||||
{"module": "ultralytics", "package": "ultralytics"},
|
||||
{"module": "albumentations", "package": "albumentations"},
|
||||
{"module": "mmengine", "package": "mmengine"},
|
||||
{"module": "mmseg", "package": "mmsegmentation"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "mmseg_full",
|
||||
"name": settings.mmseg_conda_env,
|
||||
"label": "MMSeg full mmcv runtime",
|
||||
"env_file": "envs/seg_mmcv.yml",
|
||||
"required_imports": [
|
||||
{"module": "torch", "package": "torch"},
|
||||
{"module": "cv2", "package": "opencv-python"},
|
||||
{"module": "mmcv", "package": "mmcv"},
|
||||
{"module": "mmcv._ext", "package": "mmcv"},
|
||||
{"module": "mmengine", "package": "mmengine"},
|
||||
{"module": "mmseg", "package": "mmsegmentation"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def parse_nvidia_smi_csv(output: str) -> list[dict]:
|
||||
gpus: list[dict] = []
|
||||
@@ -63,6 +122,152 @@ def get_conda_envs() -> dict:
|
||||
return {"available": True, "envs": envs, "task_default": settings.task_conda_env, "mmseg_default": settings.mmseg_conda_env}
|
||||
|
||||
|
||||
def probe_code(required_imports: list[dict[str, str]]) -> str:
|
||||
imports_json = json.dumps(required_imports, ensure_ascii=False)
|
||||
return f"""
|
||||
import importlib
|
||||
import importlib.metadata as metadata
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
|
||||
required = {imports_json}
|
||||
checks = []
|
||||
for item in required:
|
||||
module_name = item["module"]
|
||||
package_name = item.get("package") or module_name
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
version = getattr(module, "__version__", None)
|
||||
if version is None:
|
||||
try:
|
||||
version = metadata.version(package_name)
|
||||
except Exception:
|
||||
version = None
|
||||
checks.append({{
|
||||
"module": module_name,
|
||||
"package": package_name,
|
||||
"passed": True,
|
||||
"version": str(version) if version is not None else None,
|
||||
}})
|
||||
except Exception as exc:
|
||||
checks.append({{
|
||||
"module": module_name,
|
||||
"package": package_name,
|
||||
"passed": False,
|
||||
"error": f"{{type(exc).__name__}}: {{exc}}",
|
||||
}})
|
||||
|
||||
extra = {{"python": sys.version.split()[0], "platform": platform.platform()}}
|
||||
try:
|
||||
import torch
|
||||
extra["torch_cuda_available"] = bool(torch.cuda.is_available())
|
||||
extra["torch_cuda"] = getattr(torch.version, "cuda", None)
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps({{"checks": checks, "extra": extra}}, ensure_ascii=False))
|
||||
"""
|
||||
|
||||
|
||||
def parse_probe_stdout(stdout: str) -> dict[str, Any]:
|
||||
for line in reversed(stdout.splitlines()):
|
||||
text = line.strip()
|
||||
if not text or not text.startswith("{"):
|
||||
continue
|
||||
return json.loads(text)
|
||||
raise ValueError("probe did not emit JSON")
|
||||
|
||||
|
||||
def inspect_conda_env(env_name: str, required_imports: list[dict[str, str]], timeout: int = 45, retries: int = 1) -> dict[str, Any]:
|
||||
command = ["conda", "run", "-n", env_name, "python", "-c", probe_code(required_imports)]
|
||||
attempts = []
|
||||
result: subprocess.CompletedProcess[str] | None = None
|
||||
for attempt in range(retries + 1):
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
attempts.append({"attempt": attempt + 1, "returncode": result.returncode})
|
||||
if result.returncode == 0 or result.returncode not in {139, -11} or attempt >= retries:
|
||||
break
|
||||
time.sleep(2)
|
||||
if result is None:
|
||||
raise RuntimeError("probe did not run")
|
||||
detail = {
|
||||
"command": command[:4] + ["..."],
|
||||
"returncode": result.returncode,
|
||||
"attempts": attempts,
|
||||
"stdout_tail": result.stdout[-2000:],
|
||||
"stderr_tail": result.stderr[-2000:],
|
||||
}
|
||||
if result.returncode != 0:
|
||||
return {"passed": False, "checks": [], "extra": {}, "detail": detail}
|
||||
try:
|
||||
parsed = parse_probe_stdout(result.stdout)
|
||||
except Exception as exc:
|
||||
return {"passed": False, "checks": [], "extra": {}, "detail": {**detail, "error": str(exc)}}
|
||||
checks = parsed.get("checks", [])
|
||||
return {
|
||||
"passed": bool(checks) and all(item.get("passed") for item in checks),
|
||||
"checks": checks,
|
||||
"extra": parsed.get("extra", {}),
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
|
||||
def get_runtime_readiness(force: bool = False) -> dict[str, Any]:
|
||||
global _readiness_cache
|
||||
now = time.time()
|
||||
if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
|
||||
cached = dict(_readiness_cache[1])
|
||||
cached["cached"] = True
|
||||
return cached
|
||||
|
||||
with readiness_probe_lock():
|
||||
now = time.time()
|
||||
if not force and _readiness_cache and now - _readiness_cache[0] < READINESS_CACHE_SECONDS:
|
||||
cached = dict(_readiness_cache[1])
|
||||
cached["cached"] = True
|
||||
return cached
|
||||
|
||||
conda = get_conda_envs()
|
||||
env_paths = {item["name"]: item["path"] for item in conda.get("envs", [])}
|
||||
envs: list[dict[str, Any]] = []
|
||||
for spec in runtime_environment_specs():
|
||||
env_name = spec["name"]
|
||||
exists = env_name in env_paths
|
||||
env_report: dict[str, Any] = {
|
||||
"role": spec["role"],
|
||||
"name": env_name,
|
||||
"label": spec["label"],
|
||||
"env_file": spec["env_file"],
|
||||
"path": env_paths.get(env_name),
|
||||
"exists": exists,
|
||||
"passed": False,
|
||||
"checks": [],
|
||||
"extra": {},
|
||||
}
|
||||
if exists:
|
||||
probe = inspect_conda_env(env_name, spec["required_imports"])
|
||||
env_report.update(probe)
|
||||
envs.append(env_report)
|
||||
|
||||
payload = {
|
||||
"available": bool(conda.get("available")),
|
||||
"passed": bool(conda.get("available")) and all(item["passed"] for item in envs),
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now)),
|
||||
"cache_seconds": READINESS_CACHE_SECONDS,
|
||||
"cached": False,
|
||||
"envs": envs,
|
||||
"specs": {
|
||||
"bootstrap_script": "scripts/bootstrap_conda_envs.sh",
|
||||
"verify_script": "scripts/verify_runtime_envs.py",
|
||||
"env_files": [spec["env_file"] for spec in runtime_environment_specs()],
|
||||
"task_default": settings.task_conda_env,
|
||||
"mmseg_default": settings.mmseg_conda_env,
|
||||
},
|
||||
}
|
||||
_readiness_cache = (now, payload)
|
||||
return payload
|
||||
|
||||
|
||||
def disk_usage() -> dict:
|
||||
usage = shutil.disk_usage(settings.source_root)
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from app.modules.system.service import parse_nvidia_smi_csv
|
||||
from app.modules.system import service
|
||||
from app.modules.system.service import get_runtime_readiness, parse_nvidia_smi_csv, parse_probe_stdout
|
||||
|
||||
|
||||
def test_parse_nvidia_smi_csv():
|
||||
@@ -16,3 +17,57 @@ def test_parse_nvidia_smi_csv():
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_parse_probe_stdout_uses_last_json_line():
|
||||
parsed = parse_probe_stdout(
|
||||
"warning before json\n"
|
||||
'{"checks":[{"module":"torch","passed":true,"version":"2.6.0"}],"extra":{"python":"3.11"}}\n'
|
||||
)
|
||||
|
||||
assert parsed["checks"][0]["module"] == "torch"
|
||||
assert parsed["extra"]["python"] == "3.11"
|
||||
|
||||
|
||||
def test_runtime_readiness_marks_missing_env(monkeypatch):
|
||||
monkeypatch.setattr(service, "_readiness_cache", None)
|
||||
monkeypatch.setattr(service, "get_conda_envs", lambda: {"available": True, "envs": []})
|
||||
|
||||
readiness = get_runtime_readiness(force=True)
|
||||
|
||||
assert readiness["passed"] is False
|
||||
assert all(not item["exists"] for item in readiness["envs"])
|
||||
|
||||
|
||||
def test_runtime_readiness_aggregates_probe_results(monkeypatch):
|
||||
monkeypatch.setattr(service, "_readiness_cache", None)
|
||||
specs = [
|
||||
{"role": "task", "name": "seg_smp", "label": "task", "env_file": "envs/seg_smp.yml", "required_imports": []},
|
||||
{"role": "mmseg", "name": "seg_mmcv", "label": "mmseg", "env_file": "envs/seg_mmcv.yml", "required_imports": []},
|
||||
]
|
||||
monkeypatch.setattr(service, "runtime_environment_specs", lambda: specs)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"get_conda_envs",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"envs": [
|
||||
{"name": "seg_smp", "path": "/envs/seg_smp"},
|
||||
{"name": "seg_mmcv", "path": "/envs/seg_mmcv"},
|
||||
],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
service,
|
||||
"inspect_conda_env",
|
||||
lambda name, imports: {
|
||||
"passed": True,
|
||||
"checks": [{"module": "torch", "passed": True}],
|
||||
"extra": {"python": "3.11"},
|
||||
},
|
||||
)
|
||||
|
||||
readiness = get_runtime_readiness(force=True)
|
||||
|
||||
assert readiness["passed"] is True
|
||||
assert readiness["envs"][0]["path"] == "/envs/seg_smp"
|
||||
assert readiness["specs"]["env_files"] == ["envs/seg_smp.yml", "envs/seg_mmcv.yml"]
|
||||
|
||||
27
envs/seg_mmcv.yml
Normal file
27
envs/seg_mmcv.yml
Normal file
@@ -0,0 +1,27 @@
|
||||
name: seg_mmcv
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- python=3.10
|
||||
- pip
|
||||
- pip:
|
||||
- --index-url https://download.pytorch.org/whl/cu121
|
||||
- torch==2.1.2
|
||||
- torchvision==0.16.2
|
||||
- --extra-index-url https://pypi.org/simple
|
||||
- mmengine==0.10.7
|
||||
- mmsegmentation==1.2.2
|
||||
- -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
|
||||
- mmcv==2.1.0
|
||||
- numpy<2
|
||||
- opencv-python<4.12
|
||||
- ftfy
|
||||
- regex
|
||||
- matplotlib
|
||||
- pandas
|
||||
- scikit-learn
|
||||
- scipy
|
||||
- seaborn
|
||||
- tqdm
|
||||
- tensorboard
|
||||
31
envs/seg_smp.yml
Normal file
31
envs/seg_smp.yml
Normal file
@@ -0,0 +1,31 @@
|
||||
name: seg_smp
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- python=3.11
|
||||
- pip
|
||||
- pip:
|
||||
- --extra-index-url https://download.pytorch.org/whl/cu124
|
||||
- fastapi>=0.110
|
||||
- uvicorn[standard]>=0.27
|
||||
- pydantic>=2
|
||||
- python-multipart>=0.0.9
|
||||
- pytest>=8
|
||||
- torch==2.6.0
|
||||
- torchvision==0.21.0
|
||||
- opencv-python<4.12
|
||||
- numpy<2
|
||||
- albumentations
|
||||
- segmentation-models-pytorch
|
||||
- ultralytics
|
||||
- mmengine
|
||||
- mmsegmentation==1.2.2
|
||||
- mmcv-lite
|
||||
- matplotlib
|
||||
- pandas
|
||||
- scikit-learn
|
||||
- scipy
|
||||
- seaborn
|
||||
- tqdm
|
||||
- tensorboard
|
||||
@@ -157,6 +157,42 @@ type GpuPayload = {
|
||||
}>;
|
||||
};
|
||||
|
||||
type RuntimeCheck = {
|
||||
module: string;
|
||||
package?: string;
|
||||
passed: boolean;
|
||||
version?: string | null;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
type RuntimeEnv = {
|
||||
role: string;
|
||||
name: string;
|
||||
label: string;
|
||||
env_file: string;
|
||||
path?: string;
|
||||
exists: boolean;
|
||||
passed: boolean;
|
||||
checks: RuntimeCheck[];
|
||||
extra: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type RuntimeReadinessPayload = {
|
||||
available: boolean;
|
||||
passed: boolean;
|
||||
generated_at: string;
|
||||
cached: boolean;
|
||||
cache_seconds: number;
|
||||
envs: RuntimeEnv[];
|
||||
specs: {
|
||||
bootstrap_script: string;
|
||||
verify_script: string;
|
||||
env_files: string[];
|
||||
task_default: string;
|
||||
mmseg_default: string;
|
||||
};
|
||||
};
|
||||
|
||||
async function api<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(`${API_BASE}${path}`, {
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@@ -229,13 +265,15 @@ function useData() {
|
||||
const [coverage, setCoverage] = useState<CoveragePayload | null>(null);
|
||||
const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null);
|
||||
const [deepAcceptance, setDeepAcceptance] = useState<DeepAcceptancePayload | null>(null);
|
||||
const [runtimeReadiness, setRuntimeReadiness] = useState<RuntimeReadinessPayload | null>(null);
|
||||
const [error, setError] = useState<string>("");
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
const [catalogNext, gpusNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([
|
||||
const [catalogNext, gpusNext, readinessNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext] = await Promise.all([
|
||||
api<Catalog>("/api/catalog"),
|
||||
api<GpuPayload>("/api/system/gpus"),
|
||||
api<RuntimeReadinessPayload>("/api/system/readiness"),
|
||||
api<Job[]>("/api/jobs"),
|
||||
api<ResultItem[]>("/api/results"),
|
||||
api<TrainingCurve[]>("/api/results/curves"),
|
||||
@@ -246,6 +284,7 @@ function useData() {
|
||||
]);
|
||||
setCatalog(catalogNext);
|
||||
setGpus(gpusNext);
|
||||
setRuntimeReadiness(readinessNext);
|
||||
setJobs(jobsNext);
|
||||
setResults(resultsNext.slice(0, 80));
|
||||
setCurves(curvesNext.slice(0, 12));
|
||||
@@ -277,7 +316,7 @@ function useData() {
|
||||
return () => window.clearInterval(timer);
|
||||
}, []);
|
||||
|
||||
return { catalog, gpus, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh };
|
||||
return { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh };
|
||||
}
|
||||
|
||||
function StatusPill({ status }: { status: string }) {
|
||||
@@ -300,7 +339,7 @@ function JobProgressBar({ progress }: { progress?: JobProgress }) {
|
||||
}
|
||||
|
||||
function App() {
|
||||
const { catalog, gpus, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData();
|
||||
const { catalog, gpus, runtimeReadiness, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData();
|
||||
const [taskType, setTaskType] = useState("mock.echo");
|
||||
const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2));
|
||||
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
|
||||
@@ -492,6 +531,7 @@ function App() {
|
||||
<a href="#jobs"><Terminal size={18} />任务</a>
|
||||
<a href="#datasets"><Boxes size={18} />数据集</a>
|
||||
<a href="#gpu"><Cpu size={18} />GPU</a>
|
||||
<a href="#runtime"><ShieldCheck size={18} />环境</a>
|
||||
<a href="#coverage"><ClipboardCheck size={18} />覆盖</a>
|
||||
<a href="#weights"><HardDrive size={18} />权重</a>
|
||||
<a href="#results"><BarChart3 size={18} />结果</a>
|
||||
@@ -768,7 +808,7 @@ function App() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="grid three">
|
||||
<section className="grid four">
|
||||
<div className="panel" id="gpu">
|
||||
<div className="panelHead">
|
||||
<div>
|
||||
@@ -789,6 +829,37 @@ function App() {
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="panel" id="runtime">
|
||||
<div className="panelHead">
|
||||
<div>
|
||||
<p className="eyebrow">Runtime</p>
|
||||
<h2>环境就绪</h2>
|
||||
</div>
|
||||
<ShieldCheck size={22} />
|
||||
</div>
|
||||
<div className="envList">
|
||||
{(runtimeReadiness?.envs ?? []).map((env) => (
|
||||
<div key={env.role} className={`envCard ${env.passed ? "ok" : "bad"}`}>
|
||||
<div className="envHead">
|
||||
<div>
|
||||
<strong>{env.name}</strong>
|
||||
<small>{env.label}</small>
|
||||
</div>
|
||||
<span>{env.passed ? "READY" : env.exists ? "CHECK" : "MISSING"}</span>
|
||||
</div>
|
||||
<div className="envChecks">
|
||||
{env.checks.slice(0, 8).map((check) => (
|
||||
<span key={check.module} className={check.passed ? "ok" : "bad"} title={check.error ?? check.package}>
|
||||
{check.module}{check.version ? ` ${check.version}` : ""}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<p className="muted">{runtimeReadiness?.passed ? "runtime imports ready" : "run scripts/bootstrap_conda_envs.sh"} · {runtimeReadiness?.generated_at ?? "not checked"}</p>
|
||||
</div>
|
||||
|
||||
<div className="panel" id="weights">
|
||||
<div className="panelHead">
|
||||
<div>
|
||||
|
||||
@@ -219,6 +219,11 @@ h2 {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.grid.four {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
|
||||
.panel {
|
||||
padding: 18px;
|
||||
min-width: 0;
|
||||
@@ -748,6 +753,87 @@ meter {
|
||||
accent-color: var(--green);
|
||||
}
|
||||
|
||||
.envList {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.envCard {
|
||||
min-width: 0;
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
padding: 10px;
|
||||
border-radius: 7px;
|
||||
border: 1px solid var(--line);
|
||||
background: #101310;
|
||||
}
|
||||
|
||||
.envCard.ok {
|
||||
border-color: rgba(157, 226, 111, 0.32);
|
||||
}
|
||||
|
||||
.envCard.bad {
|
||||
border-color: rgba(240, 113, 103, 0.55);
|
||||
}
|
||||
|
||||
.envHead {
|
||||
min-width: 0;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 8px;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.envHead strong,
|
||||
.envHead small {
|
||||
display: block;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.envHead small {
|
||||
color: var(--muted);
|
||||
margin-top: 2px;
|
||||
}
|
||||
|
||||
.envHead > span {
|
||||
color: var(--green);
|
||||
font-size: 11px;
|
||||
font-weight: 760;
|
||||
}
|
||||
|
||||
.envCard.bad .envHead > span {
|
||||
color: var(--red);
|
||||
}
|
||||
|
||||
.envChecks {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.envChecks span {
|
||||
max-width: 100%;
|
||||
padding: 4px 6px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid rgba(238, 242, 232, 0.1);
|
||||
color: var(--muted);
|
||||
background: #080a08;
|
||||
font-size: 11px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.envChecks span.ok {
|
||||
color: var(--green);
|
||||
}
|
||||
|
||||
.envChecks span.bad {
|
||||
color: var(--red);
|
||||
}
|
||||
|
||||
.bigNumber {
|
||||
font-size: 54px;
|
||||
font-weight: 760;
|
||||
|
||||
58
scripts/bootstrap_conda_envs.sh
Executable file
58
scripts/bootstrap_conda_envs.sh
Executable file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
TASK_ENV="${SEG_TASK_CONDA_ENV:-seg_smp}"
|
||||
MMSEG_ENV="${SEG_MMSEG_CONDA_ENV:-seg_mmcv}"
|
||||
|
||||
env_exists() {
|
||||
conda env list | awk '{print $1}' | grep -Fxq "$1"
|
||||
}
|
||||
|
||||
create_task_env() {
|
||||
if ! env_exists "${TASK_ENV}"; then
|
||||
conda create -n "${TASK_ENV}" python=3.11 -y
|
||||
fi
|
||||
conda run -n "${TASK_ENV}" python -m pip install -U pip
|
||||
conda run -n "${TASK_ENV}" python -m pip install -r "${ROOT_DIR}/backend/requirements.txt"
|
||||
conda run -n "${TASK_ENV}" python -m pip install \
|
||||
torch==2.6.0 torchvision==0.21.0 \
|
||||
'numpy<2' 'opencv-python<4.12' albumentations segmentation-models-pytorch ultralytics \
|
||||
mmengine mmsegmentation==1.2.2 mmcv-lite \
|
||||
matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
|
||||
}
|
||||
|
||||
create_mmseg_env() {
|
||||
if ! env_exists "${MMSEG_ENV}"; then
|
||||
conda create -n "${MMSEG_ENV}" python=3.10 -y
|
||||
fi
|
||||
conda run -n "${MMSEG_ENV}" python -m pip install -U pip
|
||||
conda run -n "${MMSEG_ENV}" python -m pip install \
|
||||
torch==2.1.2 torchvision==0.16.2 \
|
||||
--index-url https://download.pytorch.org/whl/cu121
|
||||
conda run -n "${MMSEG_ENV}" python -m pip install \
|
||||
mmengine==0.10.7 mmsegmentation==1.2.2 'mmcv==2.1.0' \
|
||||
-f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html
|
||||
conda run -n "${MMSEG_ENV}" python -m pip install \
|
||||
'numpy<2' 'opencv-python<4.12' ftfy regex matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard
|
||||
}
|
||||
|
||||
case "${1:-all}" in
|
||||
all)
|
||||
create_task_env
|
||||
create_mmseg_env
|
||||
PYTHONPATH="${ROOT_DIR}/backend" conda run -n "${TASK_ENV}" python "${ROOT_DIR}/scripts/verify_runtime_envs.py" --refresh
|
||||
;;
|
||||
task)
|
||||
create_task_env
|
||||
echo "Created or repaired ${TASK_ENV}. Run '$0 all' for full runtime verification."
|
||||
;;
|
||||
mmseg)
|
||||
create_mmseg_env
|
||||
echo "Created or repaired ${MMSEG_ENV}. Run '$0 all' for full runtime verification."
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 [all|task|mmseg]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
27
scripts/verify_runtime_envs.py
Executable file
27
scripts/verify_runtime_envs.py
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "backend"))
|
||||
|
||||
from app.modules.system.service import get_runtime_readiness # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Verify Seg Data Server runtime conda environments.")
|
||||
parser.add_argument("--refresh", action="store_true", help="ignore the backend readiness cache")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = get_runtime_readiness(force=args.refresh)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report.get("passed"):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user