diff --git a/.env.example b/.env.example index 22206cd..584102d 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ SEG_DATA_SERVER_ROOT=. SEG_BACKEND_DB=var/seg_data_server.sqlite3 SEG_BACKEND_LOG_DIR=var/job_logs SEG_TASK_CONDA_ENV=seg_smp +SEG_MMSEG_CONDA_ENV=seg_mmcv SEG_BACKEND_CONDA_ENV=seg_smp SEG_WEIGHT_MODE=copy SEG_ENABLE_SHELL_TASKS=1 diff --git a/README.md b/README.md index bea7031..667e837 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,9 @@ Seg_Data_Server_Net/ cd Seg_Data_Server_Net cp .env.example .env -# Backend. The deployment env is seg_smp so the API and task wrappers share -# the same segmentation dependency stack. +# Backend. The deployment env is seg_smp so the API and most task wrappers +# share the same segmentation dependency stack. MMSeg jobs default to the +# separate SEG_MMSEG_CONDA_ENV because full mmcv wheels must match torch/CUDA. conda run -n seg_smp uvicorn app.main:app --app-dir backend --host 0.0.0.0 --port 8010 # Frontend. @@ -57,12 +58,23 @@ artifact API, runs a mock job, checks SSE log streaming, and executes one legacy image/label overlay job on tiny generated PNGs. It also runs model family readiness checks: a SegModel/SMP forward pass, a YOLO segmentation prediction on a tiny image, MMSeg config parsing, and local MMSeg pretrained -weight discovery. +weight discovery. MMSeg full-model readiness is validated in +`SEG_MMSEG_CONDA_ENV` by importing `mmcv._ext` and building a local MMSeg +`EncoderDecoder` from the existing config tree. Current `seg_smp` uses `mmcv-lite` because no `torch 2.6/cu124` full `mmcv` wheel is available on this machine and `nvcc` is not installed for source -builds. The acceptance smoke reports MMSeg full model construction as a -warning until a full `mmcv` build with `mmcv._ext` is installed. +builds. A dedicated `seg_mmcv` environment is used for MMSeg tasks and has +`torch 2.1.2+cu121`, `mmcv 2.1.0`, `mmsegmentation 1.2.2`, and NumPy 1.26. +If rebuilding the environment, keep these versions aligned: + +```bash +conda create -n seg_mmcv python=3.10 -y +conda run -n seg_mmcv python -m pip install -U pip +conda run -n seg_mmcv python -m pip install torch==2.1.2 torchvision==0.16.2 --index-url https://download.pytorch.org/whl/cu121 +conda run -n seg_mmcv python -m pip install mmengine==0.10.7 mmsegmentation==1.2.2 'mmcv==2.1.0' -f https://download.openmmlab.com/mmcv/dist/cu121/torch2.1/index.html +conda run -n seg_mmcv python -m pip install 'numpy<2' 'opencv-python<4.12' ftfy regex matplotlib pandas scikit-learn scipy seaborn tqdm tensorboard +``` ## Weight Sync @@ -118,7 +130,8 @@ Run the local evaluation and validation agents before publishing changes: PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build ``` -The validation agent checks catalog coverage, the new `seg_smp` env, GPU -visibility, no-weight Git safety, backend tests, frontend build, and live -backend/frontend endpoints when the services are running. With live validation -enabled it also runs the lightweight acceptance smoke above. +The validation agent checks catalog coverage, the `seg_smp` task env, the +`seg_mmcv` MMSeg env, GPU visibility, no-weight Git safety, backend tests, +frontend build, and live backend/frontend endpoints when the services are +running. With live validation enabled it also runs the lightweight acceptance +smoke above. diff --git a/backend/app/acceptance.py b/backend/app/acceptance.py index 0eba587..44c0a57 100644 --- a/backend/app/acceptance.py +++ b/backend/app/acceptance.py @@ -13,20 +13,53 @@ from typing import Any from .config import settings +def _run_command(command: list[str], cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]: + try: + result = subprocess.run( + command, + cwd=str(cwd or settings.project_root), + capture_output=True, + text=True, + timeout=timeout, + ) + return { + "passed": result.returncode == 0, + "returncode": result.returncode, + "stdout": result.stdout[-4000:], + "stderr": result.stderr[-4000:], + } + except subprocess.TimeoutExpired as exc: + return { + "passed": False, + "returncode": None, + "stdout": (exc.stdout or "")[-4000:] if isinstance(exc.stdout, str) else "", + "stderr": (exc.stderr or "")[-4000:] if isinstance(exc.stderr, str) else "", + "error": f"command timed out after {timeout}s", + } + except Exception as exc: + return {"passed": False, "returncode": None, "stdout": "", "stderr": "", "error": str(exc)} + + def _run_snippet(code: str, cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]: - result = subprocess.run( - [sys.executable, "-c", code], - cwd=str(cwd or settings.project_root), - capture_output=True, - text=True, - timeout=timeout, - ) - return { - "passed": result.returncode == 0, - "returncode": result.returncode, - "stdout": result.stdout[-4000:], - "stderr": result.stderr[-4000:], - } + return _run_command([sys.executable, "-c", code], cwd=cwd, timeout=timeout) + + +def _run_conda_snippet(env_name: str, code: str, cwd: Path | None = None, timeout: int = 60) -> dict[str, Any]: + detail = _run_command(["conda", "run", "-n", env_name, "python", "-c", code], cwd=cwd, timeout=timeout) + detail["env"] = env_name + return detail + + +MMSEG_FULL_BUILD_SNIPPET = ( + "from mmseg.utils import register_all_modules; " + "register_all_modules(init_default_scope=True); " + "from mmengine.config import Config; " + "from mmseg.registry import MODELS; " + "import mmcv._ext; " + "cfg=Config.fromfile({config_path!r}); " + "model=MODELS.build(cfg.model); " + "print(type(model).__name__)" +) def _request_json(method: str, url: str, payload: dict[str, Any] | None = None, timeout: int = 10) -> dict[str, Any]: @@ -182,13 +215,23 @@ def run_model_family_readiness() -> dict[str, Any]: "required": True, "detail": {"passed": mmseg_pretrained.exists(), "path": str(mmseg_pretrained), "size": mmseg_pretrained.stat().st_size if mmseg_pretrained.exists() else 0}, }, + { + "name": "mmseg_full_env_imports", + "required": True, + "detail": _run_conda_snippet( + settings.mmseg_conda_env, + "import torch, cv2, mmcv, mmengine, mmseg; " + "import mmcv._ext; " + "print(torch.__version__, torch.version.cuda, cv2.__version__, mmcv.__version__, mmseg.__version__)", + timeout=90, + ), + }, { "name": "mmseg_full_model_build", - "required": False, - "detail": _run_snippet( - "from mmengine.config import Config; from mmseg.registry import MODELS; " - f"cfg=Config.fromfile({str(mmseg_config)!r}); " - "model=MODELS.build(cfg.model); print(type(model).__name__)", + "required": True, + "detail": _run_conda_snippet( + settings.mmseg_conda_env, + MMSEG_FULL_BUILD_SNIPPET.format(config_path=str(mmseg_config)), timeout=90, ), }, diff --git a/backend/app/agents/validation_agent.py b/backend/app/agents/validation_agent.py index 6db2363..304bef8 100644 --- a/backend/app/agents/validation_agent.py +++ b/backend/app/agents/validation_agent.py @@ -55,21 +55,36 @@ def validate_project(run_build: bool = False) -> dict: checks.append({"name": "weights_manifest_present", "passed": manifest.get("count", 0) >= 1}) checks.append({"name": "gpus_query", "passed": bool(get_gpus().get("available"))}) env_names = [item["name"] for item in get_conda_envs().get("envs", [])] - checks.append({"name": "seg_smp_env_exists", "passed": "seg_smp" in env_names}) + checks.append({"name": "task_env_exists", "passed": settings.task_conda_env in env_names, "detail": {"env": settings.task_conda_env}}) + checks.append({"name": "mmseg_env_exists", "passed": settings.mmseg_conda_env in env_names, "detail": {"env": settings.mmseg_conda_env}}) smoke = _run( [ "conda", "run", "-n", - "seg_smp", + settings.task_conda_env, "python", "-c", "import fastapi, uvicorn, torch, cv2, segmentation_models_pytorch, ultralytics, albumentations, mmengine, mmseg, mmcv; print(torch.__version__, torch.cuda.is_available())", ], cwd=settings.project_root, ) - checks.append({"name": "seg_smp_backend_smoke", "passed": smoke["passed"], "detail": smoke}) + checks.append({"name": "task_env_backend_smoke", "passed": smoke["passed"], "detail": smoke}) + + mmseg_smoke = _run( + [ + "conda", + "run", + "-n", + settings.mmseg_conda_env, + "python", + "-c", + "import torch, cv2, mmcv, mmengine, mmseg; import mmcv._ext; print(torch.__version__, torch.version.cuda, cv2.__version__, mmcv.__version__, mmseg.__version__)", + ], + cwd=settings.project_root, + ) + checks.append({"name": "mmseg_env_full_mmcv_smoke", "passed": mmseg_smoke["passed"], "detail": mmseg_smoke}) no_weight = _run(["bash", "scripts/check_no_weight_git.sh"], cwd=settings.project_root) checks.append({"name": "no_weight_in_git", "passed": no_weight["passed"], "detail": no_weight}) diff --git a/backend/app/config.py b/backend/app/config.py index bec87f7..3ffbd9c 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -20,6 +20,7 @@ class Settings: log_dir: Path weights_root: Path task_conda_env: str + mmseg_conda_env: str backend_conda_env: str weight_mode: str enable_shell_tasks: bool @@ -46,6 +47,7 @@ def get_settings() -> Settings: log_dir=log_dir, weights_root=weights_root, task_conda_env=os.getenv("SEG_TASK_CONDA_ENV", "seg_smp"), + mmseg_conda_env=os.getenv("SEG_MMSEG_CONDA_ENV", "seg_mmcv"), backend_conda_env=os.getenv("SEG_BACKEND_CONDA_ENV", "seg_smp"), weight_mode=os.getenv("SEG_WEIGHT_MODE", "copy"), enable_shell_tasks=os.getenv("SEG_ENABLE_SHELL_TASKS", "1") == "1", diff --git a/backend/app/coverage.py b/backend/app/coverage.py index b20a4ac..4956e87 100644 --- a/backend/app/coverage.py +++ b/backend/app/coverage.py @@ -157,7 +157,8 @@ def build_task_checks() -> list[dict[str, Any]]: for task in TASK_TYPES: params = TASK_DEFAULTS.get(task, {}) try: - spec = build_module_task(task, dict(params), settings.task_conda_env) + conda_env = settings.mmseg_conda_env if task.startswith("mmseg.") else settings.task_conda_env + spec = build_module_task(task, dict(params), conda_env) script_path = _command_script_path(spec.command) if spec else None checks.append( { diff --git a/backend/app/jobs.py b/backend/app/jobs.py index 68b6e9d..1e6d1e8 100644 --- a/backend/app/jobs.py +++ b/backend/app/jobs.py @@ -17,8 +17,14 @@ _running: dict[str, subprocess.Popen] = {} _lock = threading.Lock() +def default_conda_env_for_job(job_type: str) -> str: + if job_type.startswith("mmseg."): + return settings.mmseg_conda_env + return settings.task_conda_env + + def _build_task(request: JobCreate) -> CommandSpec: - conda_env = request.conda_env or settings.task_conda_env + conda_env = request.conda_env or default_conda_env_for_job(request.type) spec = build_module_task(request.type, request.params, conda_env) if spec is None: raise ValueError(f"unsupported job type: {request.type}") @@ -113,4 +119,3 @@ def cancel_job(job_id: str) -> dict | None: except ProcessLookupError: pass return db.get_job(job_id) - diff --git a/backend/app/modules/system/service.py b/backend/app/modules/system/service.py index 7c981b2..008e157 100644 --- a/backend/app/modules/system/service.py +++ b/backend/app/modules/system/service.py @@ -60,7 +60,7 @@ def get_conda_envs() -> dict: parts = raw.replace("*", " ").split() if len(parts) >= 2: envs.append({"name": parts[0], "path": parts[-1], "active": marker}) - return {"available": True, "envs": envs, "task_default": settings.task_conda_env} + return {"available": True, "envs": envs, "task_default": settings.task_conda_env, "mmseg_default": settings.mmseg_conda_env} def disk_usage() -> dict: @@ -103,4 +103,3 @@ def scan_results() -> list[dict]: continue results.sort(key=lambda item: item["modified"], reverse=True) return results[:1000] - diff --git a/backend/tests/test_artifacts.py b/backend/tests/test_artifacts.py index 28e0483..bb77d11 100644 --- a/backend/tests/test_artifacts.py +++ b/backend/tests/test_artifacts.py @@ -23,6 +23,7 @@ def test_project_var_artifact_path_is_served(tmp_path, monkeypatch): log_dir=original.log_dir, weights_root=original.weights_root, task_conda_env=original.task_conda_env, + mmseg_conda_env=original.mmseg_conda_env, backend_conda_env=original.backend_conda_env, weight_mode=original.weight_mode, enable_shell_tasks=original.enable_shell_tasks, diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py new file mode 100644 index 0000000..f30c4e3 --- /dev/null +++ b/backend/tests/test_jobs.py @@ -0,0 +1,6 @@ +from app.jobs import default_conda_env_for_job + + +def test_mmseg_jobs_use_mmseg_conda_env_by_default(): + assert default_conda_env_for_job("mmseg.train") == "seg_mmcv" + assert default_conda_env_for_job("segmodel.train") == "seg_smp"