Add real YOLO train acceptance
This commit is contained in:
30
README.md
30
README.md
@@ -198,10 +198,11 @@ scripts/check_no_weight_git.sh
|
|||||||
|
|
||||||
For a fast non-training validation pass, run agents with
|
For a fast non-training validation pass, run agents with
|
||||||
`PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --no-deep`.
|
`PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --no-deep`.
|
||||||
Add `--live`, `--acceptance`, or `--real` only after the backend and frontend
|
Add `--live`, `--acceptance`, `--real`, or `--real-train` only after the
|
||||||
are running and you want HTTP endpoint, smoke, or real-workspace checks. The
|
backend and frontend are running and you want HTTP endpoint, smoke,
|
||||||
browser dashboard exposes the same readiness, coverage, GPU, weight, result,
|
real-workspace, or real short-training checks. The browser dashboard exposes
|
||||||
and agent checks through the UI.
|
the same readiness, coverage, GPU, weight, result, and agent checks through
|
||||||
|
the UI.
|
||||||
|
|
||||||
The web UI includes a dataset bench for creating upload workspaces, uploading
|
The web UI includes a dataset bench for creating upload workspaces, uploading
|
||||||
images/labels/masks, and jumping into the existing rename, PNG conversion,
|
images/labels/masks, and jumping into the existing rename, PNG conversion,
|
||||||
@@ -272,6 +273,16 @@ real image/mask pair. The latest report is available from
|
|||||||
`GET /api/acceptance/real/latest` and is shown in the coverage panel as
|
`GET /api/acceptance/real/latest` and is shown in the coverage panel as
|
||||||
`真实数据`.
|
`真实数据`.
|
||||||
|
|
||||||
|
`POST /api/acceptance/real-train` goes one step further and launches a short
|
||||||
|
operator-style YOLO loop on real workspace data. It uploads a real YOLO
|
||||||
|
image/txt pair, generates `dataset.yaml`, runs one CPU epoch through
|
||||||
|
`yolo.train_custom`, verifies `results.csv` and `best.pt`, then uses that
|
||||||
|
trained checkpoint for prediction and GradCAM heatmap jobs. The latest report
|
||||||
|
is available from `GET /api/acceptance/real-train/latest` and is shown in the
|
||||||
|
coverage panel as `真实短训`. This is heavier than the real-data predict
|
||||||
|
acceptance, so run it when you want proof that real uploaded data can create
|
||||||
|
loss curves, trained weights, segmentation previews, and heatmap artifacts.
|
||||||
|
|
||||||
For stronger runtime proof, `POST /api/acceptance/deep` runs minimal training
|
For stronger runtime proof, `POST /api/acceptance/deep` runs minimal training
|
||||||
loops for the three model families: one SegModel optimizer step, one YOLO
|
loops for the three model families: one SegModel optimizer step, one YOLO
|
||||||
segmentation epoch on a synthetic 64x64 dataset, one YOLO GradCAM heatmap
|
segmentation epoch on a synthetic 64x64 dataset, one YOLO GradCAM heatmap
|
||||||
@@ -372,8 +383,9 @@ non-training validation pass is needed.
|
|||||||
|
|
||||||
The web dashboard calls validation in light mode by default:
|
The web dashboard calls validation in light mode by default:
|
||||||
`/api/agents/validate?run_build=false&run_acceptance=false&run_deep=false`.
|
`/api/agents/validate?run_build=false&run_acceptance=false&run_deep=false`.
|
||||||
Pass `run_live=true`, `run_acceptance=true`, `run_real=true`, or
|
Pass `run_live=true`, `run_acceptance=true`, `run_real=true`,
|
||||||
`run_deep=true` only when you explicitly want the agent to launch live endpoint
|
`run_real_train=true`, or `run_deep=true` only when you explicitly want the
|
||||||
or heavier runtime acceptance checks from the browser/API. Smoke and real data
|
agent to launch live endpoint or heavier runtime acceptance checks from the
|
||||||
acceptance automatically enable the live backend checks because they submit
|
browser/API. Smoke, real data, and real short-training acceptance
|
||||||
jobs through the API.
|
automatically enable the live backend checks because they submit jobs through
|
||||||
|
the API.
|
||||||
|
|||||||
@@ -447,6 +447,13 @@ def latest_real_acceptance_report() -> dict[str, Any]:
|
|||||||
return json.loads(path.read_text(encoding="utf-8"))
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def latest_real_train_acceptance_report() -> dict[str, Any]:
|
||||||
|
path = settings.project_root / "var" / "acceptance" / "real_train_latest.json"
|
||||||
|
if not path.exists():
|
||||||
|
return {"available": False, "path": str(path)}
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
def run_real_dataset_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
def run_real_dataset_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
||||||
"""Run the upload/predict/heatmap path against existing non-synthetic Seg data."""
|
"""Run the upload/predict/heatmap path against existing non-synthetic Seg data."""
|
||||||
acceptance_root = settings.project_root / "var" / "acceptance"
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
||||||
@@ -620,6 +627,178 @@ def run_real_dataset_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict
|
|||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def run_real_train_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
||||||
|
"""Run a short YOLO train/predict/heatmap loop using real workspace samples."""
|
||||||
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
||||||
|
run_id = uuid.uuid4().hex[:8]
|
||||||
|
fixture_root = acceptance_root / f"real_train_{run_id}"
|
||||||
|
fixture_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
samples = find_real_workspace_samples()
|
||||||
|
checks: list[dict[str, Any]] = [
|
||||||
|
{"name": "real_train_workspace_samples_discovered", "passed": samples["passed"], "detail": samples}
|
||||||
|
]
|
||||||
|
if not samples["passed"]:
|
||||||
|
report = {
|
||||||
|
"available": True,
|
||||||
|
"run_id": run_id,
|
||||||
|
"base_url": base_url,
|
||||||
|
"fixture_root": str(fixture_root),
|
||||||
|
"passed": False,
|
||||||
|
"checks": checks,
|
||||||
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||||
|
}
|
||||||
|
(acceptance_root / "real_train_latest.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
return report
|
||||||
|
|
||||||
|
dataset_name = f"real_train_acceptance_{run_id}"
|
||||||
|
created_dataset = _request_json("POST", f"{base_url}/api/datasets", {"name": dataset_name, "description": "real workspace short train acceptance"}, timeout=10)
|
||||||
|
checks.append({"name": "create_real_train_upload_dataset", "passed": created_dataset.get("passed", False), "detail": created_dataset})
|
||||||
|
|
||||||
|
yolo_image = Path(samples["yolo_pair"]["image"])
|
||||||
|
yolo_label = Path(samples["yolo_pair"]["label"])
|
||||||
|
uploads = {
|
||||||
|
"real_train_yolo_image_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/images", yolo_image, timeout=30),
|
||||||
|
"real_train_yolo_label_upload": _post_file(f"{base_url}/api/datasets/{dataset_name}/upload/labels", yolo_label, timeout=30),
|
||||||
|
}
|
||||||
|
for name, detail in uploads.items():
|
||||||
|
checks.append({"name": name, "passed": detail.get("passed", False), "detail": detail})
|
||||||
|
|
||||||
|
validation = _request_json("GET", f"{base_url}/api/datasets/{dataset_name}/validate", timeout=20)
|
||||||
|
validation_json = validation.get("json") if validation.get("passed") else {}
|
||||||
|
checks.append(
|
||||||
|
{
|
||||||
|
"name": "real_train_dataset_validate_yolo",
|
||||||
|
"passed": validation.get("passed", False) and validation_json.get("ready", {}).get("yolo"),
|
||||||
|
"detail": validation,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
class_count = max(validation_json.get("classes") or [0]) + 1
|
||||||
|
class_names = ["object"] + [f"class_{index}" for index in range(1, class_count)]
|
||||||
|
yolo_yaml = _request_json("POST", f"{base_url}/api/datasets/{dataset_name}/yolo-yaml", {"class_names": class_names}, timeout=20)
|
||||||
|
yolo_yaml_json = yolo_yaml.get("json") if yolo_yaml.get("passed") else {}
|
||||||
|
checks.append({"name": "real_train_dataset_yolo_yaml", "passed": yolo_yaml.get("passed", False), "detail": yolo_yaml})
|
||||||
|
|
||||||
|
train_name = f"{dataset_name}_train"
|
||||||
|
train = _create_job_and_wait(
|
||||||
|
base_url,
|
||||||
|
"yolo.train_custom",
|
||||||
|
{
|
||||||
|
"data": yolo_yaml_json.get("relative_path", f"var/uploads/datasets/{dataset_name}/dataset.yaml"),
|
||||||
|
"model": str(settings.source_root / "Seg_All_In_One_YoloModel" / "yolo11n-seg.pt"),
|
||||||
|
"project": "var/custom_yolo_runs",
|
||||||
|
"name": train_name,
|
||||||
|
"epochs": 1,
|
||||||
|
"imgsz": 96,
|
||||||
|
"batch": 1,
|
||||||
|
"workers": 0,
|
||||||
|
"device": "cpu",
|
||||||
|
"exist_ok": True,
|
||||||
|
},
|
||||||
|
timeout=240,
|
||||||
|
)
|
||||||
|
train_root = settings.project_root / "var" / "custom_yolo_runs" / train_name
|
||||||
|
best_weight = train_root / "weights" / "best.pt"
|
||||||
|
last_weight = train_root / "weights" / "last.pt"
|
||||||
|
results_csv = train_root / "results.csv"
|
||||||
|
checks.append(
|
||||||
|
{
|
||||||
|
"name": "real_train_yolo_one_epoch_job_runner",
|
||||||
|
"passed": train.get("passed", False) and best_weight.exists() and results_csv.exists() and results_csv.stat().st_size > 0,
|
||||||
|
"detail": {
|
||||||
|
**train,
|
||||||
|
"best_weight": _relative_to_project(best_weight) if best_weight.exists() else None,
|
||||||
|
"last_weight": _relative_to_project(last_weight) if last_weight.exists() else None,
|
||||||
|
"results_csv": _relative_to_project(results_csv) if results_csv.exists() else None,
|
||||||
|
"results_csv_size": results_csv.stat().st_size if results_csv.exists() else 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
uploaded_image_json = uploads["real_train_yolo_image_upload"].get("json", {})
|
||||||
|
uploaded_image = uploaded_image_json.get("saved", [{}])[0].get("relative_path")
|
||||||
|
predict_name = f"{dataset_name}_predict_trained"
|
||||||
|
if best_weight.exists() and uploaded_image:
|
||||||
|
predict = _create_job_and_wait(
|
||||||
|
base_url,
|
||||||
|
"yolo.predict_custom",
|
||||||
|
{
|
||||||
|
"weights": str(best_weight),
|
||||||
|
"source": uploaded_image,
|
||||||
|
"project": "var/custom_yolo_runs",
|
||||||
|
"name": predict_name,
|
||||||
|
"imgsz": 96,
|
||||||
|
"conf": 0.01,
|
||||||
|
"device": "cpu",
|
||||||
|
"exist_ok": True,
|
||||||
|
},
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
predict = {"passed": False, "error": "skipped because training did not produce best.pt or upload path"}
|
||||||
|
predict_root = settings.project_root / "var" / "custom_yolo_runs" / predict_name
|
||||||
|
predict_outputs = _result_files(predict_root, {".png", ".jpg", ".jpeg"})
|
||||||
|
checks.append(
|
||||||
|
{
|
||||||
|
"name": "real_train_trained_weight_predict_job_runner",
|
||||||
|
"passed": predict.get("passed", False) and bool(predict_outputs),
|
||||||
|
"detail": {**predict, "output_count": len(predict_outputs), "outputs": [_relative_to_project(path) for path in predict_outputs[:8]]},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
heatmap_name = f"{dataset_name}_heatmap_trained"
|
||||||
|
if best_weight.exists() and uploaded_image:
|
||||||
|
heatmap = _create_job_and_wait(
|
||||||
|
base_url,
|
||||||
|
"yolo.heatmap_custom",
|
||||||
|
{
|
||||||
|
"weights": str(best_weight),
|
||||||
|
"source": uploaded_image,
|
||||||
|
"project": "var/custom_yolo_runs",
|
||||||
|
"name": heatmap_name,
|
||||||
|
"model_key": "YOLO11n-seg",
|
||||||
|
"pt_name": "best.pt",
|
||||||
|
"cam_method": "GradCAM",
|
||||||
|
"target_layers": "model.model.model[9]",
|
||||||
|
"limit": 1,
|
||||||
|
},
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
heatmap = {"passed": False, "error": "skipped because training did not produce best.pt or upload path"}
|
||||||
|
heatmap_root = settings.project_root / "var" / "custom_yolo_runs" / heatmap_name / "HeartMap_Visual"
|
||||||
|
heatmap_outputs = _result_files(heatmap_root, {".jpg", ".jpeg", ".png"})
|
||||||
|
checks.append(
|
||||||
|
{
|
||||||
|
"name": "real_train_trained_weight_heatmap_job_runner",
|
||||||
|
"passed": heatmap.get("passed", False) and len(heatmap_outputs) >= 2,
|
||||||
|
"detail": {**heatmap, "output_count": len(heatmap_outputs), "outputs": [_relative_to_project(path) for path in heatmap_outputs[:8]]},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
report = {
|
||||||
|
"available": True,
|
||||||
|
"run_id": run_id,
|
||||||
|
"base_url": base_url,
|
||||||
|
"fixture_root": str(fixture_root),
|
||||||
|
"dataset_name": dataset_name,
|
||||||
|
"samples": samples,
|
||||||
|
"artifacts": {
|
||||||
|
"train_root": _relative_to_project(train_root),
|
||||||
|
"best_weight": _relative_to_project(best_weight) if best_weight.exists() else None,
|
||||||
|
"results_csv": _relative_to_project(results_csv) if results_csv.exists() else None,
|
||||||
|
"predict_outputs": [_relative_to_project(path) for path in predict_outputs[:8]],
|
||||||
|
"heatmap_outputs": [_relative_to_project(path) for path in heatmap_outputs[:8]],
|
||||||
|
},
|
||||||
|
"passed": all(item["passed"] for item in checks),
|
||||||
|
"checks": checks,
|
||||||
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||||
|
}
|
||||||
|
(acceptance_root / "real_train_latest.json").write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
def run_deep_acceptance() -> dict[str, Any]:
|
def run_deep_acceptance() -> dict[str, Any]:
|
||||||
"""Run minimal training loops for each model family without full datasets."""
|
"""Run minimal training loops for each model family without full datasets."""
|
||||||
acceptance_root = settings.project_root / "var" / "acceptance"
|
acceptance_root = settings.project_root / "var" / "acceptance"
|
||||||
|
|||||||
@@ -108,6 +108,12 @@ def evaluate_project() -> dict:
|
|||||||
and "real_workspace_yolo_predict_job_runner" in acceptance_text
|
and "real_workspace_yolo_predict_job_runner" in acceptance_text
|
||||||
and "real_workspace_yolo_heatmap_job_runner" in acceptance_text
|
and "real_workspace_yolo_heatmap_job_runner" in acceptance_text
|
||||||
and "real_workspace_stack_job_runner" in acceptance_text,
|
and "real_workspace_stack_job_runner" in acceptance_text,
|
||||||
|
"real_train_acceptance": "/api/acceptance/real-train" in backend_text
|
||||||
|
and "runRealTrainAcceptance" in frontend_text
|
||||||
|
and "真实短训" in frontend_text
|
||||||
|
and "real_train_yolo_one_epoch_job_runner" in acceptance_text
|
||||||
|
and "real_train_trained_weight_predict_job_runner" in acceptance_text
|
||||||
|
and "real_train_trained_weight_heatmap_job_runner" in acceptance_text,
|
||||||
"agent_api": "/api/agents/evaluate" in backend_text and "/api/agents/validate" in backend_text,
|
"agent_api": "/api/agents/evaluate" in backend_text and "/api/agents/validate" in backend_text,
|
||||||
"agent_panel_ui": "runAgentValidation" in frontend_text and "评价建议" in frontend_text and "Validation Agent" in frontend_text,
|
"agent_panel_ui": "runAgentValidation" in frontend_text and "评价建议" in frontend_text and "Validation Agent" in frontend_text,
|
||||||
"coverage_api": "/api/coverage" in backend_text and coverage["task_build_passed"],
|
"coverage_api": "/api/coverage" in backend_text and coverage["task_build_passed"],
|
||||||
@@ -132,7 +138,7 @@ def evaluate_project() -> dict:
|
|||||||
if coverage["unmapped_user_scripts"]:
|
if coverage["unmapped_user_scripts"]:
|
||||||
suggestions.append(f"Map remaining user-facing scripts: {len(coverage['unmapped_user_scripts'])}")
|
suggestions.append(f"Map remaining user-facing scripts: {len(coverage['unmapped_user_scripts'])}")
|
||||||
if not suggestions:
|
if not suggestions:
|
||||||
suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.")
|
suggestions.append("Current platform covers the requested control-plane features, uploaded YOLO dataset train/predict/heatmap actions, live uploaded-data YOLO predict/heatmap acceptance, real workspace data acceptance, real short-train acceptance, and synthetic deep training acceptance; next focus is a longer operator-run task on a full dataset.")
|
||||||
|
|
||||||
passed_count = sum(1 for item in checks if item["passed"])
|
passed_count = sum(1 for item in checks if item["passed"])
|
||||||
total_count = max(len(checks), 1)
|
total_count = max(len(checks), 1)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..acceptance import run_deep_acceptance, run_live_acceptance, run_real_dataset_acceptance
|
from ..acceptance import run_deep_acceptance, run_live_acceptance, run_real_dataset_acceptance, run_real_train_acceptance
|
||||||
from ..capabilities import get_capability_matrix
|
from ..capabilities import get_capability_matrix
|
||||||
from ..catalog import get_catalog
|
from ..catalog import get_catalog
|
||||||
from ..config import settings
|
from ..config import settings
|
||||||
@@ -47,6 +47,7 @@ def validate_project(
|
|||||||
run_acceptance: bool | None = None,
|
run_acceptance: bool | None = None,
|
||||||
run_deep: bool | None = None,
|
run_deep: bool | None = None,
|
||||||
run_real: bool | None = None,
|
run_real: bool | None = None,
|
||||||
|
run_real_train: bool | None = None,
|
||||||
run_live: bool | None = None,
|
run_live: bool | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Validate current runtime readiness without launching heavy training."""
|
"""Validate current runtime readiness without launching heavy training."""
|
||||||
@@ -122,8 +123,9 @@ def validate_project(
|
|||||||
acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1"
|
acceptance_enabled = run_acceptance if run_acceptance is not None else os.getenv("SEG_VALIDATE_ACCEPTANCE", "0") == "1"
|
||||||
deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
|
deep_enabled = run_deep if run_deep is not None else os.getenv("SEG_VALIDATE_DEEP", "1") == "1"
|
||||||
real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
|
real_enabled = run_real if run_real is not None else os.getenv("SEG_VALIDATE_REAL", "0") == "1"
|
||||||
|
real_train_enabled = run_real_train if run_real_train is not None else os.getenv("SEG_VALIDATE_REAL_TRAIN", "0") == "1"
|
||||||
live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1"
|
live_enabled = run_live if run_live is not None else os.getenv("SEG_VALIDATE_LIVE", "0") == "1"
|
||||||
live_enabled = live_enabled or acceptance_enabled or real_enabled
|
live_enabled = live_enabled or acceptance_enabled or real_enabled or real_train_enabled
|
||||||
|
|
||||||
if live_enabled:
|
if live_enabled:
|
||||||
backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010")
|
backend_url = os.getenv("SEG_VALIDATE_BACKEND_URL", "http://127.0.0.1:8010")
|
||||||
@@ -166,6 +168,9 @@ def validate_project(
|
|||||||
if real_enabled:
|
if real_enabled:
|
||||||
real_acceptance = run_real_dataset_acceptance(backend_url)
|
real_acceptance = run_real_dataset_acceptance(backend_url)
|
||||||
checks.append({"name": "real_workspace_acceptance", "passed": real_acceptance["passed"], "detail": real_acceptance})
|
checks.append({"name": "real_workspace_acceptance", "passed": real_acceptance["passed"], "detail": real_acceptance})
|
||||||
|
if real_train_enabled:
|
||||||
|
real_train_acceptance = run_real_train_acceptance(backend_url)
|
||||||
|
checks.append({"name": "real_train_acceptance", "passed": real_train_acceptance["passed"], "detail": real_train_acceptance})
|
||||||
if deep_enabled:
|
if deep_enabled:
|
||||||
deep_acceptance = run_deep_acceptance()
|
deep_acceptance = run_deep_acceptance()
|
||||||
checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
|
checks.append({"name": "deep_training_acceptance", "passed": deep_acceptance["passed"], "detail": deep_acceptance})
|
||||||
|
|||||||
@@ -3,7 +3,12 @@ from __future__ import annotations
|
|||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, latest_real_acceptance_report
|
from .acceptance import (
|
||||||
|
latest_acceptance_report,
|
||||||
|
latest_deep_acceptance_report,
|
||||||
|
latest_real_acceptance_report,
|
||||||
|
latest_real_train_acceptance_report,
|
||||||
|
)
|
||||||
from .catalog import get_catalog
|
from .catalog import get_catalog
|
||||||
from .coverage import get_coverage_report
|
from .coverage import get_coverage_report
|
||||||
from .modules.dataset.service import list_uploaded_datasets
|
from .modules.dataset.service import list_uploaded_datasets
|
||||||
@@ -157,6 +162,7 @@ def get_capability_matrix() -> dict[str, Any]:
|
|||||||
gpus = get_gpus()
|
gpus = get_gpus()
|
||||||
acceptance = latest_acceptance_report()
|
acceptance = latest_acceptance_report()
|
||||||
real_acceptance = latest_real_acceptance_report()
|
real_acceptance = latest_real_acceptance_report()
|
||||||
|
real_train_acceptance = latest_real_train_acceptance_report()
|
||||||
deep_acceptance = latest_deep_acceptance_report()
|
deep_acceptance = latest_deep_acceptance_report()
|
||||||
|
|
||||||
all_tasks = catalog["task_types"]
|
all_tasks = catalog["task_types"]
|
||||||
@@ -277,6 +283,12 @@ def get_capability_matrix() -> dict[str, Any]:
|
|||||||
"passed": bool(real_acceptance.get("passed")),
|
"passed": bool(real_acceptance.get("passed")),
|
||||||
"detail": real_acceptance.get("run_id", "not run"),
|
"detail": real_acceptance.get("run_id", "not run"),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "real_train_acceptance",
|
||||||
|
"label": "真实短训练验收",
|
||||||
|
"passed": bool(real_train_acceptance.get("passed")),
|
||||||
|
"detail": real_train_acceptance.get("run_id", "not run"),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "weights_manifest",
|
"id": "weights_manifest",
|
||||||
"label": "权重清单",
|
"label": "权重清单",
|
||||||
@@ -302,6 +314,7 @@ def get_capability_matrix() -> dict[str, Any]:
|
|||||||
"gpus_available": bool(gpus.get("available")),
|
"gpus_available": bool(gpus.get("available")),
|
||||||
"acceptance_passed": bool(acceptance.get("passed")),
|
"acceptance_passed": bool(acceptance.get("passed")),
|
||||||
"real_acceptance_passed": bool(real_acceptance.get("passed")),
|
"real_acceptance_passed": bool(real_acceptance.get("passed")),
|
||||||
|
"real_train_acceptance_passed": bool(real_train_acceptance.get("passed")),
|
||||||
"deep_acceptance_passed": bool(deep_acceptance.get("passed")),
|
"deep_acceptance_passed": bool(deep_acceptance.get("passed")),
|
||||||
},
|
},
|
||||||
"requirements": requirements,
|
"requirements": requirements,
|
||||||
|
|||||||
@@ -13,9 +13,11 @@ from .acceptance import (
|
|||||||
latest_acceptance_report,
|
latest_acceptance_report,
|
||||||
latest_deep_acceptance_report,
|
latest_deep_acceptance_report,
|
||||||
latest_real_acceptance_report,
|
latest_real_acceptance_report,
|
||||||
|
latest_real_train_acceptance_report,
|
||||||
run_deep_acceptance,
|
run_deep_acceptance,
|
||||||
run_live_acceptance,
|
run_live_acceptance,
|
||||||
run_real_dataset_acceptance,
|
run_real_dataset_acceptance,
|
||||||
|
run_real_train_acceptance,
|
||||||
)
|
)
|
||||||
from .capabilities import get_capability_matrix
|
from .capabilities import get_capability_matrix
|
||||||
from .catalog import get_catalog
|
from .catalog import get_catalog
|
||||||
@@ -131,6 +133,16 @@ def api_real_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict:
|
|||||||
return run_real_dataset_acceptance(base_url)
|
return run_real_dataset_acceptance(base_url)
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/acceptance/real-train/latest")
|
||||||
|
def api_real_train_acceptance_latest() -> dict:
|
||||||
|
return latest_real_train_acceptance_report()
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/acceptance/real-train")
|
||||||
|
def api_real_train_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict:
|
||||||
|
return run_real_train_acceptance(base_url)
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/datasets")
|
@app.get("/api/datasets")
|
||||||
def api_datasets() -> list[dict]:
|
def api_datasets() -> list[dict]:
|
||||||
return list_uploaded_datasets()
|
return list_uploaded_datasets()
|
||||||
@@ -295,6 +307,7 @@ def api_agent_validate(
|
|||||||
run_acceptance: bool = False,
|
run_acceptance: bool = False,
|
||||||
run_deep: bool = False,
|
run_deep: bool = False,
|
||||||
run_real: bool = False,
|
run_real: bool = False,
|
||||||
|
run_real_train: bool = False,
|
||||||
run_live: bool | None = None,
|
run_live: bool | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
return validate_project(
|
return validate_project(
|
||||||
@@ -302,5 +315,6 @@ def api_agent_validate(
|
|||||||
run_acceptance=run_acceptance,
|
run_acceptance=run_acceptance,
|
||||||
run_deep=run_deep,
|
run_deep=run_deep,
|
||||||
run_real=run_real,
|
run_real=run_real,
|
||||||
|
run_real_train=run_real_train,
|
||||||
run_live=run_live,
|
run_live=run_live,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ def test_evaluation_agent_returns_checks():
|
|||||||
assert result["summary"]["passed_checks"] == result["summary"]["total_checks"]
|
assert result["summary"]["passed_checks"] == result["summary"]["total_checks"]
|
||||||
checks = {item["name"]: item["passed"] for item in result["checks"]}
|
checks = {item["name"]: item["passed"] for item in result["checks"]}
|
||||||
assert checks["real_workspace_acceptance"] is True
|
assert checks["real_workspace_acceptance"] is True
|
||||||
|
assert checks["real_train_acceptance"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_validation_agent_lightweight(monkeypatch):
|
def test_validation_agent_lightweight(monkeypatch):
|
||||||
|
|||||||
@@ -20,3 +20,4 @@ def test_capability_matrix_tracks_user_requirements():
|
|||||||
assert requirements["yolo_heatmap"]["passed"] is True
|
assert requirements["yolo_heatmap"]["passed"] is True
|
||||||
assert requirements["training_curves"]["passed"] is True
|
assert requirements["training_curves"]["passed"] is True
|
||||||
assert requirements["real_workspace_acceptance"]["passed"] is True
|
assert requirements["real_workspace_acceptance"]["passed"] is True
|
||||||
|
assert requirements["real_train_acceptance"]["passed"] is True
|
||||||
|
|||||||
@@ -380,6 +380,7 @@ function useData() {
|
|||||||
const [coverage, setCoverage] = useState<CoveragePayload | null>(null);
|
const [coverage, setCoverage] = useState<CoveragePayload | null>(null);
|
||||||
const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null);
|
const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null);
|
||||||
const [realAcceptance, setRealAcceptance] = useState<AcceptancePayload | null>(null);
|
const [realAcceptance, setRealAcceptance] = useState<AcceptancePayload | null>(null);
|
||||||
|
const [realTrainAcceptance, setRealTrainAcceptance] = useState<AcceptancePayload | null>(null);
|
||||||
const [deepAcceptance, setDeepAcceptance] = useState<DeepAcceptancePayload | null>(null);
|
const [deepAcceptance, setDeepAcceptance] = useState<DeepAcceptancePayload | null>(null);
|
||||||
const [runtimeReadiness, setRuntimeReadiness] = useState<RuntimeReadinessPayload | null>(null);
|
const [runtimeReadiness, setRuntimeReadiness] = useState<RuntimeReadinessPayload | null>(null);
|
||||||
const [capabilities, setCapabilities] = useState<CapabilityPayload | null>(null);
|
const [capabilities, setCapabilities] = useState<CapabilityPayload | null>(null);
|
||||||
@@ -388,7 +389,7 @@ function useData() {
|
|||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
try {
|
try {
|
||||||
const [catalogNext, gpusNext, envsNext, readinessNext, capabilitiesNext, jobsNext, resultsNext, curvesNext, weightsNext, datasetsNext, coverageNext, acceptanceNext, realAcceptanceNext, deepAcceptanceNext, agentEvaluationNext] = await Promise.all([
|
const [catalogNext, gpusNext, envsNext, readinessNext, capabilitiesNext, jobsNext, resultsNext, curvesNext, weightsNext, datasetsNext, coverageNext, acceptanceNext, realAcceptanceNext, realTrainAcceptanceNext, deepAcceptanceNext, agentEvaluationNext] = await Promise.all([
|
||||||
api<Catalog>("/api/catalog"),
|
api<Catalog>("/api/catalog"),
|
||||||
api<GpuPayload>("/api/system/gpus"),
|
api<GpuPayload>("/api/system/gpus"),
|
||||||
api<CondaEnvPayload>("/api/system/envs"),
|
api<CondaEnvPayload>("/api/system/envs"),
|
||||||
@@ -402,6 +403,7 @@ function useData() {
|
|||||||
api<CoveragePayload>("/api/coverage"),
|
api<CoveragePayload>("/api/coverage"),
|
||||||
api<AcceptancePayload>("/api/acceptance/latest"),
|
api<AcceptancePayload>("/api/acceptance/latest"),
|
||||||
api<AcceptancePayload>("/api/acceptance/real/latest"),
|
api<AcceptancePayload>("/api/acceptance/real/latest"),
|
||||||
|
api<AcceptancePayload>("/api/acceptance/real-train/latest"),
|
||||||
api<DeepAcceptancePayload>("/api/acceptance/deep/latest"),
|
api<DeepAcceptancePayload>("/api/acceptance/deep/latest"),
|
||||||
api<EvaluationAgentPayload>("/api/agents/evaluate")
|
api<EvaluationAgentPayload>("/api/agents/evaluate")
|
||||||
]);
|
]);
|
||||||
@@ -430,6 +432,7 @@ function useData() {
|
|||||||
setCoverage(coverageNext);
|
setCoverage(coverageNext);
|
||||||
setAcceptance(acceptanceNext);
|
setAcceptance(acceptanceNext);
|
||||||
setRealAcceptance(realAcceptanceNext);
|
setRealAcceptance(realAcceptanceNext);
|
||||||
|
setRealTrainAcceptance(realTrainAcceptanceNext);
|
||||||
setDeepAcceptance(deepAcceptanceNext);
|
setDeepAcceptance(deepAcceptanceNext);
|
||||||
setAgentEvaluation(agentEvaluationNext);
|
setAgentEvaluation(agentEvaluationNext);
|
||||||
setError("");
|
setError("");
|
||||||
@@ -444,7 +447,7 @@ function useData() {
|
|||||||
return () => window.clearInterval(timer);
|
return () => window.clearInterval(timer);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, realAcceptance, deepAcceptance, error, refresh };
|
return { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, realAcceptance, realTrainAcceptance, deepAcceptance, error, refresh };
|
||||||
}
|
}
|
||||||
|
|
||||||
function StatusPill({ status }: { status: string }) {
|
function StatusPill({ status }: { status: string }) {
|
||||||
@@ -467,7 +470,7 @@ function JobProgressBar({ progress }: { progress?: JobProgress }) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function App() {
|
function App() {
|
||||||
const { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, realAcceptance, deepAcceptance, error, refresh } = useData();
|
const { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, realAcceptance, realTrainAcceptance, deepAcceptance, error, refresh } = useData();
|
||||||
const [taskType, setTaskType] = useState("mock.echo");
|
const [taskType, setTaskType] = useState("mock.echo");
|
||||||
const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2));
|
const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2));
|
||||||
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
|
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
|
||||||
@@ -671,6 +674,16 @@ function App() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runRealTrainAcceptance() {
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
await api("/api/acceptance/real-train", { method: "POST" });
|
||||||
|
await refresh();
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function runAgentValidation() {
|
async function runAgentValidation() {
|
||||||
setAgentBusy(true);
|
setAgentBusy(true);
|
||||||
try {
|
try {
|
||||||
@@ -1133,6 +1146,9 @@ function App() {
|
|||||||
<button className="iconButton" disabled={busy} onClick={runRealAcceptance} title="运行真实数据验收">
|
<button className="iconButton" disabled={busy} onClick={runRealAcceptance} title="运行真实数据验收">
|
||||||
<FileSearch size={18} />
|
<FileSearch size={18} />
|
||||||
</button>
|
</button>
|
||||||
|
<button className="iconButton" disabled={busy} onClick={runRealTrainAcceptance} title="运行真实短训验收">
|
||||||
|
<Play size={18} />
|
||||||
|
</button>
|
||||||
<button className="iconButton" disabled={busy} onClick={runDeepAcceptance} title="运行深度训练验收">
|
<button className="iconButton" disabled={busy} onClick={runDeepAcceptance} title="运行深度训练验收">
|
||||||
<Activity size={18} />
|
<Activity size={18} />
|
||||||
</button>
|
</button>
|
||||||
@@ -1163,6 +1179,10 @@ function App() {
|
|||||||
<span>真实数据</span>
|
<span>真实数据</span>
|
||||||
<strong>{realAcceptance?.available === false ? "New" : realAcceptance?.passed ? "OK" : "Check"}</strong>
|
<strong>{realAcceptance?.available === false ? "New" : realAcceptance?.passed ? "OK" : "Check"}</strong>
|
||||||
</div>
|
</div>
|
||||||
|
<div>
|
||||||
|
<span>真实短训</span>
|
||||||
|
<strong>{realTrainAcceptance?.available === false ? "New" : realTrainAcceptance?.passed ? "OK" : "Check"}</strong>
|
||||||
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<span>深度训练</span>
|
<span>深度训练</span>
|
||||||
<strong>{deepAcceptance?.available === false ? "New" : deepAcceptance?.passed ? "OK" : "Check"}</strong>
|
<strong>{deepAcceptance?.available === false ? "New" : deepAcceptance?.passed ? "OK" : "Check"}</strong>
|
||||||
@@ -1174,6 +1194,7 @@ function App() {
|
|||||||
<span>当前用户侧脚本已全部映射到网页任务。</span>
|
<span>当前用户侧脚本已全部映射到网页任务。</span>
|
||||||
<span>最近验收:{acceptance?.created_at ?? "尚未运行"} {acceptance?.run_id ? `#${acceptance.run_id}` : ""}</span>
|
<span>最近验收:{acceptance?.created_at ?? "尚未运行"} {acceptance?.run_id ? `#${acceptance.run_id}` : ""}</span>
|
||||||
<span>真实数据:{realAcceptance?.created_at ?? "尚未运行"} {realAcceptance?.run_id ? `#${realAcceptance.run_id}` : ""},通过 {realAcceptance?.checks?.filter((item) => item.passed).length ?? 0}/{realAcceptance?.checks?.length ?? 0}</span>
|
<span>真实数据:{realAcceptance?.created_at ?? "尚未运行"} {realAcceptance?.run_id ? `#${realAcceptance.run_id}` : ""},通过 {realAcceptance?.checks?.filter((item) => item.passed).length ?? 0}/{realAcceptance?.checks?.length ?? 0}</span>
|
||||||
|
<span>真实短训:{realTrainAcceptance?.created_at ?? "尚未运行"} {realTrainAcceptance?.run_id ? `#${realTrainAcceptance.run_id}` : ""},通过 {realTrainAcceptance?.checks?.filter((item) => item.passed).length ?? 0}/{realTrainAcceptance?.checks?.length ?? 0}</span>
|
||||||
<span>深度验收:{deepAcceptance?.created_at ?? "尚未运行"} {deepAcceptance?.run_id ? `#${deepAcceptance.run_id}` : ""},通过 {deepAcceptance?.checks?.filter((item) => item.passed).length ?? 0}/{deepAcceptance?.checks?.length ?? 0}</span>
|
<span>深度验收:{deepAcceptance?.created_at ?? "尚未运行"} {deepAcceptance?.run_id ? `#${deepAcceptance.run_id}` : ""},通过 {deepAcceptance?.checks?.filter((item) => item.passed).length ?? 0}/{deepAcceptance?.checks?.length ?? 0}</span>
|
||||||
<span>模型族 readiness:{acceptance?.model_family_readiness?.checks?.filter((item) => item.passed).length ?? 0}/{acceptance?.model_family_readiness?.checks?.length ?? 0},warnings {acceptance?.model_family_readiness?.warnings?.length ?? 0}</span>
|
<span>模型族 readiness:{acceptance?.model_family_readiness?.checks?.filter((item) => item.passed).length ?? 0}/{acceptance?.model_family_readiness?.checks?.length ?? 0},warnings {acceptance?.model_family_readiness?.warnings?.length ?? 0}</span>
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ def main() -> None:
|
|||||||
parser.add_argument("--live", action="store_true", help="also check live backend/frontend HTTP endpoints")
|
parser.add_argument("--live", action="store_true", help="also check live backend/frontend HTTP endpoints")
|
||||||
parser.add_argument("--acceptance", action="store_true", help="run the lightweight live acceptance smoke")
|
parser.add_argument("--acceptance", action="store_true", help="run the lightweight live acceptance smoke")
|
||||||
parser.add_argument("--real", action="store_true", help="run real workspace data acceptance through the live backend")
|
parser.add_argument("--real", action="store_true", help="run real workspace data acceptance through the live backend")
|
||||||
|
parser.add_argument("--real-train", action="store_true", help="run a short real workspace YOLO train/predict/heatmap acceptance")
|
||||||
parser.add_argument("--no-deep", action="store_true", help="skip synthetic deep training acceptance")
|
parser.add_argument("--no-deep", action="store_true", help="skip synthetic deep training acceptance")
|
||||||
parser.add_argument("--out", default="var/agent_reports/latest.json")
|
parser.add_argument("--out", default="var/agent_reports/latest.json")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -29,6 +30,7 @@ def main() -> None:
|
|||||||
run_live=args.live,
|
run_live=args.live,
|
||||||
run_acceptance=args.acceptance,
|
run_acceptance=args.acceptance,
|
||||||
run_real=args.real,
|
run_real=args.real,
|
||||||
|
run_real_train=args.real_train,
|
||||||
run_deep=not args.no_deep,
|
run_deep=not args.no_deep,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user