Add live acceptance smoke validation
This commit is contained in:
@@ -51,6 +51,11 @@ scripts from the existing `Seg/` workspace are mapped to web jobs. MMSeg
|
||||
vendored internals, docs, build outputs, converters, and config templates are
|
||||
classified as supporting artifacts rather than direct web actions.
|
||||
|
||||
The same panel can run `POST /api/acceptance/smoke`, a lightweight live smoke
|
||||
that creates an upload dataset, uploads a label, downloads it through the
|
||||
artifact API, runs a mock job, checks SSE log streaming, and executes one
|
||||
legacy image/label overlay job on tiny generated PNGs.
|
||||
|
||||
## Weight Sync
|
||||
|
||||
The current workspace contains tens of GB of pretrained and trained weights.
|
||||
@@ -107,4 +112,5 @@ PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build
|
||||
|
||||
The validation agent checks catalog coverage, the new `seg_smp` env, GPU
|
||||
visibility, no-weight Git safety, backend tests, frontend build, and live
|
||||
backend/frontend endpoints when the services are running.
|
||||
backend/frontend endpoints when the services are running. With live validation
|
||||
enabled it also runs the lightweight acceptance smoke above.
|
||||
|
||||
201
backend/app/acceptance.py
Normal file
201
backend/app/acceptance.py
Normal file
@@ -0,0 +1,201 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .config import settings
|
||||
|
||||
|
||||
def _request_json(method: str, url: str, payload: dict[str, Any] | None = None, timeout: int = 10) -> dict[str, Any]:
|
||||
data = None
|
||||
headers = {"Accept": "application/json"}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
return {"passed": 200 <= response.status < 300, "status": response.status, "body": body, "json": json.loads(body) if body else None}
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
return {"passed": False, "status": exc.code, "body": body}
|
||||
except Exception as exc:
|
||||
return {"passed": False, "error": str(exc)}
|
||||
|
||||
|
||||
def _request_text(url: str, timeout: int = 10) -> dict[str, Any]:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
return {"passed": 200 <= response.status < 300, "status": response.status, "body": body}
|
||||
except Exception as exc:
|
||||
return {"passed": False, "error": str(exc)}
|
||||
|
||||
|
||||
def _post_multipart(url: str, field: str, filename: str, content: bytes, content_type: str = "text/plain", timeout: int = 10) -> dict[str, Any]:
|
||||
boundary = f"----SegAcceptance{uuid.uuid4().hex}"
|
||||
body = b"".join(
|
||||
[
|
||||
f"--{boundary}\r\n".encode(),
|
||||
f'Content-Disposition: form-data; name="{field}"; filename="{filename}"\r\n'.encode(),
|
||||
f"Content-Type: {content_type}\r\n\r\n".encode(),
|
||||
content,
|
||||
f"\r\n--{boundary}--\r\n".encode(),
|
||||
]
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": f"multipart/form-data; boundary={boundary}", "Accept": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
text = response.read().decode("utf-8", errors="replace")
|
||||
return {"passed": 200 <= response.status < 300, "status": response.status, "body": text, "json": json.loads(text) if text else None}
|
||||
except urllib.error.HTTPError as exc:
|
||||
text = exc.read().decode("utf-8", errors="replace")
|
||||
return {"passed": False, "status": exc.code, "body": text}
|
||||
except Exception as exc:
|
||||
return {"passed": False, "error": str(exc)}
|
||||
|
||||
|
||||
def _poll_job(base_url: str, job_id: str, timeout: int = 90) -> dict[str, Any]:
|
||||
deadline = time.time() + timeout
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
result = _request_json("GET", f"{base_url}/api/jobs/{job_id}", timeout=10)
|
||||
last = result
|
||||
job = result.get("json") if result.get("passed") else None
|
||||
if job and job.get("status") in {"success", "failed", "cancelled"}:
|
||||
return {"passed": job.get("status") == "success", "job": job}
|
||||
time.sleep(1)
|
||||
return {"passed": False, "error": "job timed out", "last": last}
|
||||
|
||||
|
||||
def _create_job_and_wait(base_url: str, task_type: str, params: dict[str, Any], timeout: int = 90) -> dict[str, Any]:
|
||||
created = _request_json("POST", f"{base_url}/api/jobs", {"type": task_type, "params": params}, timeout=10)
|
||||
if not created.get("passed") or not created.get("json"):
|
||||
return {"passed": False, "created": created}
|
||||
job_id = created["json"]["id"]
|
||||
polled = _poll_job(base_url, job_id, timeout=timeout)
|
||||
events = _request_text(f"{base_url}/api/jobs/{job_id}/events", timeout=10)
|
||||
return {"passed": polled["passed"], "created": created["json"], "polled": polled, "events": events}
|
||||
|
||||
|
||||
def _create_job_with_retry(base_url: str, task_type: str, params: dict[str, Any], attempts: int = 2, timeout: int = 90) -> dict[str, Any]:
|
||||
results = []
|
||||
for _ in range(attempts):
|
||||
result = _create_job_and_wait(base_url, task_type, params, timeout=timeout)
|
||||
results.append(result)
|
||||
if result.get("passed"):
|
||||
return {"passed": True, "attempts": results}
|
||||
time.sleep(1)
|
||||
return {"passed": False, "attempts": results}
|
||||
|
||||
|
||||
def _write_acceptance_images(root: Path) -> tuple[Path, Path, Path]:
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
image_dir = root / "images"
|
||||
label_dir = root / "labels"
|
||||
result_dir = root / "stacked"
|
||||
image_dir.mkdir(parents=True, exist_ok=True)
|
||||
label_dir.mkdir(parents=True, exist_ok=True)
|
||||
result_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image = np.zeros((16, 16, 3), dtype=np.uint8)
|
||||
image[:, :, 1] = 180
|
||||
label = np.zeros((16, 16, 3), dtype=np.uint8)
|
||||
label[4:12, 4:12, 2] = 255
|
||||
image_path = image_dir / "sample.png"
|
||||
label_path = label_dir / "sample.png"
|
||||
cv2.imwrite(str(image_path), image)
|
||||
cv2.imwrite(str(label_path), label)
|
||||
return image_path, label_path, result_dir
|
||||
|
||||
|
||||
def latest_acceptance_report() -> dict[str, Any]:
|
||||
path = settings.project_root / "var" / "acceptance" / "latest.json"
|
||||
if not path.exists():
|
||||
return {"available": False, "path": str(path)}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def run_live_acceptance(base_url: str = "http://127.0.0.1:8010") -> dict[str, Any]:
|
||||
"""Run a lightweight end-to-end smoke against the live API and job runner."""
|
||||
acceptance_root = settings.project_root / "var" / "acceptance"
|
||||
run_id = uuid.uuid4().hex[:8]
|
||||
fixture_root = acceptance_root / f"run_{run_id}"
|
||||
fixture_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
checks: list[dict[str, Any]] = []
|
||||
|
||||
dataset_name = f"acceptance_{run_id}"
|
||||
created_dataset = _request_json("POST", f"{base_url}/api/datasets", {"name": dataset_name, "description": "acceptance smoke"}, timeout=10)
|
||||
checks.append({"name": "create_dataset_api", "passed": created_dataset.get("passed", False), "detail": created_dataset})
|
||||
|
||||
upload = _post_multipart(
|
||||
f"{base_url}/api/datasets/{dataset_name}/upload/labels",
|
||||
"files",
|
||||
"label 01.txt",
|
||||
b"0 0.5 0.5 0.25 0.25\n",
|
||||
"text/plain",
|
||||
timeout=10,
|
||||
)
|
||||
checks.append({"name": "upload_label_api", "passed": upload.get("passed", False), "detail": upload})
|
||||
|
||||
artifact_ok = False
|
||||
artifact_detail: dict[str, Any] = {"skipped": True}
|
||||
try:
|
||||
relative_path = upload["json"]["saved"][0]["relative_path"]
|
||||
artifact_detail = _request_text(f"{base_url}/api/artifacts/{relative_path}", timeout=10)
|
||||
artifact_ok = artifact_detail.get("passed", False) and "0 0.5" in artifact_detail.get("body", "")
|
||||
except Exception as exc:
|
||||
artifact_detail = {"error": str(exc)}
|
||||
checks.append({"name": "artifact_api_for_uploaded_label", "passed": artifact_ok, "detail": artifact_detail})
|
||||
|
||||
mock = _create_job_and_wait(base_url, "mock.echo", {"message": f"acceptance {run_id}"}, timeout=45)
|
||||
mock_log = mock.get("polled", {}).get("job", {}).get("log_tail", "")
|
||||
checks.append({"name": "mock_job_runner", "passed": mock.get("passed", False) and f"acceptance {run_id}" in mock_log, "detail": mock})
|
||||
|
||||
image_path, label_path, result_dir = _write_acceptance_images(fixture_root)
|
||||
stack = _create_job_with_retry(
|
||||
base_url,
|
||||
"dataset.stack_single",
|
||||
{
|
||||
"image_path": str(image_path),
|
||||
"label_path": str(label_path),
|
||||
"result_dir": str(result_dir),
|
||||
"alpha": 0.35,
|
||||
},
|
||||
attempts=2,
|
||||
timeout=90,
|
||||
)
|
||||
output_path = result_dir / "sample.png"
|
||||
checks.append(
|
||||
{
|
||||
"name": "legacy_stack_job_runner",
|
||||
"passed": stack.get("passed", False) and output_path.exists() and output_path.stat().st_size > 0,
|
||||
"detail": {**stack, "output_path": str(output_path), "output_exists": output_path.exists()},
|
||||
}
|
||||
)
|
||||
|
||||
report = {
|
||||
"available": True,
|
||||
"run_id": run_id,
|
||||
"base_url": base_url,
|
||||
"passed": all(item["passed"] for item in checks),
|
||||
"checks": checks,
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}
|
||||
latest = acceptance_root / "latest.json"
|
||||
latest.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return report
|
||||
@@ -8,6 +8,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ..acceptance import run_live_acceptance
|
||||
from ..catalog import get_catalog
|
||||
from ..config import settings
|
||||
from ..coverage import get_coverage_report
|
||||
@@ -84,6 +85,9 @@ def validate_project(run_build: bool = False) -> dict:
|
||||
checks.append({"name": "live_dataset_api", "passed": datasets["passed"] and datasets.get("body", "").lstrip().startswith("["), "detail": datasets})
|
||||
checks.append({"name": "live_coverage_api", "passed": live_coverage["passed"] and '"task_build_passed":true' in live_coverage.get("body", "").replace(" ", ""), "detail": live_coverage})
|
||||
checks.append({"name": "live_frontend_index", "passed": frontend["passed"] and "Seg Data Server" in frontend.get("body", ""), "detail": frontend})
|
||||
if os.getenv("SEG_VALIDATE_ACCEPTANCE", "1") == "1":
|
||||
acceptance = run_live_acceptance(backend_url)
|
||||
checks.append({"name": "live_acceptance_smoke", "passed": acceptance["passed"], "detail": acceptance})
|
||||
|
||||
if run_build:
|
||||
tests = _run(["conda", "run", "-n", settings.backend_conda_env, "python", "-m", "pytest", "-q"], cwd=settings.project_root, timeout=120)
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
|
||||
from . import db
|
||||
from .acceptance import latest_acceptance_report, run_live_acceptance
|
||||
from .catalog import get_catalog
|
||||
from .config import settings
|
||||
from .coverage import get_coverage_report
|
||||
@@ -69,6 +70,16 @@ def api_coverage() -> dict:
|
||||
return get_coverage_report()
|
||||
|
||||
|
||||
@app.get("/api/acceptance/latest")
|
||||
def api_acceptance_latest() -> dict:
|
||||
return latest_acceptance_report()
|
||||
|
||||
|
||||
@app.post("/api/acceptance/smoke")
|
||||
def api_acceptance_smoke(base_url: str = "http://127.0.0.1:8010") -> dict:
|
||||
return run_live_acceptance(base_url)
|
||||
|
||||
|
||||
@app.get("/api/datasets")
|
||||
def api_datasets() -> list[dict]:
|
||||
return list_uploaded_datasets()
|
||||
@@ -163,6 +174,10 @@ def api_results() -> list[dict]:
|
||||
def api_artifact(artifact_path: str):
|
||||
candidate = Path(artifact_path)
|
||||
if not candidate.is_absolute():
|
||||
first_part = candidate.parts[0] if candidate.parts else ""
|
||||
if first_part in {"var", "weights"}:
|
||||
candidate = settings.project_root / candidate
|
||||
else:
|
||||
candidate = settings.source_root / candidate
|
||||
try:
|
||||
resolved = candidate.resolve()
|
||||
|
||||
34
backend/tests/test_artifacts.py
Normal file
34
backend/tests/test_artifacts.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
def test_project_var_artifact_path_is_served(tmp_path, monkeypatch):
|
||||
from app import main
|
||||
|
||||
project_root = tmp_path / "project"
|
||||
source_root = tmp_path / "source"
|
||||
artifact = project_root / "var" / "uploads" / "datasets" / "case" / "labels" / "label.txt"
|
||||
artifact.parent.mkdir(parents=True)
|
||||
artifact.write_text("ok", encoding="utf-8")
|
||||
source_root.mkdir()
|
||||
|
||||
original = main.settings
|
||||
patched = type(original)(
|
||||
project_root=project_root,
|
||||
source_root=source_root,
|
||||
db_path=original.db_path,
|
||||
log_dir=original.log_dir,
|
||||
weights_root=original.weights_root,
|
||||
task_conda_env=original.task_conda_env,
|
||||
backend_conda_env=original.backend_conda_env,
|
||||
weight_mode=original.weight_mode,
|
||||
enable_shell_tasks=original.enable_shell_tasks,
|
||||
)
|
||||
monkeypatch.setattr(main, "settings", patched)
|
||||
|
||||
response = TestClient(app).get("/api/artifacts/var/uploads/datasets/case/labels/label.txt")
|
||||
assert response.status_code == 200
|
||||
assert response.text == "ok"
|
||||
@@ -72,6 +72,14 @@ type CoveragePayload = {
|
||||
task_build_checks: Array<{ task: string; passed: boolean; script_exists?: boolean; error?: string }>;
|
||||
};
|
||||
|
||||
type AcceptancePayload = {
|
||||
available?: boolean;
|
||||
passed?: boolean;
|
||||
run_id?: string;
|
||||
created_at?: string;
|
||||
checks?: Array<{ name: string; passed: boolean }>;
|
||||
};
|
||||
|
||||
type GpuPayload = {
|
||||
available: boolean;
|
||||
gpus: Array<{
|
||||
@@ -152,17 +160,19 @@ function useData() {
|
||||
const [results, setResults] = useState<ResultItem[]>([]);
|
||||
const [datasets, setDatasets] = useState<UploadedDataset[]>([]);
|
||||
const [coverage, setCoverage] = useState<CoveragePayload | null>(null);
|
||||
const [acceptance, setAcceptance] = useState<AcceptancePayload | null>(null);
|
||||
const [error, setError] = useState<string>("");
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
const [catalogNext, gpusNext, jobsNext, resultsNext, datasetsNext, coverageNext] = await Promise.all([
|
||||
const [catalogNext, gpusNext, jobsNext, resultsNext, datasetsNext, coverageNext, acceptanceNext] = await Promise.all([
|
||||
api<Catalog>("/api/catalog"),
|
||||
api<GpuPayload>("/api/system/gpus"),
|
||||
api<Job[]>("/api/jobs"),
|
||||
api<ResultItem[]>("/api/results"),
|
||||
api<UploadedDataset[]>("/api/datasets"),
|
||||
api<CoveragePayload>("/api/coverage")
|
||||
api<CoveragePayload>("/api/coverage"),
|
||||
api<AcceptancePayload>("/api/acceptance/latest")
|
||||
]);
|
||||
setCatalog(catalogNext);
|
||||
setGpus(gpusNext);
|
||||
@@ -170,6 +180,7 @@ function useData() {
|
||||
setResults(resultsNext.slice(0, 80));
|
||||
setDatasets(datasetsNext);
|
||||
setCoverage(coverageNext);
|
||||
setAcceptance(acceptanceNext);
|
||||
setError("");
|
||||
} catch (err) {
|
||||
setError(String(err));
|
||||
@@ -182,7 +193,7 @@ function useData() {
|
||||
return () => window.clearInterval(timer);
|
||||
}, []);
|
||||
|
||||
return { catalog, gpus, jobs, results, datasets, coverage, error, refresh };
|
||||
return { catalog, gpus, jobs, results, datasets, coverage, acceptance, error, refresh };
|
||||
}
|
||||
|
||||
function StatusPill({ status }: { status: string }) {
|
||||
@@ -190,7 +201,7 @@ function StatusPill({ status }: { status: string }) {
|
||||
}
|
||||
|
||||
function App() {
|
||||
const { catalog, gpus, jobs, results, datasets, coverage, error, refresh } = useData();
|
||||
const { catalog, gpus, jobs, results, datasets, coverage, acceptance, error, refresh } = useData();
|
||||
const [taskType, setTaskType] = useState("mock.echo");
|
||||
const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2));
|
||||
const [selectedJob, setSelectedJob] = useState<Job | null>(null);
|
||||
@@ -255,6 +266,16 @@ function App() {
|
||||
}
|
||||
}
|
||||
|
||||
async function runAcceptanceSmoke() {
|
||||
setBusy(true);
|
||||
try {
|
||||
await api("/api/acceptance/smoke", { method: "POST" });
|
||||
await refresh();
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function createDataset() {
|
||||
setBusy(true);
|
||||
try {
|
||||
@@ -497,7 +518,9 @@ function App() {
|
||||
<p className="eyebrow">Coverage</p>
|
||||
<h2>Seg 功能覆盖</h2>
|
||||
</div>
|
||||
<ClipboardCheck size={22} />
|
||||
<button className="iconButton" disabled={busy} onClick={runAcceptanceSmoke} title="运行轻量验收">
|
||||
<ClipboardCheck size={18} />
|
||||
</button>
|
||||
</div>
|
||||
<div className="coverageGrid">
|
||||
<div>
|
||||
@@ -512,10 +535,17 @@ function App() {
|
||||
<span>任务构建</span>
|
||||
<strong>{coverage?.task_build_passed ? "OK" : "Check"}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>轻量验收</span>
|
||||
<strong>{acceptance?.available === false ? "New" : acceptance?.passed ? "OK" : "Check"}</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div className="coverageStatus">
|
||||
{(coverage?.unmapped_user_scripts.length ?? 0) === 0 ? (
|
||||
<>
|
||||
<span>当前用户侧脚本已全部映射到网页任务。</span>
|
||||
<span>最近验收：{acceptance?.created_at ?? "尚未运行"} {acceptance?.run_id ? `#${acceptance.run_id}` : ""}</span>
|
||||
</>
|
||||
) : (
|
||||
coverage?.unmapped_user_scripts.slice(0, 8).map((item) => <code key={item}>{item}</code>)
|
||||
)}
|
||||
|
||||
@@ -399,7 +399,7 @@ textarea {
|
||||
|
||||
.coverageGrid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 10px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user