278 lines
8.7 KiB
Python
278 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from fastapi import FastAPI, File, HTTPException, Query, UploadFile
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import FileResponse, StreamingResponse
|
|
|
|
from . import db
|
|
from .acceptance import latest_acceptance_report, latest_deep_acceptance_report, run_deep_acceptance, run_live_acceptance
|
|
from .capabilities import get_capability_matrix
|
|
from .catalog import get_catalog
|
|
from .config import settings
|
|
from .coverage import get_coverage_report
|
|
from .jobs import cancel_job, create_job
|
|
from .modules.results.service import scan_results, scan_training_curves
|
|
from .modules.system.service import disk_usage, get_conda_envs, get_gpus, get_runtime_readiness
|
|
from .modules.dataset.service import create_dataset, generate_yolo_dataset_yaml, list_uploaded_datasets, save_upload, validate_dataset
|
|
from .modules.weights.service import load_manifest, sync_weights, verify_weights
|
|
from .agents.evaluation_agent import evaluate_project
|
|
from .agents.validation_agent import validate_project
|
|
from .paths import ensure_inside
|
|
from .progress import progress_from_log_path
|
|
from .schemas import DatasetCreate, DatasetYoloYamlRequest, JobCreate, ProfileCreate, WeightSyncRequest
|
|
|
|
app = FastAPI(title="Seg Data Server Net", version="0.1.0")
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
|
|
def _job_with_progress(job: dict, include_log_tail: bool = False) -> dict:
|
|
enriched = dict(job)
|
|
max_bytes = 65536 if include_log_tail else 32768
|
|
log_path = Path(enriched["log_path"])
|
|
enriched["log_size"] = log_path.stat().st_size if log_path.exists() else 0
|
|
enriched["progress"] = progress_from_log_path(enriched["log_path"], enriched["status"], max_bytes=max_bytes)
|
|
if include_log_tail:
|
|
enriched["log_tail"] = db.log_tail(enriched["log_path"], max_bytes=max_bytes)
|
|
return enriched
|
|
|
|
|
|
@app.on_event("startup")
|
|
def startup() -> None:
|
|
db.init_db()
|
|
settings.log_dir.mkdir(parents=True, exist_ok=True)
|
|
settings.weights_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
@app.get("/api/health")
|
|
def health() -> dict:
|
|
return {
|
|
"ok": True,
|
|
"source_root": str(settings.source_root),
|
|
"project_root": str(settings.project_root),
|
|
"disk": disk_usage(),
|
|
}
|
|
|
|
|
|
@app.get("/api/system/gpus")
|
|
def api_gpus() -> dict:
|
|
return get_gpus()
|
|
|
|
|
|
@app.get("/api/system/envs")
|
|
def api_envs() -> dict:
|
|
return get_conda_envs()
|
|
|
|
|
|
@app.get("/api/system/readiness")
|
|
def api_runtime_readiness(refresh: bool = False) -> dict:
|
|
return get_runtime_readiness(force=refresh)
|
|
|
|
|
|
@app.get("/api/catalog")
|
|
def api_catalog() -> dict:
|
|
return get_catalog()
|
|
|
|
|
|
@app.get("/api/coverage")
|
|
def api_coverage() -> dict:
|
|
return get_coverage_report()
|
|
|
|
|
|
@app.get("/api/capabilities")
|
|
def api_capabilities() -> dict:
|
|
return get_capability_matrix()
|
|
|
|
|
|
@app.get("/api/acceptance/latest")
|
|
def api_acceptance_latest() -> dict:
|
|
return latest_acceptance_report()
|
|
|
|
|
|
@app.post("/api/acceptance/smoke")
|
|
def api_acceptance_smoke(base_url: str = "http://127.0.0.1:8010") -> dict:
|
|
return run_live_acceptance(base_url)
|
|
|
|
|
|
@app.get("/api/acceptance/deep/latest")
|
|
def api_deep_acceptance_latest() -> dict:
|
|
return latest_deep_acceptance_report()
|
|
|
|
|
|
@app.post("/api/acceptance/deep")
|
|
def api_deep_acceptance() -> dict:
|
|
return run_deep_acceptance()
|
|
|
|
|
|
@app.get("/api/datasets")
|
|
def api_datasets() -> list[dict]:
|
|
return list_uploaded_datasets()
|
|
|
|
|
|
@app.post("/api/datasets")
|
|
def api_create_dataset(dataset: DatasetCreate) -> dict:
|
|
return create_dataset(dataset.name, dataset.description)
|
|
|
|
|
|
@app.post("/api/datasets/{dataset_name}/upload/{kind}")
|
|
async def api_upload_dataset_files(dataset_name: str, kind: str, files: list[UploadFile] = File(...)) -> dict:
|
|
try:
|
|
return await save_upload(dataset_name, kind, files)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/api/datasets/{dataset_name}/validate")
|
|
def api_validate_dataset(dataset_name: str) -> dict:
|
|
return validate_dataset(dataset_name)
|
|
|
|
|
|
@app.post("/api/datasets/{dataset_name}/yolo-yaml")
|
|
def api_generate_dataset_yolo_yaml(dataset_name: str, request: DatasetYoloYamlRequest) -> dict:
|
|
try:
|
|
return generate_yolo_dataset_yaml(dataset_name, request.class_names)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/api/profiles")
|
|
def api_profiles(kind: str | None = None) -> list[dict]:
|
|
return db.list_profiles(kind)
|
|
|
|
|
|
@app.post("/api/profiles")
|
|
def api_save_profile(profile: ProfileCreate) -> dict:
|
|
return db.upsert_profile(profile.name, profile.kind, profile.data)
|
|
|
|
|
|
@app.post("/api/jobs")
|
|
def api_create_job(request: JobCreate) -> dict:
|
|
try:
|
|
return _job_with_progress(create_job(request))
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
|
|
|
|
@app.get("/api/jobs")
|
|
def api_jobs(limit: int = 100) -> list[dict]:
|
|
return [_job_with_progress(job) for job in db.list_jobs(limit)]
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}")
|
|
def api_job(job_id: str) -> dict:
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="job not found")
|
|
return _job_with_progress(job, include_log_tail=True)
|
|
|
|
|
|
@app.post("/api/jobs/{job_id}/cancel")
|
|
def api_cancel_job(job_id: str) -> dict:
|
|
job = cancel_job(job_id)
|
|
if not job:
|
|
raise HTTPException(status_code=404, detail="job not found")
|
|
return _job_with_progress(job)
|
|
|
|
|
|
@app.get("/api/jobs/{job_id}/events")
|
|
async def api_job_events(job_id: str, offset: int = Query(0, ge=0)):
|
|
async def stream():
|
|
last_size = offset
|
|
while True:
|
|
job = db.get_job(job_id)
|
|
if not job:
|
|
yield "event: error\ndata: job not found\n\n"
|
|
return
|
|
job = _job_with_progress(job)
|
|
path = Path(job["log_path"])
|
|
chunk = ""
|
|
if path.exists():
|
|
size = path.stat().st_size
|
|
if last_size > size:
|
|
last_size = size
|
|
if size > last_size:
|
|
with path.open("rb") as handle:
|
|
handle.seek(last_size)
|
|
chunk = handle.read(size - last_size).decode("utf-8", errors="replace")
|
|
last_size = size
|
|
payload = {"job": job, "chunk": chunk}
|
|
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
|
if job["status"] in {"success", "failed", "cancelled"}:
|
|
return
|
|
await asyncio.sleep(1)
|
|
|
|
return StreamingResponse(stream(), media_type="text/event-stream")
|
|
|
|
|
|
@app.get("/api/results")
|
|
def api_results(limit: int = Query(1000, ge=1, le=5000)) -> list[dict]:
|
|
return scan_results(limit=limit)
|
|
|
|
|
|
@app.get("/api/results/curves")
|
|
def api_result_curves(limit: int = Query(100, ge=1, le=500)) -> list[dict]:
|
|
return scan_training_curves(limit=limit)
|
|
|
|
|
|
@app.get("/api/artifacts/{artifact_path:path}")
|
|
def api_artifact(artifact_path: str):
|
|
candidate = Path(artifact_path)
|
|
if not candidate.is_absolute():
|
|
first_part = candidate.parts[0] if candidate.parts else ""
|
|
if first_part in {"var", "weights"}:
|
|
candidate = settings.project_root / candidate
|
|
else:
|
|
candidate = settings.source_root / candidate
|
|
try:
|
|
resolved = candidate.resolve()
|
|
allowed = False
|
|
for root in (settings.source_root, settings.project_root):
|
|
try:
|
|
ensure_inside(resolved, root)
|
|
allowed = True
|
|
break
|
|
except Exception:
|
|
continue
|
|
if not allowed:
|
|
raise ValueError("artifact path is outside allowed roots")
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
if not resolved.exists() or not resolved.is_file():
|
|
raise HTTPException(status_code=404, detail="artifact not found")
|
|
return FileResponse(resolved)
|
|
|
|
|
|
@app.get("/api/weights")
|
|
def api_weights() -> dict:
|
|
return load_manifest()
|
|
|
|
|
|
@app.post("/api/weights/sync")
|
|
def api_weight_sync(request: WeightSyncRequest) -> dict:
|
|
return sync_weights(request.mode, request.hash_files, request.skip_existing)
|
|
|
|
|
|
@app.post("/api/weights/verify")
|
|
def api_weight_verify() -> dict:
|
|
return verify_weights()
|
|
|
|
|
|
@app.get("/api/agents/evaluate")
|
|
def api_agent_evaluate() -> dict:
|
|
return evaluate_project()
|
|
|
|
|
|
@app.get("/api/agents/validate")
|
|
def api_agent_validate(run_build: bool = False, run_acceptance: bool = False, run_deep: bool = False) -> dict:
|
|
return validate_project(run_build=run_build, run_acceptance=run_acceptance, run_deep=run_deep)
|