Stream job logs from current offset
This commit is contained in:
@@ -59,6 +59,11 @@ def evaluate_project() -> dict:
|
||||
and "setResults(resultsNext)" in frontend_text
|
||||
and "slice(0, 240)" not in frontend_text,
|
||||
"job_progress_ui": "JobProgressBar" in frontend_text and "progressTrack" in frontend_text,
|
||||
"live_log_stream_ui": "EventSource" in frontend_text
|
||||
and "eventSourceRef" in frontend_text
|
||||
and "log_size" in frontend_text
|
||||
and "events?offset=" in frontend_text,
|
||||
"live_log_offset_api": "log_size" in backend_text and "offset: int = Query(0, ge=0)" in backend_text,
|
||||
"runtime_readiness_ui": "runtimeReadiness" in frontend_text and "环境就绪" in frontend_text,
|
||||
"capability_matrix_ui": "capabilities" in frontend_text and "全功能矩阵" in frontend_text,
|
||||
"dataset_api": "/api/datasets" in backend_text and "api_upload_dataset_files" in backend_text,
|
||||
|
||||
@@ -39,6 +39,8 @@ app.add_middleware(
|
||||
def _job_with_progress(job: dict, include_log_tail: bool = False) -> dict:
|
||||
enriched = dict(job)
|
||||
max_bytes = 65536 if include_log_tail else 32768
|
||||
log_path = Path(enriched["log_path"])
|
||||
enriched["log_size"] = log_path.stat().st_size if log_path.exists() else 0
|
||||
enriched["progress"] = progress_from_log_path(enriched["log_path"], enriched["status"], max_bytes=max_bytes)
|
||||
if include_log_tail:
|
||||
enriched["log_tail"] = db.log_tail(enriched["log_path"], max_bytes=max_bytes)
|
||||
@@ -183,9 +185,9 @@ def api_cancel_job(job_id: str) -> dict:
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/events")
|
||||
async def api_job_events(job_id: str):
|
||||
async def api_job_events(job_id: str, offset: int = Query(0, ge=0)):
|
||||
async def stream():
|
||||
last_size = 0
|
||||
last_size = offset
|
||||
while True:
|
||||
job = db.get_job(job_id)
|
||||
if not job:
|
||||
@@ -196,6 +198,8 @@ async def api_job_events(job_id: str):
|
||||
chunk = ""
|
||||
if path.exists():
|
||||
size = path.stat().st_size
|
||||
if last_size > size:
|
||||
last_size = size
|
||||
if size > last_size:
|
||||
with path.open("rb") as handle:
|
||||
handle.seek(last_size)
|
||||
|
||||
@@ -1,6 +1,38 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.jobs import default_conda_env_for_job
|
||||
from app.main import _job_with_progress, app
|
||||
|
||||
|
||||
def test_mmseg_jobs_use_mmseg_conda_env_by_default():
|
||||
assert default_conda_env_for_job("mmseg.train") == "seg_mmcv"
|
||||
assert default_conda_env_for_job("segmodel.train") == "seg_smp"
|
||||
|
||||
|
||||
def test_job_progress_reports_log_size(tmp_path):
|
||||
log_path = tmp_path / "job.log"
|
||||
log_path.write_text("line one\nline two\n", encoding="utf-8")
|
||||
job = {"id": "job1", "status": "running", "log_path": str(log_path)}
|
||||
|
||||
enriched = _job_with_progress(job, include_log_tail=True)
|
||||
|
||||
assert enriched["log_size"] == log_path.stat().st_size
|
||||
assert enriched["log_tail"] == "line one\nline two\n"
|
||||
|
||||
|
||||
def test_job_events_respect_log_offset(tmp_path, monkeypatch):
|
||||
from app import main
|
||||
|
||||
log_path = tmp_path / "job.log"
|
||||
old_chunk = "old chunk\n"
|
||||
log_path.write_text(old_chunk + "new chunk\n", encoding="utf-8")
|
||||
|
||||
def fake_get_job(job_id):
|
||||
return {"id": job_id, "status": "success", "log_path": str(log_path)}
|
||||
|
||||
monkeypatch.setattr(main.db, "get_job", fake_get_job)
|
||||
response = TestClient(app).get(f"/api/jobs/job1/events?offset={len(old_chunk.encode('utf-8'))}")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "new chunk" in response.text
|
||||
assert "old chunk" not in response.text
|
||||
|
||||
Reference in New Issue
Block a user