From 1d3891d949ead1c0303cd0dc3e66261ca933f624 Mon Sep 17 00:00:00 2001 From: admin <572701190@qq.com> Date: Tue, 30 Jun 2026 16:53:55 +0800 Subject: [PATCH] Document deployment and expose weight manifest --- README.md | 123 ++++++++++++++++++++++-- backend/app/agents/evaluation_agent.py | 5 + backend/app/config.py | 34 +++++-- backend/tests/test_config.py | 28 ++++++ frontend/src/main.tsx | 126 +++++++++++++++++++++---- frontend/src/styles.css | 69 ++++++++++++++ scripts/run_backend.sh | 7 ++ scripts/run_frontend.sh | 8 +- 8 files changed, 367 insertions(+), 33 deletions(-) create mode 100644 backend/tests/test_config.py diff --git a/README.md b/README.md index f1c2eb7..fdcc2db 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,124 @@ cp .env.example .env # Create or repair the two runtime environments, then verify imports. scripts/bootstrap_conda_envs.sh -# Backend. The deployment env is seg_smp so the API and most task wrappers -# share the same segmentation dependency stack. MMSeg jobs default to the -# separate SEG_MMSEG_CONDA_ENV because full mmcv wheels must match torch/CUDA. -conda run -n seg_smp uvicorn app.main:app --app-dir backend --host 0.0.0.0 --port 8010 +# Backend. The script loads .env and uses SEG_BACKEND_CONDA_ENV/PORT/HOST. +scripts/run_backend.sh -# Frontend. -cd frontend -npm install -npm run dev -- --host 0.0.0.0 +# Frontend. The script loads .env, installs npm packages, and starts Vite. +scripts/run_frontend.sh ``` Open the Vite URL shown in the terminal. The frontend expects the backend at `http://localhost:8010` by default. +## Deployment and Environment + +The expected deployment layout keeps the original algorithm workspace next to +this web project: + +```text +/home/wkmgc/Desktop/Data_Disk_1/Seg/ + Seg/ existing algorithms, datasets, logs, and raw outputs + Seg_Data_Server_Net/ this FastAPI + React control plane +``` + +Clone or update the web project from Gitea: + +```bash +git clone https://gitea.huijutec.cn/admin/Seg_Data_Server_Net.git +cd Seg_Data_Server_Net +cp .env.example .env +``` + +Edit `.env` for the server before first boot. Relative paths are resolved from +the `Seg_Data_Server_Net/` project root; absolute paths are safest when moving +to another machine. + +```bash +SEG_SOURCE_ROOT=../Seg +SEG_DATA_SERVER_ROOT=. +SEG_BACKEND_DB=var/seg_data_server.sqlite3 +SEG_BACKEND_LOG_DIR=var/job_logs +SEG_TASK_CONDA_ENV=seg_smp +SEG_MMSEG_CONDA_ENV=seg_mmcv +SEG_BACKEND_CONDA_ENV=seg_smp +SEG_WEIGHT_MODE=copy +SEG_ENABLE_SHELL_TASKS=1 +SEG_VALIDATE_DEEP=1 +VITE_API_BASE=http://localhost:8010 +``` + +Install system prerequisites first: Git, Conda or Miniconda, Node.js/npm, a +working NVIDIA driver and `nvidia-smi` for GPU discovery, and enough free disk +space for the copied weights. Full weight sync currently needs about 35 GB +plus normal training output space. + +Create the Python runtimes with the bundled bootstrap script: + +```bash +scripts/bootstrap_conda_envs.sh all +``` + +This creates `seg_smp` for the backend, SegModel, YOLO, dataset tools, and +general analysis jobs, plus `seg_mmcv` for full MMSeg/mmcv execution. To repair +only one environment: + +```bash +scripts/bootstrap_conda_envs.sh task +scripts/bootstrap_conda_envs.sh mmseg +``` + +Install and build the frontend once during deployment: + +```bash +cd frontend +npm install +npm run build +cd .. +``` + +Synchronize weights locally after the original `Seg/` directory is present. +The command copies `.pt`, `.pth`, `.onnx`, and `.engine` files into +`weights/files/` and writes `weights/manifest.json`. + +```bash +python scripts/sync_weights.py --mode copy --hash +``` + +If the deployment filesystem supports copy-on-write reflinks and disk space is +tight, set `SEG_WEIGHT_MODE=reflink` or pass `--mode reflink`; otherwise keep +the default `copy` mode. Weight binaries must stay out of normal Git commits. + +Start the services: + +```bash +# Terminal 1 +scripts/run_backend.sh + +# Terminal 2 +scripts/run_frontend.sh +``` + +Defaults are `0.0.0.0:8010` for the backend and Vite's dev port for the +frontend. Override backend binding with `SEG_BACKEND_HOST` and +`SEG_BACKEND_PORT`. For a production process manager such as systemd, +supervisor, or Docker Compose, call the same two scripts so `.env` and the +configured conda environments are used consistently. + +Validate a deployment before handing it to operators: + +```bash +curl http://127.0.0.1:8010/api/health +curl http://127.0.0.1:8010/api/system/readiness +PYTHONPATH=backend conda run -n seg_smp python scripts/verify_runtime_envs.py --refresh +PYTHONPATH=backend conda run -n seg_smp python scripts/run_agents.py --build +scripts/check_no_weight_git.sh +``` + +For a fast non-training validation pass, run agents with +`SEG_VALIDATE_DEEP=0`. The browser dashboard exposes the same readiness, +coverage, GPU, weight, result, and agent checks through the UI. + The web UI includes a dataset bench for creating upload workspaces, uploading images/labels/masks, and jumping into the existing rename, PNG conversion, resize, pair-check, label rebuild, transparent overlay, stitch, and video-frame @@ -132,6 +236,9 @@ scripts/bootstrap_conda_envs.sh mmseg The current workspace contains tens of GB of pretrained and trained weights. They are copied into `weights/files/` and indexed in `weights/manifest.json`. +The web weight panel reads `GET /api/weights`, shows family/role counts and +sample source paths, and can run `POST /api/weights/verify` to check the local +manifest entries without putting the large files in Git. ```bash cd Seg_Data_Server_Net diff --git a/backend/app/agents/evaluation_agent.py b/backend/app/agents/evaluation_agent.py index 3c7d06a..a414f4b 100644 --- a/backend/app/agents/evaluation_agent.py +++ b/backend/app/agents/evaluation_agent.py @@ -78,6 +78,11 @@ def evaluate_project() -> dict: and "runtimeSelector" in frontend_text and "conda_env" in frontend_text and "request.conda_env" in jobs_text, + "weight_manifest_ui": "WeightPanel" in frontend_text + and "/api/weights" in frontend_text + and "/api/weights/verify" in frontend_text + and "weightList" in frontend_text + and "weightFamilies" in frontend_text, "live_log_stream_ui": "EventSource" in frontend_text and "eventSourceRef" in frontend_text and "log_size" in frontend_text diff --git a/backend/app/config.py b/backend/app/config.py index 3ffbd9c..8256273 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -5,10 +5,26 @@ from dataclasses import dataclass from pathlib import Path -def _resolve_path(value: str | None, default: Path) -> Path: +def _load_dotenv(path: Path) -> None: + if not path.exists(): + return + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export ") :].strip() + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip("'\"") + if key: + os.environ.setdefault(key, value) + + +def _resolve_path(value: str | None, default: Path, base_dir: Path) -> Path: base = Path(value).expanduser() if value else default if not base.is_absolute(): - base = default.parent / base + base = base_dir / base return base.resolve() @@ -27,17 +43,21 @@ class Settings: def get_settings() -> Settings: - project_root = Path(os.getenv("SEG_DATA_SERVER_ROOT", Path(__file__).resolve().parents[2])).expanduser() + default_project_root = Path(__file__).resolve().parents[2] + _load_dotenv(default_project_root / ".env") + + project_root = Path(os.getenv("SEG_DATA_SERVER_ROOT", default_project_root)).expanduser() if not project_root.is_absolute(): - project_root = (Path(__file__).resolve().parents[2] / project_root).resolve() + project_root = (default_project_root / project_root).resolve() else: project_root = project_root.resolve() + _load_dotenv(project_root / ".env") sibling_source = project_root.parent / "Seg" default_source = sibling_source if sibling_source.exists() else project_root.parent - source_root = _resolve_path(os.getenv("SEG_SOURCE_ROOT"), default_source) - db_path = _resolve_path(os.getenv("SEG_BACKEND_DB"), project_root / "var" / "seg_data_server.sqlite3") - log_dir = _resolve_path(os.getenv("SEG_BACKEND_LOG_DIR"), project_root / "var" / "job_logs") + source_root = _resolve_path(os.getenv("SEG_SOURCE_ROOT"), default_source, project_root) + db_path = _resolve_path(os.getenv("SEG_BACKEND_DB"), project_root / "var" / "seg_data_server.sqlite3", project_root) + log_dir = _resolve_path(os.getenv("SEG_BACKEND_LOG_DIR"), project_root / "var" / "job_logs", project_root) weights_root = (project_root / "weights").resolve() return Settings( diff --git a/backend/tests/test_config.py b/backend/tests/test_config.py new file mode 100644 index 0000000..83c209b --- /dev/null +++ b/backend/tests/test_config.py @@ -0,0 +1,28 @@ +def test_env_file_paths_are_project_relative(tmp_path, monkeypatch): + project = tmp_path / "Seg_Data_Server_Net" + source = tmp_path / "Seg" + project.mkdir() + source.mkdir() + (project / ".env").write_text( + "\n".join( + [ + "SEG_SOURCE_ROOT=../Seg", + "SEG_BACKEND_DB=var/seg_data_server.sqlite3", + "SEG_BACKEND_LOG_DIR=var/job_logs", + ] + ), + encoding="utf-8", + ) + + monkeypatch.setenv("SEG_DATA_SERVER_ROOT", str(project)) + monkeypatch.delenv("SEG_SOURCE_ROOT", raising=False) + monkeypatch.delenv("SEG_BACKEND_DB", raising=False) + monkeypatch.delenv("SEG_BACKEND_LOG_DIR", raising=False) + + from app.config import get_settings + + settings = get_settings() + + assert settings.source_root == source.resolve() + assert settings.db_path == (project / "var" / "seg_data_server.sqlite3").resolve() + assert settings.log_dir == (project / "var" / "job_logs").resolve() diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 2a3153c..b798015 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -130,6 +130,30 @@ type DatasetYoloOutputsPayload = { heatmaps: ResultItem[]; }; +type WeightEntry = { + source_path: string; + stored_path: string; + size: number; + family: string; + role: string; + sha256?: string; +}; + +type WeightManifest = { + generated_at?: string | null; + updated_at?: string | null; + source_root?: string; + count: number; + total_bytes: number; + files: WeightEntry[]; +}; + +type WeightVerifyPayload = { + count: number; + ok_count: number; + items: Array<{ stored_path: string; exists: boolean; size_ok: boolean; hash_ok?: boolean | null; ok: boolean }>; +}; + type CoveragePayload = { scripts_total: number; user_scripts_total: number; @@ -344,6 +368,7 @@ function useData() { const [jobs, setJobs] = useState([]); const [results, setResults] = useState([]); const [curves, setCurves] = useState([]); + const [weightManifest, setWeightManifest] = useState(null); const [datasets, setDatasets] = useState([]); const [datasetValidations, setDatasetValidations] = useState>({}); const [coverage, setCoverage] = useState(null); @@ -356,7 +381,7 @@ function useData() { async function refresh() { try { - const [catalogNext, gpusNext, envsNext, readinessNext, capabilitiesNext, jobsNext, resultsNext, curvesNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext, agentEvaluationNext] = await Promise.all([ + const [catalogNext, gpusNext, envsNext, readinessNext, capabilitiesNext, jobsNext, resultsNext, curvesNext, weightsNext, datasetsNext, coverageNext, acceptanceNext, deepAcceptanceNext, agentEvaluationNext] = await Promise.all([ api("/api/catalog"), api("/api/system/gpus"), api("/api/system/envs"), @@ -365,6 +390,7 @@ function useData() { api("/api/jobs"), api("/api/results?limit=1000"), api("/api/results/curves?limit=100"), + api("/api/weights"), api("/api/datasets"), api("/api/coverage"), api("/api/acceptance/latest"), @@ -379,6 +405,7 @@ function useData() { setJobs(jobsNext); setResults(resultsNext); setCurves(curvesNext); + setWeightManifest(weightsNext); setDatasets(datasetsNext); const validationEntries: Array<[string, DatasetValidation]> = []; await Promise.all( @@ -408,7 +435,7 @@ function useData() { return () => window.clearInterval(timer); }, []); - return { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh }; + return { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh }; } function StatusPill({ status }: { status: string }) { @@ -431,7 +458,7 @@ function JobProgressBar({ progress }: { progress?: JobProgress }) { } function App() { - const { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData(); + const { catalog, gpus, condaEnvs, runtimeReadiness, capabilities, agentEvaluation, jobs, results, curves, weightManifest, datasets, datasetValidations, coverage, acceptance, deepAcceptance, error, refresh } = useData(); const [taskType, setTaskType] = useState("mock.echo"); const [params, setParams] = useState(JSON.stringify(defaultParams["mock.echo"], null, 2)); const [selectedJob, setSelectedJob] = useState(null); @@ -448,6 +475,7 @@ function App() { const [uploadKind, setUploadKind] = useState<"images" | "labels" | "masks">("images"); const [uploadFiles, setUploadFiles] = useState(null); const [agentValidation, setAgentValidation] = useState(null); + const [weightVerification, setWeightVerification] = useState(null); const [agentBusy, setAgentBusy] = useState(false); const eventSourceRef = useRef(null); @@ -587,12 +615,23 @@ function App() { method: "POST", body: JSON.stringify({ mode: "copy", hash_files: true, skip_existing: true }) }); + setWeightVerification(null); await refresh(); } finally { setBusy(false); } } + async function verifyWeights() { + setBusy(true); + try { + const result = await api("/api/weights/verify", { method: "POST" }); + setWeightVerification(result); + } finally { + setBusy(false); + } + } + async function runAcceptanceSmoke() { setBusy(true); try { @@ -1227,20 +1266,14 @@ function App() {

{runtimeReadiness?.passed ? "runtime imports ready" : "run scripts/bootstrap_conda_envs.sh"} · {runtimeReadiness?.generated_at ?? "not checked"}

-
-
-
-

Assets

-

权重

-
- -
-
{catalog?.weights.count ?? 0}
-

{formatBytes(catalog?.weights.total_bytes)} indexed

-

{catalog?.weights.updated_at ?? "manifest not generated"}

-
+
@@ -1364,6 +1397,65 @@ function JobDiagnostics({ job }: { job: Job }) { ); } +function WeightPanel({ + catalog, + manifest, + verification, + busy, + onSync, + onVerify +}: { + catalog: Catalog | null; + manifest: WeightManifest | null; + verification: WeightVerifyPayload | null; + busy: boolean; + onSync: () => void; + onVerify: () => void; +}) { + const files = manifest?.files ?? []; + const familyStats = Array.from(files.reduce((counts, item) => counts.set(item.family, (counts.get(item.family) ?? 0) + 1), new Map()).entries()) + .sort((a, b) => b[1] - a[1]); + return ( +
+
+
+

Assets

+

权重

+
+
+ + +
+
+
{manifest?.count ?? catalog?.weights.count ?? 0}
+

{formatBytes(manifest?.total_bytes ?? catalog?.weights.total_bytes)} indexed

+

{manifest?.updated_at ?? catalog?.weights.updated_at ?? "manifest not generated"}

+ {verification && ( +
+ {verification.ok_count}/{verification.count} verified +
+ )} +
+ {familyStats.slice(0, 4).map(([family, count]) => ( + {family} {count} + ))} +
+ +
+ ); +} + function AgentCheckList({ checks, limit }: { checks: AgentCheck[]; limit: number }) { if (!checks.length) { return

尚未运行验证。

; diff --git a/frontend/src/styles.css b/frontend/src/styles.css index ea072d5..4c73715 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -1314,6 +1314,75 @@ meter { line-height: 1; } +.weightVerify { + margin-top: 10px; + padding: 7px 8px; + border-radius: 6px; + border: 1px solid var(--line); + font-size: 12px; +} + +.weightVerify.ok { + color: var(--green); + border-color: rgba(157, 226, 111, 0.34); + background: rgba(157, 226, 111, 0.08); +} + +.weightVerify.bad { + color: var(--red); + border-color: rgba(240, 113, 103, 0.45); + background: rgba(240, 113, 103, 0.08); +} + +.weightFamilies { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 12px; +} + +.weightFamilies span { + padding: 5px 7px; + border-radius: 999px; + border: 1px solid var(--line); + background: #080a08; + color: var(--muted); + font-size: 11px; +} + +.weightList { + display: grid; + gap: 7px; + max-height: 220px; + margin-top: 12px; + overflow: auto; +} + +.weightList a { + min-width: 0; + display: grid; + gap: 3px; + padding: 8px; + border-radius: 6px; + border: 1px solid var(--line); + background: #080a08; + color: var(--ink); + text-decoration: none; +} + +.weightList span, +.weightList small { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.weightList small { + color: var(--muted); + font-size: 11px; +} + .chips { display: flex; flex-wrap: wrap; diff --git a/scripts/run_backend.sh b/scripts/run_backend.sh index 084fb28..e0b9ffd 100755 --- a/scripts/run_backend.sh +++ b/scripts/run_backend.sh @@ -2,6 +2,13 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -f "${ROOT_DIR}/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${ROOT_DIR}/.env" + set +a +fi + BACKEND_ENV="${SEG_BACKEND_CONDA_ENV:-seg_smp}" HOST="${SEG_BACKEND_HOST:-0.0.0.0}" PORT="${SEG_BACKEND_PORT:-8010}" diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.sh index 00b144d..658601e 100755 --- a/scripts/run_frontend.sh +++ b/scripts/run_frontend.sh @@ -2,7 +2,13 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -f "${ROOT_DIR}/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${ROOT_DIR}/.env" + set +a +fi + cd "${ROOT_DIR}/frontend" npm install exec npm run dev -- --host 0.0.0.0 -