BZJZ_Material/文档润色流和知识库构建流/claude-scholar/skills/architecture-design/examples/pipeline_example.sh

#!/bin/bash
###############################################################################
# Training Pipeline Script
#
# This script demonstrates the standard training pipeline execution pattern.
# It handles environment setup, configuration, and execution with proper
# error handling and logging.
#
# Usage:
#   ./run/pipeline/training/train.sh --config-name=config_example
###############################################################################

set -e  # Exit on error
set -o pipefail  # Exit on pipe failure

# Script configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
EXPERIMENT_NAME="baseline_experiment"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

###############################################################################
# Helper Functions
###############################################################################

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

cleanup() {
    log_info "Cleaning up..."
    # Add cleanup logic here (e.g., kill background processes)
}

trap cleanup EXIT

###############################################################################
# Environment Setup
###############################################################################

setup_environment() {
    log_info "Setting up environment..."

    # Activate virtual environment if it exists
    if [ -f "${PROJECT_ROOT}/.venv/bin/activate" ]; then
        source "${PROJECT_ROOT}/.venv/bin/activate"
        log_info "Activated virtual environment"
    fi

    # Check required commands
    command -v python >/dev/null 2>&1 || { log_error "Python not found"; exit 1; }

    # Set Python path
    export PYTHONPATH="${PROJECT_ROOT}/src:${PYTHONPATH}"
    log_info "PYTHONPATH set to: ${PYTHONPATH}"
}

###############################################################################
# Configuration
###############################################################################

parse_arguments() {
    # Default values
    CONFIG="default"
    GPUS=0
    SEED=42

    # Parse arguments
    while [[ $# -gt 0 ]]; do
        case $1 in
            --config-name|-c)
                CONFIG="$2"
                shift 2
                ;;
            --gpus|-g)
                GPUS="$2"
                shift 2
                ;;
            --seed|-s)
                SEED="$2"
                shift 2
                ;;
            *)
                log_warn "Unknown argument: $1"
                shift
                ;;
        esac
    done

    log_info "Configuration: ${CONFIG}"
    log_info "GPUs: ${GPUS}"
    log_info "Seed: ${SEED}"
}

###############################################################################
# Main Training Function
###############################################################################

run_training() {
    log_info "Starting training..."

    # Output directory for this run
    OUTPUT_DIR="${PROJECT_ROOT}/outputs/${EXPERIMENT_NAME}/${TIMESTAMP}"
    mkdir -p "${OUTPUT_DIR}"

    log_info "Output directory: ${OUTPUT_DIR}"

    # Training command with Hydra
    python "${PROJECT_ROOT}/train.py" \
        --config-name="${CONFIG}" \
        seed=${SEED} \
        dir.output_dir="${OUTPUT_DIR}" \
        training.device=cuda \
        hydra.output_dir="${OUTPUT_DIR}/hydra" \
        hydra.run.dir="${OUTPUT_DIR}/hydra" || {
        log_error "Training failed!"
        exit 1
    }

    log_info "Training completed successfully!"
}

###############################################################################
# Post-Processing
###############################################################################

post_process() {
    log_info "Post-processing results..."

    # Copy logs to output directory
    if [ -f "${OUTPUT_DIR}/hydra/*.log" ]; then
        cp "${OUTPUT_DIR}/hydra/"*.log "${OUTPUT_DIR}/"
    fi

    # Generate summary
    log_info "Run summary:"
    log_info "  Config: ${CONFIG}"
    log_info "  Seed: ${SEED}"
    log_info "  Output: ${OUTPUT_DIR}"

    # Print path to best checkpoint
    BEST_CHECKPOINT=$(find "${OUTPUT_DIR}" -name "best*.pt" | head -n 1)
    if [ -n "${BEST_CHECKPOINT}" ]; then
        log_info "  Best checkpoint: ${BEST_CHECKPOINT}"
    fi
}

###############################################################################
# Main Execution
###############################################################################

main() {
    log_info "=========================================="
    log_info "Training Pipeline"
    log_info "=========================================="

    # Setup
    setup_environment

    # Parse arguments
    parse_arguments "$@"

    # Run training
    run_training

    # Post-process
    post_process

    log_info "=========================================="
    log_info "Pipeline completed successfully!"
    log_info "=========================================="
}

# Run main function
main "$@"