Jenkins-py/pipeline.py

#!/usr/bin/env python3
"""
pipeline.py - 域控软件集成流水线主入口

完全替代原 Shell 脚本的主流程。
使用模块化设计，每个步骤对应一个或多个模块函数。
"""
import os
import sys
import logging
from pathlib import Path

# 导入各模块
from config import PipelineConfig
from environment import setup_environment
from logging_conf import setup_logging

# 其他模块按需导入，避免循环依赖（可在函数内导入）
# 这里为了代码清晰，提前导入全部
from binary_repo import pull_binaries, commit_binaries
from repo_manager import download_mcu, download_soc
from gerrit import query_changes, apply_changes, label_verified
from build_mcu import prepare_mcu, compile_mcu
from build_soc import prepare_soc, compile_soc
from artifacts import (
    collect_logs,
    collect_newest_one_commit,
    collect_review_records,
    collect_qac_report,
    prepare_ftp_upload,
    commit_snapshot_manifest,
    clean_csv_add_header,
)
from ftp_upload import upload_directory
from utils import CommandError, RepoError, BuildMCUError, BuildSOCError, GerritError, ArtifactError


def run_pipeline():
    """执行完整的 CI/CD 流水线。"""

    # ---------- 1. 加载配置 ----------
    cfg = PipelineConfig.from_env()

    # ---------- 2. 配置日志 ----------
    log_file = os.path.join(cfg.workspace, 'pipeline.log')
    setup_logging(level=logging.INFO, log_file=log_file, console=True)
    logger = logging.getLogger(__name__)
    logger.info("Pipeline started - configuration: %s", cfg)

    try:
        # ---------- 3. 初始化环境 ----------
        logger.info("===== Setting up environment =====")
        setup_environment()
        # 确保 PYTHONUNBUFFERED 在子进程中也生效（已在 environment 中设置）

        # ---------- 4. 同步二进制仓库 ----------
        logger.info("===== Syncing binary repositories =====")
        for repo in cfg.binary_repos:
            pull_binaries(repo)

        # ---------- 5. 下载源码 ----------
        logger.info("===== Downloading source code =====")
        download_mcu(
            cfg.mcu_dir,
            cfg.mcu_manifest_url,
            cfg.repo_branch,
            f"ssh://{cfg.gerrit_host}/tools/repo",
            cfg.mcu_release_version,
            cfg.snapshot,
        )
        download_soc(
            cfg.soc_dir,
            cfg.soc_manifest_url,
            cfg.repo_branch,
            f"ssh://{cfg.gerrit_host}/tools/repo",
            cfg.soc_release_version,
            cfg.snapshot,
        )

        # ---------- 6. 应用 Gerrit 变更 ----------
        changes = []
        if cfg.gerrit_flag:
            logger.info("===== Downloading Gerrit changes =====")
            changes = query_changes(
                cfg.gerrit_host,
                cfg.gerrit_port,
                cfg.gerrit_name,
                cfg.gerrit_changes_list,
            )
            apply_changes(changes, cfg.mcu_dir, cfg.soc_dir)

        # ---------- 7. 编译 MCU ----------
        logger.info("===== Compiling MCU =====")
        prepare_mcu(cfg.mcu_sdk_dir, cfg.soc_dir)   # 当前为空，保留扩展
        compile_mcu(cfg.mcu_sdk_dir, cfg.profile_mode, cfg.soc_project)

        # ---------- 8. 编译 SOC ----------
        logger.info("===== Compiling SOC =====")
        prepare_soc(cfg.mcu_sdk_dir, cfg.soc_dir, cfg.soc_project)
        compile_soc(
            cfg.soc_dir,
            cfg.soc_project,
            cfg.secure_enable,
            cfg.factory_emmc_img_enable,
            cfg.profile_mode,
        )

        # ---------- 9. Gerrit 打分 ----------
        if cfg.gerrit_flag and changes:
            logger.info("===== Labeling Gerrit changes =====")
            label_verified(
                cfg.gerrit_host,
                cfg.gerrit_port,
                cfg.gerrit_name,
                changes,
            )

        # ---------- 10. 自动提交二进制产物 ----------
        if cfg.auto_commit_server:
            logger.info("===== Committing binary repos =====")
            for repo in cfg.binary_repos:
                commit_binaries(repo, cfg.auto_commit_branch)

        # ---------- 11. 收集日志、快照、QAC ----------
        logger.info("===== Collecting commit logs and snapshots =====")
        collect_log_file = Path(cfg.workspace) / 'commit_log.csv'
        newest_log_file = Path(cfg.workspace) / 'newest_one_commit_log.csv'
        review_records_dir = Path(cfg.workspace) / 'review_records'

        # 清理旧文件
        collect_log_file.unlink(missing_ok=True)
        newest_log_file.unlink(missing_ok=True)
        if review_records_dir.exists():
            import shutil
            shutil.rmtree(review_records_dir)
        review_records_dir.mkdir(parents=True)

        for repo_dir in [cfg.mcu_dir, cfg.soc_dir]:
            logger.info("Collecting logs from %s", repo_dir)
            collect_logs(repo_dir, str(collect_log_file))
            collect_newest_one_commit(repo_dir, str(newest_log_file))
            collect_review_records(repo_dir, str(review_records_dir))

        # 清理 CSV 并添加表头
        header_commit = [
            'project name', 'commit hash', 'auth name', 'auth email',
            'auth date', 'commit date', 'subject',
        ]
        header_newest = header_commit + ['code lines']  # 在新脚本中可保留兼容
        clean_csv_add_header(collect_log_file, header_commit)
        clean_csv_add_header(newest_log_file, header_newest)

        # 收集 QAC 报告
        qac_report_dir = Path(cfg.workspace) / 'qac_report'
        collect_qac_report(cfg.mcu_sdk_dir, str(qac_report_dir))

        # ---------- 12. FTP 上传准备 ----------
        logger.info("===== Preparing FTP upload =====")
        prepare_ftp_upload(
            cfg,
            str(collect_log_file),
            str(newest_log_file),
            str(review_records_dir),
            str(qac_report_dir) if qac_report_dir.exists() else None,
        )

        # ---------- 13. 提交发布基线 manifest ----------
        logger.info("===== Committing release manifests =====")
        commit_snapshot_manifest(cfg.mcu_dir, cfg.mcu_release_version)
        commit_snapshot_manifest(cfg.soc_dir, cfg.soc_release_version)

        # ---------- 14. 可选：触发实际的 FTP 上传 ----------
        if os.environ.get('FTP_UPLOAD_ENABLE', 'true').lower() == 'true':
            logger.info("===== Uploading to FTP =====")
            # 需要本地部署目录存在，由 prepare_ftp_upload 创建
            deploy_dir = Path(cfg.deploy_image_local_dir) / cfg.repo_branch
            # 找到最新的部署子目录（时间戳命名）
            deploy_subdirs = sorted(deploy_dir.iterdir(), key=lambda p: p.name, reverse=True)
            if deploy_subdirs:
                latest_deploy = deploy_subdirs[0]
                upload_directory(cfg, str(latest_deploy))
            else:
                logger.warning("No deploy directory found for FTP upload")

        logger.info("===== Pipeline completed successfully =====")

    except (CommandError, BuildMCUError, BuildSOCError, GerritError,
            ArtifactError, Exception) as e:
        logger.exception("Pipeline failed: %s", e)
        sys.exit(1)


if __name__ == '__main__':
    run_pipeline()