c4c-download/batch_download.py

#!/usr/bin/env python3
"""批量下载 SAP C4C 附件，从 CSV 读取 Ticket ID，并行执行并记录错误日志"""
import subprocess
import sys
import os
import json
import datetime
import csv
import glob
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading

# ── 配置 ──────────────────────────────────────────────────────────────────────
TENANT   = "https://my300375.c4c.saphybriscloud.cn"
USER     = "admin"
PASSWORD = "Xjait.1?"
OUTPUT   = "./downloads"
DSM_URL  = "http://10.0.10.235:5000"
DSM_USER = "PLM"
DSM_PASS = "123456"
DSM_PATH = "/Newgonow/AU-SPFJ"

WORKERS     = 5

SCRIPT      = os.path.join(os.path.dirname(__file__), "sap-c4c-AttachmentFolder.py")
ERROR_LOG   = os.path.join(os.path.dirname(__file__), "error_log.txt")
FAILED_CSV  = os.path.join(os.path.dirname(__file__), "failed_tickets.csv")
DATASOURCE  = os.path.join(os.path.dirname(__file__), "datasource")

print_lock = threading.Lock()
failed_lock = threading.Lock()
failed_ids = set()
# ─────────────────────────────────────────────────────────────────────────────


def get_ticket_ids():
    ids, seen = [], set()

    def _add(val):
        val = str(val).strip()
        if val and val not in seen:
            seen.add(val)
            ids.append(val)

    for csv_file in glob.glob(os.path.join(DATASOURCE, "*.csv")):
        with open(csv_file, encoding="utf-8-sig") as f:
            reader = csv.reader(f)
            next(reader, None)
            for row in reader:
                if len(row) >= 2:
                    _add(row[1])

    for xlsx_file in glob.glob(os.path.join(DATASOURCE, "*.xlsx")):
        import openpyxl
        wb = openpyxl.load_workbook(xlsx_file, read_only=True, data_only=True)
        ws = wb.active
        first = True
        for row in ws.iter_rows(min_col=2, max_col=2, values_only=True):
            if first:
                first = False
                continue
            if row[0] is not None:
                _add(row[0])
        wb.close()

    return ids


def log_error(ticket_id, message):
    ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    with open(ERROR_LOG, "a", encoding="utf-8") as f:
        f.write(f"[{ts}] Ticket {ticket_id}: {message}\n")
    with failed_lock:
        if ticket_id not in failed_ids:
            failed_ids.add(ticket_id)
            with open(FAILED_CSV, "a", encoding="utf-8", newline="") as f:
                csv.writer(f).writerow([ticket_id])


def run_ticket(ticket_id, index, total):
    with print_lock:
        print(f"\n[{index}/{total}] 开始下载 Ticket {ticket_id} ...")
    cmd = [
        sys.executable, SCRIPT,
        "--tenant",       TENANT,
        "--user",         USER,
        "--password",     PASSWORD,
        "--ticket",       str(ticket_id),
        "--output-dir",   os.path.join(OUTPUT, str(ticket_id)),
        "--json",
        "--dsm-url",      DSM_URL,
        "--dsm-user",     DSM_USER,
        "--dsm-password", DSM_PASS,
        "--dsm-path",     DSM_PATH,
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
        # 解析 JSON 输出
        try:
            data = json.loads(result.stdout)
            success = data.get("success", False)
            downloaded = len(data.get("downloadedFiles", []))
            dsm_uploads = data.get("dsmUpload", [])
            ok_uploads  = sum(1 for u in dsm_uploads if u.get("success"))
            fail_uploads = len(dsm_uploads) - ok_uploads

            if success:
                with print_lock:
                    print(f"  ✓ [{ticket_id}] 下载 {downloaded} 个文件, DSM 上传 {ok_uploads} 成功 {fail_uploads} 失败")
                if fail_uploads:
                    fails = [u for u in dsm_uploads if not u.get("success")]
                    for u in fails:
                        log_error(ticket_id, f"DSM 上传失败: {u.get('file')} - {u.get('error')}")
            else:
                err = data.get("error", result.stderr or "未知错误")
                with print_lock:
                    print(f"  ✗ [{ticket_id}] 失败: {err}")
                log_error(ticket_id, err)
        except json.JSONDecodeError:
            if result.returncode == 0:
                with print_lock:
                    print(f"  ✓ [{ticket_id}] 完成")
            else:
                err = result.stderr.strip() or result.stdout.strip() or "未知错误"
                with print_lock:
                    print(f"  ✗ [{ticket_id}] 失败: {err}")
                log_error(ticket_id, err)

    except subprocess.TimeoutExpired:
        msg = "超时 (300s)"
        with print_lock:
            print(f"  ✗ [{ticket_id}] {msg}")
        log_error(ticket_id, msg)
    except Exception as e:
        with print_lock:
            print(f"  ✗ [{ticket_id}] 异常: {e}")
        log_error(ticket_id, str(e))


def main():
    global failed_ids
    print("读取 Ticket ID ...")
    ids = get_ticket_ids()
    if not ids:
        print("未找到任何 Ticket ID，请检查 datasource 目录")
        sys.exit(1)

    print(f"共 {len(ids)} 个 Ticket: {', '.join(ids)}")

    # 清空/创建 error_log 和 failed_tickets.csv
    open(ERROR_LOG, "w").close()
    open(FAILED_CSV, "w", encoding="utf-8", newline="").close()
    failed_ids.clear()

    with ThreadPoolExecutor(max_workers=WORKERS) as executor:
        futures = {executor.submit(run_ticket, tid, i, len(ids)): tid
                   for i, tid in enumerate(ids, 1)}
        for future in as_completed(futures):
            future.result()  # 触发异常传播（已在 run_ticket 内处理）

    print("\n全部完成。")
    if failed_ids:
        print(f"失败 {len(failed_ids)} 个 Ticket，已保存到 {FAILED_CSV}")
        if os.path.getsize(ERROR_LOG) > 0:
            print(f"错误详情见 {ERROR_LOG}")
    else:
        print("全部成功，无失败。")


if __name__ == "__main__":
    main()