提交 80bbb6ac authored 作者: 陈泽健's avatar 陈泽健

fix(定时任务): 看门狗失败执行也生成报告 + 单用例超时兜底 + Chromium 崩溃恢复

A 修复(看门狗失败执行也生成报告):
- scheduler_service.py 新增 _has_any_case_results 辅助函数
- UI/安全两条路径报告条件 run_ok -> run_ok or _has_any_case_results
- 解决 5.44 每日任务连续 4 次看门狗失败、报告目录无一文件

B 修复(定位并消除卡死点):
- B1: case_results 写入 start_time(由实际耗时回推),支持定位卡死用例
- B2: 用例级硬超时兜底(_case_start_mono + case_remaining_seconds,
      步骤/等待/微应用轮询均受用例剩余时间约束,防单用例无限挂起)
- B3: Chromium 崩溃自动恢复(崩溃标记检测 + 重建页面复用 context
      保留登录态,失败步骤起重试,每用例最多恢复一次)

部署配套:
- deploy/docker-compose.yml: app 容器日志轮转 50m x 5
- deploy_fix_560.py: 增加 scheduler_service.py / execution_service.py 上传
Co-Authored-By: 's avatarClaude <noreply@anthropic.com>
上级 3a088e6a
......@@ -19,7 +19,7 @@ import time
import threading
from pathlib import Path
from typing import Optional, List, Tuple, Dict, Any, Callable
from datetime import datetime
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor
from sqlalchemy.ext.asyncio import AsyncSession
......@@ -682,6 +682,11 @@ class ExecutionService:
case_result_obj.status = result.status
case_result_obj.duration = result.duration
# B1:由实际耗时回推 start_time,支持定位卡死用例
# (历史数据全为 NULL,修复后生效)
case_result_obj.start_time = datetime.now() - timedelta(
seconds=result.duration or 0
)
case_result_obj.end_time = datetime.now()
case_result_obj.retry_count = result.retry_count
case_result_obj.error_message = result.error_message
......
......@@ -23,12 +23,13 @@ import logging
from datetime import datetime, timedelta
from typing import Optional, List
from sqlalchemy import select
from sqlalchemy import func, select
from app.database import async_session_maker
from app.models.scheduled_task import ScheduledTask
from app.models.test_case import TestCase
from app.models.execution import Execution
from app.models.case_result import CaseResult
from app.services.execution_service import ExecutionService, is_execution_running
from app.services.report_service import ReportService
from app.utils.id_generator import generate_id
......@@ -146,6 +147,30 @@ async def _has_running_security_execution(db) -> bool:
return result.scalar_one_or_none() is not None
async def _has_any_case_results(db, execution_id: str) -> bool:
"""
执行是否已产生任何用例结果。
看门狗把「长时间无进展」的执行标记为 failed 时,工作线程已被中断,
本执行走到的失败/成功用例结果已实时写库。此时虽非 run_ok,仍应生成
报告,否则 auto_report 任务的每次失败执行都无报告输出(5.44 事件形态:
每日任务连续 4 次看门狗失败、报告目录无一文件)。
Args:
db: 异步数据库会话
execution_id (str): 执行记录 ID
Returns:
bool: 已存在至少一条用例结果
"""
result = await db.execute(
select(func.count(CaseResult.id)).where(
CaseResult.execution_id == execution_id,
).limit(1)
)
return (result.scalar_one() or 0) > 0
async def _run_security_task_once(
task: ScheduledTask,
db,
......@@ -212,8 +237,9 @@ async def _run_security_task_once(
run_ok = False
logger.error(f"[定时任务] 安全任务「{task.name}」执行异常: {e}")
# 5. 自动生成报告(执行成功后才生成)
if task.auto_report and run_ok:
# 5. 自动生成报告(执行成功后才生成;看门狗失败但已有用例结果也生成,
# 避免多次失败执行无报告输出——5.44 每日任务连续 4 次看门狗失败均无报告)
if task.auto_report and (run_ok or await _has_any_case_results(db, execution.id)):
try:
report_service = SecurityReportService()
await report_service.generate_report(execution.id, db)
......@@ -336,8 +362,10 @@ async def run_task_once(task_id: str, manual: bool = False) -> Optional[str]:
run_ok = False
logger.error(f"[定时任务] 任务「{task.name}」执行异常: {e}")
# 5. 自动生成报告(执行成功后才生成,避免冲突失败记录产出空报告)
if task.auto_report and run_ok:
# 5. 自动生成报告(执行成功后才生成,避免冲突失败记录产出空报告;
# 看门狗失败但已有用例结果也生成,避免多次失败执行无报告输出——
# 5.44 每日任务连续 4 次看门狗失败、报告目录无一文件)
if task.auto_report and (run_ok or await _has_any_case_results(db, execution.id)):
try:
report_service = ReportService()
html = await report_service.generate_html(execution.id, db)
......
......@@ -36,6 +36,8 @@ PROD_CASE = "case_13650e0406e64779b66e6fa5de35c24a"
# 本地 → 服务器路径
UPLOADS = [
("app/executors/playwright_executor.py", BACKEND_HOST + "/app/executors/playwright_executor.py"),
("app/services/scheduler_service.py", BACKEND_HOST + "/app/services/scheduler_service.py"),
("app/services/execution_service.py", BACKEND_HOST + "/app/services/execution_service.py"),
("scripts/fix_step12.py", BACKEND_HOST + "/scripts/fix_step12.py"),
("scripts/run_case_verify.py", BACKEND_HOST + "/scripts/run_case_verify.py"),
]
......
......@@ -63,6 +63,13 @@ services:
- /data/third_party/plat-auto-test/logs:/app/logs
ports:
- "80:80"
# 容器日志轮转:限制 json-file 日志大小与文件数,避免宿主磁盘被撑满
# (5.44 卡死时段 docker logs 已因默认无轮转丢失,见执行计划遗留问题 3)
logging:
driver: json-file
options:
max-size: "50m"
max-file: "5"
networks:
- plat-auto-test-net
healthcheck:
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论