feat: 新增视频分析流水线、Web界面、孤儿清理工具；修复健壮性问题
- 新增 analysis/ 模块：MOG2活动检测、YOLO人物跟踪、MediaPipe摔倒检测、Ollama活动打标、缩略图抽取 - 新增 webapp/：Flask事件浏览界面，按需HEVC→H264转码缓存 - 新增 analyze.py：7阶段状态机流水线（scan→backfill→motion→person→fall→activity→thumbnail） - 新增 cleanup_orphans.py：清理文件已删除的数据库孤儿记录 - 修复：motion/person/activity三阶段加单视频异常兜底，避免批次中断 - 修复：ollama_client加健康检查和3次重试，num_ctx从8192降到2048 - 修复：transcode_cache加fcntl文件锁，防止并发转码同一视频 - 修复：SQLite开WAL模式+busy_timeout，支持读写并发 - 修复：sync.py ffmpeg路径改为shutil.which查找，删除automate.py死代码PYTHON - 修复：thumbnail失败加3次重试上限，避免无限重试损坏文件 - 新增：requirements.lock锁定版本、README文档 - 更新：.gitignore排除.playwright-mcp、截图、备份文件
This commit is contained in:
yuming
@@ -0,0 +1,130 @@
|
||||
"""基于 dedup_finder 检测出的字节级重复文件，生成清理计划 / 执行实际清理。
|
||||
|
||||
处理原则：同一组 MD5 完全相同的文件里，只保留文件名（时间戳）最小的一份，
|
||||
其余文件从磁盘删除；这些多余视频如果已经跑过 Phase3-5 分析，产生的
|
||||
person_presence/fall/activity 事件会让统计虚高，所以对应的 videos/events
|
||||
数据库记录、缩略图、转码缓存也一并清理，避免留下孤儿数据。
|
||||
|
||||
默认只生成计划（不改动任何文件/数据库），确认无误后加 --execute 才真正执行，
|
||||
且执行前还会要求在终端输入 yes 二次确认。
|
||||
"""
|
||||
import argparse
|
||||
|
||||
from analysis.config import PROJECT_ROOT, TRANSCODED_DIR
|
||||
from analysis.db import get_connection
|
||||
from analysis.dedup_finder import find_duplicates
|
||||
|
||||
|
||||
def _video_id_for_path(conn, rel_path):
|
||||
row = conn.execute(
|
||||
"SELECT id FROM videos WHERE file_path = ?", (str(rel_path),)
|
||||
).fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
|
||||
def build_plan():
|
||||
"""返回删除计划列表，每项对应一个要删除的多余文件。"""
|
||||
dup_groups = find_duplicates()
|
||||
|
||||
plan = []
|
||||
with get_connection() as conn:
|
||||
for paths in dup_groups.values():
|
||||
paths = sorted(paths, key=lambda p: p.name)
|
||||
keep, remove = paths[0], paths[1:]
|
||||
|
||||
for path in remove:
|
||||
rel_path = path.relative_to(PROJECT_ROOT)
|
||||
video_id = _video_id_for_path(conn, rel_path)
|
||||
thumbnails = []
|
||||
event_count = 0
|
||||
if video_id is not None:
|
||||
rows = conn.execute(
|
||||
"SELECT thumbnail_path FROM events WHERE video_id = ?", (video_id,)
|
||||
).fetchall()
|
||||
event_count = len(rows)
|
||||
thumbnails = [r["thumbnail_path"] for r in rows if r["thumbnail_path"]]
|
||||
|
||||
plan.append({
|
||||
"keep_path": str(keep.relative_to(PROJECT_ROOT)),
|
||||
"remove_path": str(rel_path),
|
||||
"video_id": video_id,
|
||||
"event_count": event_count,
|
||||
"thumbnails": thumbnails,
|
||||
"size_bytes": path.stat().st_size,
|
||||
})
|
||||
|
||||
return plan
|
||||
|
||||
|
||||
def print_plan(plan):
|
||||
total_bytes = sum(p["size_bytes"] for p in plan)
|
||||
total_events = sum(p["event_count"] for p in plan)
|
||||
orphan = [p for p in plan if p["video_id"] is None]
|
||||
|
||||
print(f"计划删除 {len(plan)} 个重复文件，释放 {total_bytes / 1024 / 1024:.1f} MB")
|
||||
print(f"将同步清理 {total_events} 条关联 events 记录")
|
||||
if orphan:
|
||||
print(f"其中 {len(orphan)} 个文件在 analysis.db 中还没有对应记录（尚未扫描过），只删文件不动数据库")
|
||||
|
||||
preview = plan[:20]
|
||||
for p in preview:
|
||||
print(f" - {p['remove_path']} (保留 {p['keep_path']}, video_id={p['video_id']}, events={p['event_count']})")
|
||||
if len(plan) > len(preview):
|
||||
print(f" ...(其余 {len(plan) - len(preview)} 条略)")
|
||||
|
||||
|
||||
def execute_plan(plan):
|
||||
deleted_files = 0
|
||||
deleted_events = 0
|
||||
deleted_videos = 0
|
||||
|
||||
with get_connection() as conn:
|
||||
for p in plan:
|
||||
remove_path = PROJECT_ROOT / p["remove_path"]
|
||||
if remove_path.exists():
|
||||
remove_path.unlink()
|
||||
deleted_files += 1
|
||||
|
||||
for thumb in p["thumbnails"]:
|
||||
thumb_path = PROJECT_ROOT / thumb
|
||||
if thumb_path.exists():
|
||||
thumb_path.unlink()
|
||||
|
||||
if p["video_id"] is not None:
|
||||
cache_path = TRANSCODED_DIR / f"{p['video_id']}.mp4"
|
||||
if cache_path.exists():
|
||||
cache_path.unlink()
|
||||
|
||||
cur = conn.execute("DELETE FROM events WHERE video_id = ?", (p["video_id"],))
|
||||
deleted_events += cur.rowcount
|
||||
conn.execute("DELETE FROM videos WHERE id = ?", (p["video_id"],))
|
||||
deleted_videos += 1
|
||||
|
||||
return {
|
||||
"deleted_files": deleted_files,
|
||||
"deleted_events": deleted_events,
|
||||
"deleted_videos": deleted_videos,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="重复视频清理：默认只打印计划，加 --execute 才真正执行")
|
||||
parser.add_argument("--execute", action="store_true", help="实际执行删除")
|
||||
args = parser.parse_args()
|
||||
|
||||
plan = build_plan()
|
||||
print_plan(plan)
|
||||
|
||||
if not args.execute:
|
||||
print("\n（当前只是计划预览，未做任何改动；确认无误后加 --execute 参数执行）")
|
||||
else:
|
||||
answer = input(f"\n确认删除以上 {len(plan)} 个文件及关联数据库记录？输入 yes 继续：")
|
||||
if answer.strip().lower() == "yes":
|
||||
result = execute_plan(plan)
|
||||
print(
|
||||
f"\n完成：删除文件 {result['deleted_files']} 个，"
|
||||
f"删除 videos 记录 {result['deleted_videos']} 条，"
|
||||
f"删除 events 记录 {result['deleted_events']} 条"
|
||||
)
|
||||
else:
|
||||
print("已取消")
|
||||
Reference in New Issue
Block a user