adminym/dysync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 视频预览/数据看板/下载配置 三项业务功能

Browse Source 
- 新增视频预览：VideoPlayer 组件、VideoController/DyCollectVideoService/Repository、VideoPageRequestDTO 对应后端改动
- 新增数据看板：workplace/statics.vue、RecordTable、相关 store/http/axios 改动
- 新增下载配置：DownloadConfigService、ConfigController、AppSet.vue、AppConfig
- 其他：DyHttpClientService / 同步 Job(Collect, Favorit) / Cookie 表 / CommonService / ServiceExtension 的配套调整
- 仓库卫生：.gitignore 追加 feiniu 过滤；运维脚本归档到 docs/scripts/
This commit is contained in:
yuming
2026-04-25 00:22:33 +08:00
parent ca530312d0
commit b328e82eb2
27 changed files with 3602 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/CommonService.cs
+7
View File
@@ -54,6 +54,13 @@ namespace dy.net.service
{
cc.Cron = config.Cron;
cc.BatchCount = config.BatchCount;
// 更新下载配置
cc.EnableAutoDownload = config.EnableAutoDownload;
cc.DownloadDaysRange = config.DownloadDaysRange;
cc.AuthorWhitelist = config.AuthorWhitelist;
cc.AuthorBlacklist = config.AuthorBlacklist;
cc.DownloadFavorite = config.DownloadFavorite;
cc.DownloadCollect = config.DownloadCollect;
}
var update = await sqlSugarClient.Updateable<AppConfig>(cc).ExecuteCommandAsync();
service/DownloadConfigService.cs
+124
View File
@@ -0,0 +1,124 @@
using dy.net.model;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
namespace dy.net.service
{
/// <summary>
/// 下载配置服务，用于判断是否应该下载视频
/// </summary>
public class DownloadConfigService
{
private readonly CommonService _commonService;
public DownloadConfigService(CommonService commonService)
{
_commonService = commonService;
}
private AppConfig GetConfig()
{
return _commonService.GetConfig();
}
/// <summary>
/// 判断是否应该下载视频
/// </summary>
/// <param name="videoType">视频类型："1"=喜欢，"2"=收藏</param>
/// <param name="authorId">作者ID</param>
/// <param name="authorName">作者名称</param>
/// <param name="createTime">视频创建时间</param>
/// <returns>true=应该下载，false=不应该下载</returns>
public bool ShouldDownload(string videoType, string authorId, string authorName, DateTime createTime)
{
var config = GetConfig();
if (config == null)
{
return false;
}
// 如果未启用自动下载，直接返回false
if (config.EnableAutoDownload != true)
{
return false;
}
// 检查视频类型是否允许下载
if (videoType == "1" && config.DownloadFavorite != true)
{
return false; // 喜欢类型但未启用下载
}
if (videoType == "2" && config.DownloadCollect != true)
{
return false; // 收藏类型但未启用下载
}
// 检查时间范围
if (config.DownloadDaysRange.HasValue && config.DownloadDaysRange.Value > 0)
{
var daysDiff = (DateTime.Now - createTime).TotalDays;
if (daysDiff > config.DownloadDaysRange.Value)
{
return false; // 超出时间范围
}
}
// 检查作者白名单
if (!string.IsNullOrWhiteSpace(config.AuthorWhitelist))
{
try
{
var whitelist = JsonConvert.DeserializeObject<List<string>>(config.AuthorWhitelist);
if (whitelist != null && whitelist.Any())
{
// 如果配置了白名单，只有白名单中的作者才下载
var isInWhitelist = whitelist.Any(a =>
(!string.IsNullOrWhiteSpace(authorId) && a.Equals(authorId, StringComparison.OrdinalIgnoreCase)) ||
(!string.IsNullOrWhiteSpace(authorName) && a.Equals(authorName, StringComparison.OrdinalIgnoreCase))
);
if (!isInWhitelist)
{
return false; // 不在白名单中
}
}
}
catch (Exception ex)
{
Serilog.Log.Warning($"解析作者白名单失败：{ex.Message}");
}
}
// 检查作者黑名单
if (!string.IsNullOrWhiteSpace(config.AuthorBlacklist))
{
try
{
var blacklist = JsonConvert.DeserializeObject<List<string>>(config.AuthorBlacklist);
if (blacklist != null && blacklist.Any())
{
// 如果配置了黑名单，黑名单中的作者不下载
var isInBlacklist = blacklist.Any(a =>
(!string.IsNullOrWhiteSpace(authorId) && a.Equals(authorId, StringComparison.OrdinalIgnoreCase)) ||
(!string.IsNullOrWhiteSpace(authorName) && a.Equals(authorName, StringComparison.OrdinalIgnoreCase))
);
if (isInBlacklist)
{
return false; // 在黑名单中
}
}
}
catch (Exception ex)
{
Serilog.Log.Warning($"解析作者黑名单失败：{ex.Message}");
}
}
// 所有检查都通过，可以下载
return true;
}
}
}
service/DyCollectVideoService.cs
+189 -7
View File
@@ -41,12 +41,18 @@ namespace dy.net.service
.Where(video => !existingAwemeIds.Contains(video.AwemeId))
.ToList();
Serilog.Log.Debug($"batchInsert: 待插入{videos.Count}条，已存在{existingAwemeIds.Count}条，需要插入{videosToInsert.Count}条");
// 4. 如果没有需要插入的新记录，直接返回成功
if (!videosToInsert.Any())
{
Serilog.Log.Debug($"batchInsert: 所有视频都已存在，无需插入");
return true;
}
// 5. 批量插入过滤后的新记录
var insertedCount = await _dyCollectVideoRepository.InsertRangeAsync(videosToInsert);
Serilog.Log.Debug($"batchInsert: 实际插入{insertedCount}条记录");
// 返回是否插入成功（至少插入一条）
return insertedCount > 0;
@@ -55,16 +61,47 @@ namespace dy.net.service
public async Task<VideoStaticsDto> GetStatics()
{
List<DyCollectVideo> list = await this._dyCollectVideoRepository.GetAllAsync();
List<DyCollectVideo> allList = await this._dyCollectVideoRepository.GetAllAsync();
if (!allList.Any())
return new VideoStaticsDto();
// 只统计文件实际存在的视频
var list = allList.Where(x =>
!string.IsNullOrWhiteSpace(x.VideoSavePath) &&
System.IO.File.Exists(x.VideoSavePath)
).ToList();
Serilog.Log.Debug($"统计视频：数据库记录 {allList.Count} 条，文件存在 {list.Count} 条");
if (!list.Any())
return new VideoStaticsDto();
var Categories = list.GroupBy(x => x.Tag1).Select(x => new VideoStaticsItemDto { Name = x.Key, Count = x.LongCount() }).OrderByDescending(p => p.Count).ToList();
Categories.Where(x => string.IsNullOrWhiteSpace(x.Name)).ToList().ForEach(x => x.Name = "其他");
// 按 viedoCate（Tag1/Tag2 组合）分组统计分类
var Categories = list
.GroupBy(x =>
{
// 使用与模型 ViedoCate 属性相同的逻辑：Tag1 + "/" + Tag2
var tag1 = string.IsNullOrWhiteSpace(x.Tag1) ? "" : x.Tag1;
var tag2 = string.IsNullOrWhiteSpace(x.Tag2) ? "" : x.Tag2;
var viedoCate = tag1 + "/" + tag2;
// 如果 Tag1 和 Tag2 都为空，标记为"未分类"
if (string.IsNullOrWhiteSpace(tag1) && string.IsNullOrWhiteSpace(tag2))
{
return "未分类";
}
return viedoCate;
})
.Select(x => new VideoStaticsItemDto
{
Name = x.Key,
Count = x.LongCount()
})
.OrderByDescending(p => p.Count)
.ToList();
var data = new VideoStaticsDto
{
AuthorCount = list.Select(x => x.AuthorId).Distinct().Count(),
CategoryCount = list.Select(x => x.Tag1).Distinct().Count(),
CategoryCount = Categories.Count, // 使用实际分类数量
VideoCount = list.Count,
Categories = Categories,
FavoriteCount = list.Count(x => x.ViedoType == "1"),
@@ -85,9 +122,154 @@ namespace dy.net.service
//分页查询
public async Task<(List<DyCollectVideo> list, int totalCount)> GetPagedAsync(int pageIndex, int pageSize, string tag = null, string author = null, string viedoType = null, List<string>? dates = null)
public async Task<(List<DyCollectVideo> list, int totalCount)> GetPagedAsync(int pageIndex, int pageSize, string tag = null, List<string> tags = null, string author = null, string viedoType = null, List<string>? dates = null, bool? isDeleted = null, string sortField = null, string sortOrder = null)
{
return await _dyCollectVideoRepository.GetPagedAsync(pageIndex, pageSize, tag, author, viedoType, dates);
return await _dyCollectVideoRepository.GetPagedAsync(pageIndex, pageSize, tag, tags, author, viedoType, dates, isDeleted, sortField, sortOrder);
}
/// <summary>
/// 获取所有视频类型列表（用于下拉框）
/// </summary>
/// <returns></returns>
public async Task<List<string>> GetAllCategoriesAsync()
{
try
{
// 获取所有视频记录
var allVideos = await _dyCollectVideoRepository.GetAllAsync();
Serilog.Log.Debug($"获取视频类型列表：共 {allVideos?.Count ?? 0} 条视频记录");
// 提取所有非空的 viedoCate（Tag1/Tag2 组合），去重并排序
// 使用与模型 ViedoCate 属性相同的逻辑：Tag1 + "/" + Tag2
var categories = allVideos
.Select(x =>
{
// 使用与模型 ViedoCate 属性相同的逻辑
var tag1 = string.IsNullOrWhiteSpace(x.Tag1) ? "" : x.Tag1;
var tag2 = string.IsNullOrWhiteSpace(x.Tag2) ? "" : x.Tag2;
var viedoCate = tag1 + "/" + tag2;
return viedoCate;
})
.Where(x => !string.IsNullOrWhiteSpace(x) && x != "/") // 过滤空值和只有 "/" 的情况
.Distinct()
.OrderBy(x => x)
.ToList();
// 检查是否存在空类型的视频（Tag1 和 Tag2 都为空）
var hasEmptyCategory = allVideos.Any(x => string.IsNullOrWhiteSpace(x.Tag1) && string.IsNullOrWhiteSpace(x.Tag2));
if (hasEmptyCategory)
{
// 在列表开头添加"未分类"选项
categories.Insert(0, "未分类");
}
Serilog.Log.Debug($"提取到的视频类型数量：{categories.Count}，类型列表：{string.Join(", ", categories)}");
return categories;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "获取视频类型列表失败");
return new List<string>();
}
}
/// <summary>
/// 根据ID获取视频信息
/// </summary>
/// <param name="id">视频ID</param>
/// <returns></returns>
public async Task<DyCollectVideo> GetByIdAsync(string id)
{
return await _dyCollectVideoRepository.GetByIdAsync(id);
}
/// <summary>
/// 清理数据库中文件不存在的视频记录
/// </summary>
/// <returns>清理的记录数量</returns>
public async Task<int> CleanupMissingFilesAsync()
{
try
{
var allVideos = await _dyCollectVideoRepository.GetAllAsync();
if (!allVideos.Any())
return 0;
// 找出需要清理的视频ID：
// 1. VideoSavePath 不为空但文件不存在的记录
// 2. VideoSavePath 为空或null的记录（这些记录没有视频文件，也应该被清理）
var missingFileIds = allVideos
.Where(x =>
// 情况1：有路径但文件不存在
(!string.IsNullOrWhiteSpace(x.VideoSavePath) && !System.IO.File.Exists(x.VideoSavePath)) ||
// 情况2：路径为空（这些记录没有视频文件，也应该清理）
string.IsNullOrWhiteSpace(x.VideoSavePath)
)
.Select(x => x.Id)
.ToList();
if (!missingFileIds.Any())
{
Serilog.Log.Debug("清理检查：所有视频文件都存在，无需清理");
return 0;
}
var pathEmptyCount = allVideos.Count(x => string.IsNullOrWhiteSpace(x.VideoSavePath));
var fileMissingCount = allVideos.Count(x => !string.IsNullOrWhiteSpace(x.VideoSavePath) && !System.IO.File.Exists(x.VideoSavePath));
Serilog.Log.Debug($"清理检查：发现 {missingFileIds.Count} 条需要清理的记录（路径为空：{pathEmptyCount} 条，文件不存在：{fileMissingCount} 条）");
// 批量删除
var deletedCount = await _dyCollectVideoRepository.DeleteByIdsAsync(missingFileIds);
Serilog.Log.Debug($"清理完成：删除了 {deletedCount} 条记录");
return deletedCount;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "清理文件不存在的记录失败");
throw;
}
}
/// <summary>
/// 删除指定作者的所有视频数据
/// </summary>
/// <param name="authorName">作者名称</param>
/// <returns>删除的记录数量</returns>
public async Task<int> DeleteAuthorVideosAsync(string authorName)
{
try
{
var allVideos = await _dyCollectVideoRepository.GetAllAsync();
if (!allVideos.Any())
return 0;
// 找出该作者的所有视频ID
var authorVideoIds = allVideos
.Where(x => !string.IsNullOrWhiteSpace(x.Author) && x.Author.Equals(authorName, StringComparison.OrdinalIgnoreCase))
.Select(x => x.Id)
.ToList();
if (!authorVideoIds.Any())
{
Serilog.Log.Debug($"删除作者视频：未找到作者 {authorName} 的视频记录");
return 0;
}
Serilog.Log.Debug($"删除作者视频：发现作者 {authorName} 的 {authorVideoIds.Count} 条视频记录");
// 批量删除
var deletedCount = await _dyCollectVideoRepository.DeleteByIdsAsync(authorVideoIds);
Serilog.Log.Debug($"删除完成：删除了 {deletedCount} 条记录");
return deletedCount;
}
catch (Exception ex)
{
Serilog.Log.Error(ex, $"删除作者 {authorName} 的视频失败");
throw;
}
}
}
service/DyHttpClientService.cs
+31 -4
View File
@@ -68,11 +68,24 @@ namespace dy.net.service
if (respose.IsSuccessStatusCode)
{
var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<CollectVideoInfo>(data);
Serilog.Log.Debug($"SyncCollectVideos API响应长度: {data?.Length ?? 0}");
var result = JsonConvert.DeserializeObject<CollectVideoInfo>(data);
if (result != null)
{
Serilog.Log.Debug($"SyncCollectVideos 解析结果: HasMore={result.HasMore}, AwemeList数量={result.AwemeList?.Count ?? 0}");
}
else
{
var preview = data != null && data.Length > 0 ? data.Substring(0, Math.Min(500, data.Length)) : "空响应";
Serilog.Log.Warning($"SyncCollectVideos 解析失败，响应内容前500字符: {preview}");
}
return result;
}
else
{
Serilog.Log.Error($"SyncCollectVideos fail: {respose.StatusCode}");
var errorContent = await respose.Content.ReadAsStringAsync();
var errorPreview = errorContent != null && errorContent.Length > 0 ? errorContent.Substring(0, Math.Min(500, errorContent.Length)) : "空响应";
Serilog.Log.Error($"SyncCollectVideos fail: {respose.StatusCode}, 响应内容: {errorPreview}");
return null;
}
}
@@ -129,15 +142,29 @@ namespace dy.net.service
// 构建请求URL
var queryString = new FormUrlEncodedContent(dics);
string fullUrl = $"{DouYinApi}/favorite?{await queryString.ReadAsStringAsync()}";
Serilog.Log.Debug($"SyncFavoriteVideos 请求URL: {fullUrl.Substring(0, Math.Min(200, fullUrl.Length))}...");
var respose = await httpClient.GetAsync(fullUrl);
if (respose.IsSuccessStatusCode)
{
var data = await respose.Content.ReadAsStringAsync();
return JsonConvert.DeserializeObject<CollectVideoInfo>(data);
Serilog.Log.Debug($"SyncFavoriteVideos API响应长度: {data?.Length ?? 0}");
var result = JsonConvert.DeserializeObject<CollectVideoInfo>(data);
if (result != null)
{
Serilog.Log.Debug($"SyncFavoriteVideos 解析结果: HasMore={result.HasMore}, AwemeList数量={result.AwemeList?.Count ?? 0}");
}
else
{
var preview = data != null && data.Length > 0 ? data.Substring(0, Math.Min(500, data.Length)) : "空响应";
Serilog.Log.Warning($"SyncFavoriteVideos 解析失败，响应内容前500字符: {preview}");
}
return result;
}
else
{
Serilog.Log.Error($"SyncFavoriteVideos fail: {respose.StatusCode}");
var errorContent = await respose.Content.ReadAsStringAsync();
var errorPreview = errorContent != null && errorContent.Length > 0 ? errorContent.Substring(0, Math.Min(500, errorContent.Length)) : "空响应";
Serilog.Log.Error($"SyncFavoriteVideos fail: {respose.StatusCode}, 响应内容: {errorPreview}");
return null;
}
}