From 04448c6d8f10830ccc90e32d49e978f4b219957e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=B4=80=E1=B4=8D=E1=B4=9B=E1=B4=8F=E1=B4=80=E1=B4=87?= =?UTF-8?q?=CA=80?= Date: Tue, 24 Mar 2026 16:25:42 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E8=81=94=E5=90=88=E6=8A=95=E7=A8=BF=20(#681)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + Cargo.toml | 1 + .../bili_sync/src/bilibili/favorite_list.rs | 6 -- crates/bili_sync/src/bilibili/mod.rs | 10 +- crates/bili_sync/src/bilibili/submission.rs | 4 +- crates/bili_sync/src/utils/convert.rs | 2 + crates/bili_sync/src/utils/nfo.rs | 101 +++++++++--------- crates/bili_sync/src/workflow.rs | 88 ++++++++++----- crates/bili_sync_entity/Cargo.toml | 1 + .../bili_sync_entity/src/custom_type/mod.rs | 1 + .../src/custom_type/upper_vec.rs | 48 +++++++++ crates/bili_sync_entity/src/entities/video.rs | 18 ++++ crates/bili_sync_migration/src/lib.rs | 2 + .../src/m20260324_055217_add_staff.rs | 30 ++++++ 14 files changed, 221 insertions(+), 92 deletions(-) create mode 100644 crates/bili_sync_entity/src/custom_type/upper_vec.rs create mode 100644 crates/bili_sync_migration/src/m20260324_055217_add_staff.rs diff --git a/Cargo.lock b/Cargo.lock index 0418da4..5fcc2b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -416,6 +416,7 @@ name = "bili_sync_entity" version = "2.10.4" dependencies = [ "derivative", + "either", "regex", "sea-orm", "serde", diff --git a/Cargo.toml b/Cargo.toml index a5bd5f4..9c67b87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ dashmap = "6.1.0" derivative = "2.2.0" dirs = "6.0.0" dunce = "1.0.5" +either = "1.15.0" enum_dispatch = "0.3.13" float-ord = "0.3.2" futures = "0.3.31" diff --git a/crates/bili_sync/src/bilibili/favorite_list.rs b/crates/bili_sync/src/bilibili/favorite_list.rs index 1e253cf..f8f43d8 100644 --- a/crates/bili_sync/src/bilibili/favorite_list.rs +++ b/crates/bili_sync/src/bilibili/favorite_list.rs @@ -16,12 +16,6 @@ pub struct FavoriteListInfo { pub title: String, } -#[derive(Debug, serde::Deserialize)] -pub struct Upper { - pub mid: T, - pub name: String, - pub face: String, -} impl<'a> FavoriteList<'a> { pub fn new(client: &'a BiliClient, fid: String, credential: &'a Credential) -> Self { Self { diff --git a/crates/bili_sync/src/bilibili/mod.rs b/crates/bili_sync/src/bilibili/mod.rs index 41eb7a4..c7ed37e 100644 --- a/crates/bili_sync/src/bilibili/mod.rs +++ b/crates/bili_sync/src/bilibili/mod.rs @@ -4,6 +4,7 @@ use std::sync::Arc; pub use analyzer::{BestStream, FilterOption}; use anyhow::{Context, Result, bail, ensure}; use arc_swap::ArcSwapOption; +use bili_sync_entity::upper_vec::Upper; use chrono::serde::ts_seconds; use chrono::{DateTime, Utc}; pub use client::{BiliClient, Client}; @@ -13,7 +14,6 @@ pub use danmaku::DanmakuOption; pub use dynamic::Dynamic; pub use error::BiliError; pub use favorite_list::FavoriteList; -use favorite_list::Upper; pub use me::Me; use once_cell::sync::Lazy; use reqwest::{RequestBuilder, StatusCode}; @@ -133,7 +133,9 @@ pub enum VideoInfo { #[serde(rename = "pic")] cover: String, #[serde(rename = "owner")] - upper: Upper, + upper: Upper, + #[serde(default)] + staff: Option>>, #[serde(with = "ts_seconds")] ctime: DateTime, #[serde(rename = "pubdate", with = "ts_seconds")] @@ -152,7 +154,7 @@ pub enum VideoInfo { bvid: String, intro: String, cover: String, - upper: Upper, + upper: Upper, #[serde(with = "ts_seconds")] ctime: DateTime, #[serde(with = "ts_seconds")] @@ -170,7 +172,7 @@ pub enum VideoInfo { #[serde(rename = "pic")] cover: String, #[serde(rename = "owner")] - upper: Upper, + upper: Upper, #[serde(with = "ts_seconds")] ctime: DateTime, #[serde(rename = "add_at", with = "ts_seconds")] diff --git a/crates/bili_sync/src/bilibili/submission.rs b/crates/bili_sync/src/bilibili/submission.rs index 5740650..ea73d1f 100644 --- a/crates/bili_sync/src/bilibili/submission.rs +++ b/crates/bili_sync/src/bilibili/submission.rs @@ -1,10 +1,10 @@ use anyhow::{Context, Result, anyhow}; use async_stream::try_stream; +use bili_sync_entity::upper_vec::Upper; use futures::Stream; use reqwest::Method; use serde_json::Value; -use crate::bilibili::favorite_list::Upper; use crate::bilibili::{BiliClient, Credential, Dynamic, ErrorForStatusExt, MIXIN_KEY, Validate, VideoInfo, WbiSign}; pub struct Submission<'a> { client: &'a BiliClient, @@ -27,7 +27,7 @@ impl<'a> Submission<'a> { } } - pub async fn get_info(&self) -> Result> { + pub async fn get_info(&self) -> Result> { let mut res = self .client .request( diff --git a/crates/bili_sync/src/utils/convert.rs b/crates/bili_sync/src/utils/convert.rs index 9803faa..a4ea696 100644 --- a/crates/bili_sync/src/utils/convert.rs +++ b/crates/bili_sync/src/utils/convert.rs @@ -133,6 +133,7 @@ impl VideoInfo { intro, cover, upper, + staff, ctime, pubtime, state, @@ -165,6 +166,7 @@ impl VideoInfo { upper_id: Set(upper.mid), upper_name: Set(upper.name), upper_face: Set(upper.face), + staff: Set(staff.map(Into::into)), ..base_model.into_active_model() }, _ => unreachable!(), diff --git a/crates/bili_sync/src/utils/nfo.rs b/crates/bili_sync/src/utils/nfo.rs index 92053c8..3dd509a 100644 --- a/crates/bili_sync/src/utils/nfo.rs +++ b/crates/bili_sync/src/utils/nfo.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bili_sync_entity::upper_vec::Upper as EntityUpper; use bili_sync_entity::*; use chrono::NaiveDateTime; use quick_xml::Error; @@ -20,9 +21,7 @@ pub struct Movie<'a> { pub name: &'a str, pub intro: &'a str, pub bvid: &'a str, - pub upper_id: i64, - pub upper_name: &'a str, - pub upper_thumb: &'a str, + pub uppers: Vec>, pub premiered: NaiveDateTime, pub tags: Option>, } @@ -31,9 +30,7 @@ pub struct TVShow<'a> { pub name: &'a str, pub intro: &'a str, pub bvid: &'a str, - pub upper_id: i64, - pub upper_name: &'a str, - pub upper_thumb: &'a str, + pub uppers: Vec>, pub premiered: NaiveDateTime, pub tags: Option>, } @@ -87,24 +84,26 @@ impl NFO<'_> { .create_element("title") .write_text_content_async(BytesText::new(movie.name)) .await?; - writer - .create_element("actor") - .write_inner_content_async::<_, _, Error>(|writer| async move { - writer - .create_element("name") - .write_text_content_async(BytesText::new(&movie.upper_id.to_string())) - .await?; - writer - .create_element("role") - .write_text_content_async(BytesText::new(movie.upper_name)) - .await?; - writer - .create_element("thumb") - .write_text_content_async(BytesText::new(movie.upper_thumb)) - .await?; - Ok(writer) - }) - .await?; + for upper in movie.uppers { + writer + .create_element("actor") + .write_inner_content_async::<_, _, Error>(|writer| async move { + writer + .create_element("name") + .write_text_content_async(BytesText::new(&upper.mid.to_string())) + .await?; + writer + .create_element("role") + .write_text_content_async(BytesText::new(upper.role().as_ref())) + .await?; + writer + .create_element("thumb") + .write_text_content_async(BytesText::new(upper.face)) + .await?; + Ok(writer) + }) + .await?; + } writer .create_element("year") .write_text_content_async(BytesText::new(&movie.premiered.format("%Y").to_string())) @@ -145,24 +144,26 @@ impl NFO<'_> { .create_element("title") .write_text_content_async(BytesText::new(tvshow.name)) .await?; - writer - .create_element("actor") - .write_inner_content_async::<_, _, Error>(|writer| async move { - writer - .create_element("name") - .write_text_content_async(BytesText::new(&tvshow.upper_id.to_string())) - .await?; - writer - .create_element("role") - .write_text_content_async(BytesText::new(tvshow.upper_name)) - .await?; - writer - .create_element("thumb") - .write_text_content_async(BytesText::new(tvshow.upper_thumb)) - .await?; - Ok(writer) - }) - .await?; + for upper in tvshow.uppers { + writer + .create_element("actor") + .write_inner_content_async::<_, _, Error>(|writer| async move { + writer + .create_element("name") + .write_text_content_async(BytesText::new(&upper.mid.to_string())) + .await?; + writer + .create_element("role") + .write_text_content_async(BytesText::new(upper.role().as_ref())) + .await?; + writer + .create_element("thumb") + .write_text_content_async(BytesText::new(upper.face)) + .await?; + Ok(writer) + }) + .await?; + } writer .create_element("year") .write_text_content_async(BytesText::new(&tvshow.premiered.format("%Y").to_string())) @@ -320,7 +321,7 @@ mod tests { "#, ); assert_eq!( - NFO::Upper((&video).to_nfo(NFOTimeType::FavTime)) + NFO::Upper(((&video, &video.uppers().next().unwrap())).to_nfo(NFOTimeType::FavTime)) .generate_nfo() .await .unwrap(), @@ -366,9 +367,7 @@ impl<'a> ToNFO<'a, Movie<'a>> for &'a video::Model { name: &self.name, intro: &self.intro, bvid: &self.bvid, - upper_id: self.upper_id, - upper_name: &self.upper_name, - upper_thumb: &self.upper_face, + uppers: self.uppers().collect(), premiered: match nfo_time_type { NFOTimeType::FavTime => self.favtime, NFOTimeType::PubTime => self.pubtime, @@ -384,9 +383,7 @@ impl<'a> ToNFO<'a, TVShow<'a>> for &'a video::Model { name: &self.name, intro: &self.intro, bvid: &self.bvid, - upper_id: self.upper_id, - upper_name: &self.upper_name, - upper_thumb: &self.upper_face, + uppers: self.uppers().collect(), premiered: match nfo_time_type { NFOTimeType::FavTime => self.favtime, NFOTimeType::PubTime => self.pubtime, @@ -396,11 +393,11 @@ impl<'a> ToNFO<'a, TVShow<'a>> for &'a video::Model { } } -impl<'a> ToNFO<'a, Upper> for &'a video::Model { +impl<'a> ToNFO<'a, Upper> for (&video::Model, &EntityUpper) { fn to_nfo(&'a self, _nfo_time_type: NFOTimeType) -> Upper { Upper { - upper_id: self.upper_id.to_string(), - pubtime: self.pubtime, + upper_id: self.1.mid.to_string(), + pubtime: self.0.pubtime, } } } diff --git a/crates/bili_sync/src/workflow.rs b/crates/bili_sync/src/workflow.rs index 672be9a..495f5f9 100644 --- a/crates/bili_sync/src/workflow.rs +++ b/crates/bili_sync/src/workflow.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use anyhow::{Context, Result, anyhow, bail}; +use bili_sync_entity::upper_vec::Upper; use bili_sync_entity::*; use futures::stream::FuturesUnordered; use futures::{Stream, StreamExt, TryStreamExt}; @@ -188,13 +189,18 @@ pub async fn download_unprocessed_videos( let downloader = Downloader::new(bili_client.client.clone()); let cx = DownloadContext::new(bili_client, video_source, template, connection, &downloader, config); let unhandled_videos_pages = filter_unhandled_video_pages(video_source.filter_expr(), connection).await?; - let mut assigned_upper = HashSet::new(); + let mut assigned_upper_ids = HashSet::new(); let tasks = unhandled_videos_pages .into_iter() .map(|(video_model, pages_model)| { - let should_download_upper = !assigned_upper.contains(&video_model.upper_id); - assigned_upper.insert(video_model.upper_id); - download_video_pages(video_model, pages_model, &semaphore, should_download_upper, cx) + // 这里按理说是可以直接拿到 assigned_uppers 的，但rust 会错误地认为它引用了 local variable + // 导致编译出错，暂时先这样单独提取出一个 owned 的 upper id 列表，再在任务内部筛选 + let task_uids = video_model + .uppers() + .map(|u| u.mid) + .filter(|uid| assigned_upper_ids.insert(*uid)) + .collect::>(); + download_video_pages(video_model, pages_model, &semaphore, task_uids, cx) }) .collect::>(); let mut risk_control_related_error = None; @@ -229,7 +235,7 @@ pub async fn download_video_pages( video_model: video::Model, page_models: Vec, semaphore: &Semaphore, - should_download_upper: bool, + upper_uids: Vec, cx: DownloadContext<'_>, ) -> Result { let _permit = semaphore.acquire().await.context("acquire semaphore failed")?; @@ -245,14 +251,26 @@ pub async fn download_video_pages( ) }; fs::create_dir_all(&base_path).await?; + let base_path = dunce::canonicalize(base_path).context("canonicalize video path failed")?; - let upper_id = video_model.upper_id.to_string(); - let base_upper_path = cx - .config - .upper_path - .join(upper_id.chars().next().context("upper_id is empty")?.to_string()) - .join(upper_id); let is_single_page = video_model.single_page.context("single_page is null")?; + let uppers_with_path = video_model + .uppers() + .filter_map(|u| { + if !upper_uids.contains(&u.mid) { + None + } else { + let id_string = u.mid.to_string(); + Some(( + u, + cx.config + .upper_path + .join(id_string.chars().next()?.to_string()) + .join(id_string), + )) + } + }) + .collect::>(); // 对于单页视频，page 的下载已经足够 // 对于多页视频，page 下载仅包含了分集内容，需要额外补上视频的 poster 的 tvshow.nfo let (res_1, res_2, res_3, res_4, res_5) = tokio::join!( @@ -273,16 +291,15 @@ pub async fn download_video_pages( ), // 下载 Up 主头像 fetch_upper_face( - separate_status[2] && should_download_upper && !cx.config.skip_option.no_upper, - &video_model, - base_upper_path.join("folder.jpg"), + separate_status[2] && !cx.config.skip_option.no_upper, + &uppers_with_path, cx ), // 生成 Up 主信息的 nfo generate_upper_nfo( - separate_status[3] && should_download_upper && !cx.config.skip_option.no_upper, + separate_status[3] && !cx.config.skip_option.no_upper, &video_model, - base_upper_path.join("person.nfo"), + &uppers_with_path, cx, ), // 分发并执行分页下载的任务 @@ -714,33 +731,48 @@ pub async fn fetch_video_poster( pub async fn fetch_upper_face( should_run: bool, - video_model: &video::Model, - upper_face_path: PathBuf, + uppers_with_path: &[(Upper, PathBuf)], cx: DownloadContext<'_>, ) -> Result { - if !should_run { + if !should_run || uppers_with_path.is_empty() { return Ok(ExecutionStatus::Skipped); } - cx.downloader - .fetch( - &video_model.upper_face, - &upper_face_path, - &cx.config.concurrent_limit.download, - ) - .await?; + let tasks = uppers_with_path + .iter() + .map(|(upper, base_path)| async move { + cx.downloader + .fetch( + upper.face, + &base_path.join("folder.jpg"), + &cx.config.concurrent_limit.download, + ) + .await?; + Ok::<(), anyhow::Error>(()) + }) + .collect::>(); + tasks.try_collect::>().await?; Ok(ExecutionStatus::Succeeded) } pub async fn generate_upper_nfo( should_run: bool, video_model: &video::Model, - nfo_path: PathBuf, + uppers_with_path: &[(Upper, PathBuf)], cx: DownloadContext<'_>, ) -> Result { if !should_run { return Ok(ExecutionStatus::Skipped); } - generate_nfo(NFO::Upper(video_model.to_nfo(cx.config.nfo_time_type)), nfo_path).await?; + let tasks = uppers_with_path + .iter() + .map(|(upper, base_path)| { + generate_nfo( + NFO::Upper((video_model, upper).to_nfo(cx.config.nfo_time_type)), + base_path.join("person.nfo"), + ) + }) + .collect::>(); + tasks.try_collect::>().await?; Ok(ExecutionStatus::Succeeded) } diff --git a/crates/bili_sync_entity/Cargo.toml b/crates/bili_sync_entity/Cargo.toml index 51edb4e..5f499ab 100644 --- a/crates/bili_sync_entity/Cargo.toml +++ b/crates/bili_sync_entity/Cargo.toml @@ -6,6 +6,7 @@ publish = { workspace = true } [dependencies] derivative = { workspace = true } +either = { workspace = true } regex = { workspace = true } sea-orm = { workspace = true } serde = { workspace = true } diff --git a/crates/bili_sync_entity/src/custom_type/mod.rs b/crates/bili_sync_entity/src/custom_type/mod.rs index d87b844..a441f76 100644 --- a/crates/bili_sync_entity/src/custom_type/mod.rs +++ b/crates/bili_sync_entity/src/custom_type/mod.rs @@ -1,2 +1,3 @@ pub mod rule; pub mod string_vec; +pub mod upper_vec; diff --git a/crates/bili_sync_entity/src/custom_type/upper_vec.rs b/crates/bili_sync_entity/src/custom_type/upper_vec.rs new file mode 100644 index 0000000..6970700 --- /dev/null +++ b/crates/bili_sync_entity/src/custom_type/upper_vec.rs @@ -0,0 +1,48 @@ +use std::borrow::Cow; + +use sea_orm::FromJsonQueryResult; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Upper { + pub mid: T, + pub name: S, + pub face: S, + pub title: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)] +pub struct UpperVec(pub Vec>); + +impl From>> for UpperVec { + fn from(value: Vec>) -> Self { + Self(value) + } +} + +impl From for Vec> { + fn from(value: UpperVec) -> Self { + value.0 + } +} + +impl Upper { + pub fn as_ref(&self) -> Upper { + Upper { + mid: self.mid, + name: self.name.as_str(), + face: self.face.as_str(), + title: self.title.as_deref(), + } + } +} + +impl> Upper { + pub fn role(&self) -> Cow<'_, str> { + if let Some(title) = &self.title { + Cow::Owned(format!("{}「{}」", self.name.as_ref(), title.as_ref())) + } else { + Cow::Borrowed(self.name.as_ref()) + } + } +} diff --git a/crates/bili_sync_entity/src/entities/video.rs b/crates/bili_sync_entity/src/entities/video.rs index 9c51b79..a0f8b6d 100644 --- a/crates/bili_sync_entity/src/entities/video.rs +++ b/crates/bili_sync_entity/src/entities/video.rs @@ -1,8 +1,10 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15 +use either::Either; use sea_orm::entity::prelude::*; use crate::string_vec::StringVec; +use crate::upper_vec::{Upper, UpperVec}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)] #[sea_orm(table_name = "video")] @@ -16,6 +18,7 @@ pub struct Model { pub upper_id: i64, pub upper_name: String, pub upper_face: String, + pub staff: Option, pub name: String, pub path: String, pub category: i32, @@ -33,6 +36,21 @@ pub struct Model { pub created_at: String, } +impl Model { + pub fn uppers(&self) -> Either>, impl Iterator>> { + if let Some(staff) = self.staff.as_ref() { + Either::Left(staff.0.iter().map(|u| u.as_ref())) + } else { + Either::Right(std::iter::once(Upper:: { + mid: self.upper_id, + name: self.upper_name.as_str(), + face: self.upper_face.as_str(), + title: None, + })) + } + } +} + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] pub enum Relation { #[sea_orm(has_many = "super::page::Entity")] diff --git a/crates/bili_sync_migration/src/lib.rs b/crates/bili_sync_migration/src/lib.rs index d3e34a5..44c3a9c 100644 --- a/crates/bili_sync_migration/src/lib.rs +++ b/crates/bili_sync_migration/src/lib.rs @@ -10,6 +10,7 @@ mod m20250613_043257_add_config; mod m20250712_080013_add_video_created_at_index; mod m20250903_094454_add_rule_and_should_download; mod m20251009_123713_add_use_dynamic_api; +mod m20260324_055217_add_staff; pub struct Migrator; @@ -27,6 +28,7 @@ impl MigratorTrait for Migrator { Box::new(m20250712_080013_add_video_created_at_index::Migration), Box::new(m20250903_094454_add_rule_and_should_download::Migration), Box::new(m20251009_123713_add_use_dynamic_api::Migration), + Box::new(m20260324_055217_add_staff::Migration), ] } } diff --git a/crates/bili_sync_migration/src/m20260324_055217_add_staff.rs b/crates/bili_sync_migration/src/m20260324_055217_add_staff.rs new file mode 100644 index 0000000..c4a638e --- /dev/null +++ b/crates/bili_sync_migration/src/m20260324_055217_add_staff.rs @@ -0,0 +1,30 @@ +use sea_orm_migration::prelude::*; +use sea_orm_migration::schema::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(Video::Table) + .add_column(text_null(Video::Staff)) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table(Table::alter().table(Video::Table).drop_column(Video::Staff).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum Video { + Table, + Staff, +}