bf1942/bili-sync-ai
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: 支持解析联合投稿 (#681)

Browse Source
This commit is contained in:
ᴀᴍᴛᴏᴀᴇʀ
2026-03-24 16:25:42 +08:00
committed by GitHub
parent 09604fd283
commit 04448c6d8f
14 changed files with 221 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -416,6 +416,7 @@ name = "bili_sync_entity"
version = "2.10.4"
dependencies = [
"derivative",
"either",
"regex",
"sea-orm",
"serde",

1
Cargo.toml
View File

@@ -31,6 +31,7 @@ dashmap = "6.1.0"
derivative = "2.2.0"
dirs = "6.0.0"
dunce = "1.0.5"
either = "1.15.0"
enum_dispatch = "0.3.13"
float-ord = "0.3.2"
futures = "0.3.31"

6
crates/bili_sync/src/bilibili/favorite_list.rs
View File

@@ -16,12 +16,6 @@ pub struct FavoriteListInfo {
pub title: String,
}
#[derive(Debug, serde::Deserialize)]
pub struct Upper<T> {
pub mid: T,
pub name: String,
pub face: String,
}
impl<'a> FavoriteList<'a> {
pub fn new(client: &'a BiliClient, fid: String, credential: &'a Credential) -> Self {
Self {

10
crates/bili_sync/src/bilibili/mod.rs
View File

@@ -4,6 +4,7 @@ use std::sync::Arc;
pub use analyzer::{BestStream, FilterOption};
use anyhow::{Context, Result, bail, ensure};
use arc_swap::ArcSwapOption;
use bili_sync_entity::upper_vec::Upper;
use chrono::serde::ts_seconds;
use chrono::{DateTime, Utc};
pub use client::{BiliClient, Client};
@@ -13,7 +14,6 @@ pub use danmaku::DanmakuOption;
pub use dynamic::Dynamic;
pub use error::BiliError;
pub use favorite_list::FavoriteList;
use favorite_list::Upper;
pub use me::Me;
use once_cell::sync::Lazy;
use reqwest::{RequestBuilder, StatusCode};
@@ -133,7 +133,9 @@ pub enum VideoInfo {
#[serde(rename = "pic")]
cover: String,
#[serde(rename = "owner")]
upper: Upper<i64>,
upper: Upper<i64, String>,
#[serde(default)]
staff: Option<Vec<Upper<i64, String>>>,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(rename = "pubdate", with = "ts_seconds")]
@@ -152,7 +154,7 @@ pub enum VideoInfo {
bvid: String,
intro: String,
cover: String,
upper: Upper<i64>,
upper: Upper<i64, String>,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(with = "ts_seconds")]
@@ -170,7 +172,7 @@ pub enum VideoInfo {
#[serde(rename = "pic")]
cover: String,
#[serde(rename = "owner")]
upper: Upper<i64>,
upper: Upper<i64, String>,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(rename = "add_at", with = "ts_seconds")]

4
crates/bili_sync/src/bilibili/submission.rs
View File

@@ -1,10 +1,10 @@
use anyhow::{Context, Result, anyhow};
use async_stream::try_stream;
use bili_sync_entity::upper_vec::Upper;
use futures::Stream;
use reqwest::Method;
use serde_json::Value;
use crate::bilibili::favorite_list::Upper;
use crate::bilibili::{BiliClient, Credential, Dynamic, ErrorForStatusExt, MIXIN_KEY, Validate, VideoInfo, WbiSign};
pub struct Submission<'a> {
client: &'a BiliClient,
@@ -27,7 +27,7 @@ impl<'a> Submission<'a> {
}
}
pub async fn get_info(&self) -> Result<Upper<String>> {
pub async fn get_info(&self) -> Result<Upper<String, String>> {
let mut res = self
.client
.request(

2
crates/bili_sync/src/utils/convert.rs
View File

@@ -133,6 +133,7 @@ impl VideoInfo {
intro,
cover,
upper,
staff,
ctime,
pubtime,
state,
@@ -165,6 +166,7 @@ impl VideoInfo {
upper_id: Set(upper.mid),
upper_name: Set(upper.name),
upper_face: Set(upper.face),
staff: Set(staff.map(Into::into)),
..base_model.into_active_model()
},
_ => unreachable!(),

101
crates/bili_sync/src/utils/nfo.rs
View File

@@ -1,4 +1,5 @@
use anyhow::Result;
use bili_sync_entity::upper_vec::Upper as EntityUpper;
use bili_sync_entity::*;
use chrono::NaiveDateTime;
use quick_xml::Error;
@@ -20,9 +21,7 @@ pub struct Movie<'a> {
pub name: &'a str,
pub intro: &'a str,
pub bvid: &'a str,
pub upper_id: i64,
pub upper_name: &'a str,
pub upper_thumb: &'a str,
pub uppers: Vec<EntityUpper<i64, &'a str>>,
pub premiered: NaiveDateTime,
pub tags: Option<Vec<String>>,
}
@@ -31,9 +30,7 @@ pub struct TVShow<'a> {
pub name: &'a str,
pub intro: &'a str,
pub bvid: &'a str,
pub upper_id: i64,
pub upper_name: &'a str,
pub upper_thumb: &'a str,
pub uppers: Vec<EntityUpper<i64, &'a str>>,
pub premiered: NaiveDateTime,
pub tags: Option<Vec<String>>,
}
@@ -87,24 +84,26 @@ impl NFO<'_> {
.create_element("title")
.write_text_content_async(BytesText::new(movie.name))
.await?;
writer
.create_element("actor")
.write_inner_content_async::<_, _, Error>(|writer| async move {
writer
.create_element("name")
.write_text_content_async(BytesText::new(&movie.upper_id.to_string()))
.await?;
writer
.create_element("role")
.write_text_content_async(BytesText::new(movie.upper_name))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(movie.upper_thumb))
.await?;
Ok(writer)
})
.await?;
for upper in movie.uppers {
writer
.create_element("actor")
.write_inner_content_async::<_, _, Error>(|writer| async move {
writer
.create_element("name")
.write_text_content_async(BytesText::new(&upper.mid.to_string()))
.await?;
writer
.create_element("role")
.write_text_content_async(BytesText::new(upper.role().as_ref()))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(upper.face))
.await?;
Ok(writer)
})
.await?;
}
writer
.create_element("year")
.write_text_content_async(BytesText::new(&movie.premiered.format("%Y").to_string()))
@@ -145,24 +144,26 @@ impl NFO<'_> {
.create_element("title")
.write_text_content_async(BytesText::new(tvshow.name))
.await?;
writer
.create_element("actor")
.write_inner_content_async::<_, _, Error>(|writer| async move {
writer
.create_element("name")
.write_text_content_async(BytesText::new(&tvshow.upper_id.to_string()))
.await?;
writer
.create_element("role")
.write_text_content_async(BytesText::new(tvshow.upper_name))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(tvshow.upper_thumb))
.await?;
Ok(writer)
})
.await?;
for upper in tvshow.uppers {
writer
.create_element("actor")
.write_inner_content_async::<_, _, Error>(|writer| async move {
writer
.create_element("name")
.write_text_content_async(BytesText::new(&upper.mid.to_string()))
.await?;
writer
.create_element("role")
.write_text_content_async(BytesText::new(upper.role().as_ref()))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(upper.face))
.await?;
Ok(writer)
})
.await?;
}
writer
.create_element("year")
.write_text_content_async(BytesText::new(&tvshow.premiered.format("%Y").to_string()))
@@ -320,7 +321,7 @@ mod tests {
</tvshow>"#,
);
assert_eq!(
NFO::Upper((&video).to_nfo(NFOTimeType::FavTime))
NFO::Upper(((&video, &video.uppers().next().unwrap())).to_nfo(NFOTimeType::FavTime))
.generate_nfo()
.await
.unwrap(),
@@ -366,9 +367,7 @@ impl<'a> ToNFO<'a, Movie<'a>> for &'a video::Model {
name: &self.name,
intro: &self.intro,
bvid: &self.bvid,
upper_id: self.upper_id,
upper_name: &self.upper_name,
upper_thumb: &self.upper_face,
uppers: self.uppers().collect(),
premiered: match nfo_time_type {
NFOTimeType::FavTime => self.favtime,
NFOTimeType::PubTime => self.pubtime,
@@ -384,9 +383,7 @@ impl<'a> ToNFO<'a, TVShow<'a>> for &'a video::Model {
name: &self.name,
intro: &self.intro,
bvid: &self.bvid,
upper_id: self.upper_id,
upper_name: &self.upper_name,
upper_thumb: &self.upper_face,
uppers: self.uppers().collect(),
premiered: match nfo_time_type {
NFOTimeType::FavTime => self.favtime,
NFOTimeType::PubTime => self.pubtime,
@@ -396,11 +393,11 @@ impl<'a> ToNFO<'a, TVShow<'a>> for &'a video::Model {
}
}
impl<'a> ToNFO<'a, Upper> for &'a video::Model {
impl<'a> ToNFO<'a, Upper> for (&video::Model, &EntityUpper<i64, &str>) {
fn to_nfo(&'a self, _nfo_time_type: NFOTimeType) -> Upper {
Upper {
upper_id: self.upper_id.to_string(),
pubtime: self.pubtime,
upper_id: self.1.mid.to_string(),
pubtime: self.0.pubtime,
}
}
}

88
crates/bili_sync/src/workflow.rs
View File

@@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};
use std::pin::Pin;
use anyhow::{Context, Result, anyhow, bail};
use bili_sync_entity::upper_vec::Upper;
use bili_sync_entity::*;
use futures::stream::FuturesUnordered;
use futures::{Stream, StreamExt, TryStreamExt};
@@ -188,13 +189,18 @@ pub async fn download_unprocessed_videos(
let downloader = Downloader::new(bili_client.client.clone());
let cx = DownloadContext::new(bili_client, video_source, template, connection, &downloader, config);
let unhandled_videos_pages = filter_unhandled_video_pages(video_source.filter_expr(), connection).await?;
let mut assigned_upper = HashSet::new();
let mut assigned_upper_ids = HashSet::new();
let tasks = unhandled_videos_pages
.into_iter()
.map(|(video_model, pages_model)| {
let should_download_upper = !assigned_upper.contains(&video_model.upper_id);
assigned_upper.insert(video_model.upper_id);
download_video_pages(video_model, pages_model, &semaphore, should_download_upper, cx)
// 这里按理说是可以直接拿到 assigned_uppers 的，但rust 会错误地认为它引用了 local variable
// 导致编译出错，暂时先这样单独提取出一个 owned 的 upper id 列表，再在任务内部筛选
let task_uids = video_model
.uppers()
.map(|u| u.mid)
.filter(|uid| assigned_upper_ids.insert(*uid))
.collect::<Vec<_>>();
download_video_pages(video_model, pages_model, &semaphore, task_uids, cx)
})
.collect::<FuturesUnordered<_>>();
let mut risk_control_related_error = None;
@@ -229,7 +235,7 @@ pub async fn download_video_pages(
video_model: video::Model,
page_models: Vec<page::Model>,
semaphore: &Semaphore,
should_download_upper: bool,
upper_uids: Vec<i64>,
cx: DownloadContext<'_>,
) -> Result<video::ActiveModel> {
let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
@@ -245,14 +251,26 @@ pub async fn download_video_pages(
)
};
fs::create_dir_all(&base_path).await?;
let base_path = dunce::canonicalize(base_path).context("canonicalize video path failed")?;
let upper_id = video_model.upper_id.to_string();
let base_upper_path = cx
.config
.upper_path
.join(upper_id.chars().next().context("upper_id is empty")?.to_string())
.join(upper_id);
let is_single_page = video_model.single_page.context("single_page is null")?;
let uppers_with_path = video_model
.uppers()
.filter_map(|u| {
if !upper_uids.contains(&u.mid) {
None
} else {
let id_string = u.mid.to_string();
Some((
u,
cx.config
.upper_path
.join(id_string.chars().next()?.to_string())
.join(id_string),
))
}
})
.collect::<Vec<_>>();
// 对于单页视频，page 的下载已经足够
// 对于多页视频，page 下载仅包含了分集内容，需要额外补上视频的 poster 的 tvshow.nfo
let (res_1, res_2, res_3, res_4, res_5) = tokio::join!(
@@ -273,16 +291,15 @@ pub async fn download_video_pages(
),
// 下载 Up 主头像
fetch_upper_face(
separate_status[2] && should_download_upper && !cx.config.skip_option.no_upper,
&video_model,
base_upper_path.join("folder.jpg"),
separate_status[2] && !cx.config.skip_option.no_upper,
&uppers_with_path,
cx
),
// 生成 Up 主信息的 nfo
generate_upper_nfo(
separate_status[3] && should_download_upper && !cx.config.skip_option.no_upper,
separate_status[3] && !cx.config.skip_option.no_upper,
&video_model,
base_upper_path.join("person.nfo"),
&uppers_with_path,
cx,
),
// 分发并执行分页下载的任务
@@ -714,33 +731,48 @@ pub async fn fetch_video_poster(
pub async fn fetch_upper_face(
should_run: bool,
video_model: &video::Model,
upper_face_path: PathBuf,
uppers_with_path: &[(Upper<i64, &str>, PathBuf)],
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
if !should_run || uppers_with_path.is_empty() {
return Ok(ExecutionStatus::Skipped);
}
cx.downloader
.fetch(
&video_model.upper_face,
&upper_face_path,
&cx.config.concurrent_limit.download,
)
.await?;
let tasks = uppers_with_path
.iter()
.map(|(upper, base_path)| async move {
cx.downloader
.fetch(
upper.face,
&base_path.join("folder.jpg"),
&cx.config.concurrent_limit.download,
)
.await?;
Ok::<(), anyhow::Error>(())
})
.collect::<FuturesUnordered<_>>();
tasks.try_collect::<Vec<()>>().await?;
Ok(ExecutionStatus::Succeeded)
}
pub async fn generate_upper_nfo(
should_run: bool,
video_model: &video::Model,
nfo_path: PathBuf,
uppers_with_path: &[(Upper<i64, &str>, PathBuf)],
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
generate_nfo(NFO::Upper(video_model.to_nfo(cx.config.nfo_time_type)), nfo_path).await?;
let tasks = uppers_with_path
.iter()
.map(|(upper, base_path)| {
generate_nfo(
NFO::Upper((video_model, upper).to_nfo(cx.config.nfo_time_type)),
base_path.join("person.nfo"),
)
})
.collect::<FuturesUnordered<_>>();
tasks.try_collect::<Vec<()>>().await?;
Ok(ExecutionStatus::Succeeded)
}

1
crates/bili_sync_entity/Cargo.toml
View File

@@ -6,6 +6,7 @@ publish = { workspace = true }
[dependencies]
derivative = { workspace = true }
either = { workspace = true }
regex = { workspace = true }
sea-orm = { workspace = true }
serde = { workspace = true }

1
crates/bili_sync_entity/src/custom_type/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod rule;
pub mod string_vec;
pub mod upper_vec;

48
crates/bili_sync_entity/src/custom_type/upper_vec.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::borrow::Cow;
use sea_orm::FromJsonQueryResult;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Upper<T, S> {
pub mid: T,
pub name: S,
pub face: S,
pub title: Option<S>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)]
pub struct UpperVec(pub Vec<Upper<i64, String>>);
impl From<Vec<Upper<i64, String>>> for UpperVec {
fn from(value: Vec<Upper<i64, String>>) -> Self {
Self(value)
}
}
impl From<UpperVec> for Vec<Upper<i64, String>> {
fn from(value: UpperVec) -> Self {
value.0
}
}
impl<T: Copy> Upper<T, String> {
pub fn as_ref(&self) -> Upper<T, &str> {
Upper {
mid: self.mid,
name: self.name.as_str(),
face: self.face.as_str(),
title: self.title.as_deref(),
}
}
}
impl<T, S: AsRef<str>> Upper<T, S> {
pub fn role(&self) -> Cow<'_, str> {
if let Some(title) = &self.title {
Cow::Owned(format!("{}「{}」", self.name.as_ref(), title.as_ref()))
} else {
Cow::Borrowed(self.name.as_ref())
}
}
}

18
crates/bili_sync_entity/src/entities/video.rs
View File

@@ -1,8 +1,10 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15
use either::Either;
use sea_orm::entity::prelude::*;
use crate::string_vec::StringVec;
use crate::upper_vec::{Upper, UpperVec};
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
#[sea_orm(table_name = "video")]
@@ -16,6 +18,7 @@ pub struct Model {
pub upper_id: i64,
pub upper_name: String,
pub upper_face: String,
pub staff: Option<UpperVec>,
pub name: String,
pub path: String,
pub category: i32,
@@ -33,6 +36,21 @@ pub struct Model {
pub created_at: String,
}
impl Model {
pub fn uppers(&self) -> Either<impl Iterator<Item = Upper<i64, &str>>, impl Iterator<Item = Upper<i64, &str>>> {
if let Some(staff) = self.staff.as_ref() {
Either::Left(staff.0.iter().map(|u| u.as_ref()))
} else {
Either::Right(std::iter::once(Upper::<i64, &str> {
mid: self.upper_id,
name: self.upper_name.as_str(),
face: self.upper_face.as_str(),
title: None,
}))
}
}
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(has_many = "super::page::Entity")]

2
crates/bili_sync_migration/src/lib.rs
View File

@@ -10,6 +10,7 @@ mod m20250613_043257_add_config;
mod m20250712_080013_add_video_created_at_index;
mod m20250903_094454_add_rule_and_should_download;
mod m20251009_123713_add_use_dynamic_api;
mod m20260324_055217_add_staff;
pub struct Migrator;
@@ -27,6 +28,7 @@ impl MigratorTrait for Migrator {
Box::new(m20250712_080013_add_video_created_at_index::Migration),
Box::new(m20250903_094454_add_rule_and_should_download::Migration),
Box::new(m20251009_123713_add_use_dynamic_api::Migration),
Box::new(m20260324_055217_add_staff::Migration),
]
}
}

30
crates/bili_sync_migration/src/m20260324_055217_add_staff.rs Normal file
View File

@@ -0,0 +1,30 @@
use sea_orm_migration::prelude::*;
use sea_orm_migration::schema::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Video::Table)
.add_column(text_null(Video::Staff))
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(Table::alter().table(Video::Table).drop_column(Video::Staff).to_owned())
.await
}
}
#[derive(DeriveIden)]
enum Video {
Table,
Staff,
}