bf1942/bili-sync-ai
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: 大范围重构，支持视频合集下载 (#97)

Browse Source
This commit is contained in:
ᴀᴍᴛᴏᴀᴇʀ
2024-07-03 18:57:12 +08:00
committed by GitHub
parent 097f885050
commit 4c9ad2318c
27 changed files with 1545 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
crates/bili_sync/src/adapter/collection.rs Normal file
View File

@@ -0,0 +1,241 @@
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use bili_sync_entity::*;
use bili_sync_migration::OnConflict;
use filenamify::filenamify;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
use super::VideoListModel;
use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo};
use crate::config::TEMPLATE;
use crate::utils::id_time_key;
use crate::utils::model::create_video_pages;
use crate::utils::status::Status;
pub async fn collection_from<'a>(
collection_item: &'a CollectionItem,
path: &Path,
bili_client: &'a BiliClient,
connection: &DatabaseConnection,
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
let collection = Collection::new(bili_client, collection_item);
let collection_info = collection.get_info().await?;
collection::Entity::insert(collection::ActiveModel {
s_id: Set(collection_info.sid),
m_id: Set(collection_info.mid),
r#type: Set(collection_info.collection_type.into()),
name: Set(collection_info.name.clone()),
path: Set(path.to_string_lossy().to_string()),
..Default::default()
})
.on_conflict(
OnConflict::columns([
collection::Column::SId,
collection::Column::MId,
collection::Column::Type,
])
.update_columns([collection::Column::Name, collection::Column::Path])
.to_owned(),
)
.exec(connection)
.await?;
Ok((
Box::new(
collection::Entity::find()
.filter(
collection::Column::SId
.eq(collection_item.sid.clone())
.and(collection::Column::MId.eq(collection_item.mid.clone()))
.and(collection::Column::Type.eq(Into::<i32>::into(collection_item.collection_type.clone()))),
)
.one(connection)
.await?
.unwrap(),
),
Box::pin(collection.into_simple_video_stream()),
))
}
use async_trait::async_trait;
#[async_trait]
impl VideoListModel for collection::Model {
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
Ok(video::Entity::find()
.filter(video::Column::CollectionId.eq(self.id))
.count(connection)
.await?)
}
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
Ok(video::Entity::find()
.filter(
video::Column::CollectionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null()),
)
.all(connection)
.await?)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
Ok(video::Entity::find()
.filter(
video::Column::CollectionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(Status::handled()))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null()),
)
.find_with_related(page::Entity)
.all(connection)
.await?)
}
async fn exist_labels(
&self,
videos_info: &[VideoInfo],
connection: &DatabaseConnection,
) -> Result<HashSet<String>> {
let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::<Vec<_>>();
Ok(video::Entity::find()
.filter(
video::Column::CollectionId
.eq(self.id)
.and(video::Column::Bvid.is_in(bvids)),
)
.select_only()
.columns([video::Column::Bvid, video::Column::Favtime])
.into_tuple()
.all(connection)
.await?
.into_iter()
.map(|(bvid, time)| id_time_key(&bvid, &time))
.collect::<HashSet<_>>())
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
video_model.collection_id = Set(Some(self.id));
if let Some(fmt_args) = &video_info.to_fmt_args() {
video_model.path = Set(Path::new(&self.path)
.join(filenamify(
TEMPLATE
.render("video", fmt_args)
.unwrap_or_else(|_| video_info.bvid().to_string()),
))
.to_string_lossy()
.to_string());
}
video_model
}
async fn fetch_videos_detail(
&self,
bili_clent: &BiliClient,
videos_model: Vec<video::Model>,
connection: &DatabaseConnection,
) -> Result<()> {
for video_model in videos_model {
let video = Video::new(bili_clent, video_model.bvid.clone());
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Ok((tags, view_info)) => {
let VideoInfo::View { pages, .. } = &view_info else {
unreachable!("view_info must be VideoInfo::View")
};
let txn = connection.begin().await?;
// 将分页信息写入数据库
create_video_pages(pages, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model));
video_active_model.single_page = Set(Some(pages.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
error!(
"获取视频 {} - {} 的详细信息失败，错误为：{}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
continue;
}
};
}
Ok(())
}
fn log_fetch_video_start(&self) {
info!(
"开始获取{} {} - {} 的视频与分页信息...",
CollectionType::from(self.r#type),
self.s_id,
self.name
);
}
fn log_fetch_video_end(&self) {
info!(
"获取{} {} - {} 的视频与分页信息完成",
CollectionType::from(self.r#type),
self.s_id,
self.name
);
}
fn log_download_video_start(&self) {
info!(
"开始下载{}: {} - {} 中所有未处理过的视频...",
CollectionType::from(self.r#type),
self.s_id,
self.name
);
}
fn log_download_video_end(&self) {
info!(
"下载{}: {} - {} 中未处理过的视频完成",
CollectionType::from(self.r#type),
self.s_id,
self.name
);
}
fn log_refresh_video_start(&self) {
info!(
"开始扫描{}: {} - {} 的新视频...",
CollectionType::from(self.r#type),
self.s_id,
self.name
);
}
fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
info!(
"扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
CollectionType::from(self.r#type),
self.s_id,
self.name,
got_count,
new_count,
);
}
}

199
crates/bili_sync/src/adapter/favorite.rs Normal file
View File

@@ -0,0 +1,199 @@
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use bili_sync_entity::*;
use bili_sync_migration::OnConflict;
use filenamify::filenamify;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
use super::VideoListModel;
use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo};
use crate::config::TEMPLATE;
use crate::utils::id_time_key;
use crate::utils::model::create_video_pages;
use crate::utils::status::Status;
pub async fn favorite_from<'a>(
fid: &str,
path: &Path,
bili_client: &'a BiliClient,
connection: &DatabaseConnection,
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
let favorite = FavoriteList::new(bili_client, fid.to_owned());
let favorite_info = favorite.get_info().await?;
favorite::Entity::insert(favorite::ActiveModel {
f_id: Set(favorite_info.id),
name: Set(favorite_info.title.clone()),
path: Set(path.to_string_lossy().to_string()),
..Default::default()
})
.on_conflict(
OnConflict::column(favorite::Column::FId)
.update_columns([favorite::Column::Name, favorite::Column::Path])
.to_owned(),
)
.exec(connection)
.await?;
Ok((
Box::new(
favorite::Entity::find()
.filter(favorite::Column::FId.eq(favorite_info.id))
.one(connection)
.await?
.unwrap(),
),
Box::pin(favorite.into_video_stream()),
))
}
use async_trait::async_trait;
#[async_trait]
impl VideoListModel for favorite::Model {
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
Ok(video::Entity::find()
.filter(video::Column::FavoriteId.eq(self.id))
.count(connection)
.await?)
}
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
Ok(video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null()),
)
.all(connection)
.await?)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
Ok(video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(Status::handled()))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null()),
)
.find_with_related(page::Entity)
.all(connection)
.await?)
}
async fn exist_labels(
&self,
videos_info: &[VideoInfo],
connection: &DatabaseConnection,
) -> Result<HashSet<String>> {
let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::<Vec<_>>();
Ok(video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(self.id)
.and(video::Column::Bvid.is_in(bvids)),
)
.select_only()
.columns([video::Column::Bvid, video::Column::Favtime])
.into_tuple()
.all(connection)
.await?
.into_iter()
.map(|(bvid, time)| id_time_key(&bvid, &time))
.collect::<HashSet<_>>())
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
video_model.favorite_id = Set(Some(self.id));
if let Some(fmt_args) = &video_info.to_fmt_args() {
video_model.path = Set(Path::new(&self.path)
.join(filenamify(
TEMPLATE
.render("video", fmt_args)
.unwrap_or_else(|_| video_info.bvid().to_string()),
))
.to_string_lossy()
.to_string());
}
video_model
}
async fn fetch_videos_detail(
&self,
bili_clent: &BiliClient,
videos_model: Vec<video::Model>,
connection: &DatabaseConnection,
) -> Result<()> {
for video_model in videos_model {
let video = Video::new(bili_clent, video_model.bvid.clone());
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await;
match info {
Ok((tags, pages_info)) => {
let txn = connection.begin().await?;
// 将分页信息写入数据库
create_video_pages(&pages_info, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.single_page = Set(Some(pages_info.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
error!(
"获取视频 {} - {} 的详细信息失败，错误为：{}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
continue;
}
};
}
Ok(())
}
fn log_fetch_video_start(&self) {
info!("开始获取收藏夹 {} - {} 的视频与分页信息...", self.f_id, self.name);
}
fn log_fetch_video_end(&self) {
info!("获取收藏夹 {} - {} 的视频与分页信息完成", self.f_id, self.name);
}
fn log_download_video_start(&self) {
info!("开始下载收藏夹: {} - {} 中所有未处理过的视频...", self.f_id, self.name);
}
fn log_download_video_end(&self) {
info!("下载收藏夹: {} - {} 中未处理过的视频完成", self.f_id, self.name);
}
fn log_refresh_video_start(&self) {
info!("开始扫描收藏夹: {} - {} 的新视频...", self.f_id, self.name);
}
fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
info!(
"扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
self.f_id, self.name, got_count, new_count
);
}
}

89
crates/bili_sync/src/adapter/mod.rs Normal file
View File

@@ -0,0 +1,89 @@
mod collection;
mod favorite;
use std::collections::HashSet;
use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use async_trait::async_trait;
use bili_sync_migration::IntoIden;
pub use collection::collection_from;
pub use favorite::favorite_from;
use futures::Stream;
use sea_orm::DatabaseConnection;
use crate::bilibili::{BiliClient, CollectionItem, VideoInfo};
pub enum Args<'a> {
Favorite { fid: &'a str },
Collection { collection_item: &'a CollectionItem },
}
pub async fn video_list_from<'a>(
args: Args<'a>,
path: &Path,
bili_client: &'a BiliClient,
connection: &DatabaseConnection,
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
match args {
Args::Favorite { fid } => favorite_from(fid, path, bili_client, connection).await,
Args::Collection { collection_item } => collection_from(collection_item, path, bili_client, connection).await,
}
}
pub const fn unique_video_columns() -> impl IntoIterator<Item = impl IntoIden> {
[
bili_sync_entity::video::Column::CollectionId,
bili_sync_entity::video::Column::FavoriteId,
bili_sync_entity::video::Column::Bvid,
]
}
#[async_trait]
pub trait VideoListModel {
/* 逻辑相关 */
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64>;
/// 获取未填充的视频
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<bili_sync_entity::video::Model>>;
/// 获取未处理的视频和分页
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(bili_sync_entity::video::Model, Vec<bili_sync_entity::page::Model>)>>;
/// 获取该批次视频的存在标记
async fn exist_labels(&self, videos_info: &[VideoInfo], connection: &DatabaseConnection)
-> Result<HashSet<String>>;
/// 获取视频信息对应的视频 model
fn video_model_by_info(
&self,
video_info: &VideoInfo,
base_model: Option<bili_sync_entity::video::Model>,
) -> bili_sync_entity::video::ActiveModel;
/// 获取视频 model 中缺失的信息
async fn fetch_videos_detail(
&self,
bili_client: &BiliClient,
videos_model: Vec<bili_sync_entity::video::Model>,
connection: &DatabaseConnection,
) -> Result<()>;
/* 日志相关 */
fn log_fetch_video_start(&self);
fn log_fetch_video_end(&self);
fn log_download_video_start(&self);
fn log_download_video_end(&self);
fn log_refresh_video_start(&self);
fn log_refresh_video_end(&self, got_count: usize, new_count: u64);
}

265
crates/bili_sync/src/bilibili/collection.rs Normal file
View File

@@ -0,0 +1,265 @@
#![allow(dead_code)]
use std::fmt::{Display, Formatter};
use anyhow::Result;
use async_stream::stream;
use futures::Stream;
use reqwest::Method;
use serde::Deserialize;
use serde_json::Value;
use crate::bilibili::{BiliClient, Validate, VideoInfo};
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
pub enum CollectionType {
Series,
Season,
}
impl From<CollectionType> for i32 {
fn from(v: CollectionType) -> Self {
match v {
CollectionType::Series => 1,
CollectionType::Season => 2,
}
}
}
impl From<i32> for CollectionType {
fn from(v: i32) -> Self {
match v {
1 => CollectionType::Series,
2 => CollectionType::Season,
_ => panic!("invalid collection type"),
}
}
}
impl Display for CollectionType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let s = match self {
CollectionType::Series => "视频列表",
CollectionType::Season => "视频合集",
};
write!(f, "{}", s)
}
}
#[derive(PartialEq, Eq, Hash, Debug)]
pub struct CollectionItem {
pub mid: String,
pub sid: String,
pub collection_type: CollectionType,
}
pub struct Collection<'a> {
client: &'a BiliClient,
collection: &'a CollectionItem,
}
#[derive(Debug, PartialEq)]
pub struct CollectionInfo {
pub name: String,
pub mid: i64,
pub sid: i64,
pub collection_type: CollectionType,
}
impl<'de> Deserialize<'de> for CollectionInfo {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
struct CollectionInfoRaw {
mid: i64,
name: String,
season_id: Option<i64>,
series_id: Option<i64>,
}
let raw = CollectionInfoRaw::deserialize(deserializer)?;
let (sid, collection_type) = match (raw.season_id, raw.series_id) {
(Some(sid), None) => (sid, CollectionType::Season),
(None, Some(sid)) => (sid, CollectionType::Series),
_ => return Err(serde::de::Error::custom("invalid collection info")),
};
Ok(CollectionInfo {
mid: raw.mid,
name: raw.name,
sid,
collection_type,
})
}
}
impl<'a> Collection<'a> {
pub fn new(client: &'a BiliClient, collection: &'a CollectionItem) -> Self {
Self { client, collection }
}
pub async fn get_info(&self) -> Result<CollectionInfo> {
let meta = match self.collection.collection_type {
// 没有找到专门获取 Season 信息的接口，所以直接获取第一页，从里面取 meta 信息
CollectionType::Season => self.get_videos(1).await?["data"]["meta"].take(),
CollectionType::Series => self.get_series_info().await?["data"]["meta"].take(),
};
Ok(serde_json::from_value(meta)?)
}
async fn get_series_info(&self) -> Result<Value> {
assert!(
self.collection.collection_type == CollectionType::Series,
"collection type is not series"
);
self.client
.request(Method::GET, "https://api.bilibili.com/x/series/series")
.query(&[("series_id", self.collection.sid.as_str())])
.send()
.await?
.error_for_status()?
.json::<Value>()
.await?
.validate()
}
async fn get_videos(&self, page: i32) -> Result<Value> {
let page = page.to_string();
let (url, query) = match self.collection.collection_type {
CollectionType::Series => (
"https://api.bilibili.com/x/series/archives",
vec![
("mid", self.collection.mid.as_str()),
("series_id", self.collection.sid.as_str()),
("only_normal", "true"),
("sort", "desc"),
("pn", page.as_str()),
("ps", "30"),
],
),
CollectionType::Season => (
"https://api.bilibili.com/x/polymer/web-space/seasons_archives_list",
vec![
("mid", self.collection.mid.as_str()),
("season_id", self.collection.sid.as_str()),
("sort_reverse", "true"),
("page_num", page.as_str()),
("page_size", "30"),
],
),
};
self.client
.request(Method::GET, url)
.query(&query)
.send()
.await?
.error_for_status()?
.json::<Value>()
.await?
.validate()
}
pub fn into_simple_video_stream(self) -> impl Stream<Item = VideoInfo> + 'a {
stream! {
let mut page = 1;
loop {
let mut videos = match self.get_videos(page).await {
Ok(v) => v,
Err(e) => {
error!("failed to get videos of collection {:?} page {}: {}", self.collection, page, e);
break;
},
};
if !videos["data"]["archives"].is_array() {
warn!("no videos found in collection {:?} page {}", self.collection, page);
break;
}
let videos_info = match serde_json::from_value::<Vec<VideoInfo>>(videos["data"]["archives"].take()) {
Ok(v) => v,
Err(e) => {
error!("failed to parse videos of collection {:?} page {}: {}", self.collection, page, e);
break;
},
};
for video_info in videos_info.into_iter(){
yield video_info;
}
let fields = match self.collection.collection_type{
CollectionType::Series => ["num", "size", "total"],
CollectionType::Season => ["page_num", "page_size", "total"],
};
let fields = fields.into_iter().map(|f| videos["data"]["page"][f].as_i64()).collect::<Option<Vec<i64>>>().map(|v| (v[0], v[1], v[2]));
let Some((num, size, total)) = fields else {
error!("failed to get pages of collection {:?} page {}: {:?}", self.collection, page, fields);
break;
};
if num * size >= total {
break;
}
page += 1;
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_collection_info_parse() {
let testcases = vec![
(
r#"
{
"category": 0,
"cover": "https://archive.biliimg.com/bfs/archive/a6fbf7a7b9f4af09d9cf40482268634df387ef68.jpg",
"description": "",
"mid": 521722088,
"name": "合集·【命运方舟全剧情解说】",
"ptime": 1714701600,
"season_id": 1987140,
"total": 10
}
"#,
CollectionInfo {
mid: 521722088,
name: "合集·【命运方舟全剧情解说】".to_owned(),
sid: 1987140,
collection_type: CollectionType::Season,
},
),
(
r#"
{
"series_id": 387212,
"mid": 521722088,
"name": "提瓦特冒险记",
"description": "原神沙雕般的游戏体验",
"keywords": [
""
],
"creator": "",
"state": 2,
"last_update_ts": 1633167320,
"total": 3,
"ctime": 1633167320,
"mtime": 1633167320,
"raw_keywords": "",
"category": 1
}
"#,
CollectionInfo {
mid: 521722088,
name: "提瓦特冒险记".to_owned(),
sid: 387212,
collection_type: CollectionType::Series,
},
),
];
for (json, expect) in testcases {
let info: CollectionInfo = serde_json::from_str(json).unwrap();
assert_eq!(info, expect);
}
}
}

22
crates/bili_sync/src/bilibili/favorite_list.rs
View File

@@ -1,11 +1,9 @@
use anyhow::Result;
use async_stream::stream;
use chrono::serde::ts_seconds;
use chrono::{DateTime, Utc};
use futures::Stream;
use serde_json::Value;
use crate::bilibili::{BiliClient, Validate};
use crate::bilibili::{BiliClient, Validate, VideoInfo};
pub struct FavoriteList<'a> {
client: &'a BiliClient,
fid: String,
@@ -17,24 +15,6 @@ pub struct FavoriteListInfo {
pub title: String,
}
#[derive(Debug, serde::Deserialize)]
pub struct VideoInfo {
pub title: String,
#[serde(rename = "type")]
pub vtype: i32,
pub bvid: String,
pub intro: String,
pub cover: String,
pub upper: Upper,
#[serde(with = "ts_seconds")]
pub ctime: DateTime<Utc>,
#[serde(with = "ts_seconds")]
pub fav_time: DateTime<Utc>,
#[serde(with = "ts_seconds")]
pub pubtime: DateTime<Utc>,
pub attr: i32,
}
#[derive(Debug, serde::Deserialize)]
pub struct Upper {
pub mid: i64,

59
crates/bili_sync/src/bilibili/mod.rs
View File

@@ -1,14 +1,19 @@
pub use analyzer::{BestStream, FilterOption};
use anyhow::{bail, Result};
use chrono::serde::ts_seconds;
use chrono::{DateTime, Utc};
pub use client::{BiliClient, Client};
pub use collection::{Collection, CollectionItem, CollectionType};
pub use credential::Credential;
pub use danmaku::DanmakuOption;
pub use error::BiliError;
pub use favorite_list::{FavoriteList, FavoriteListInfo, VideoInfo};
pub use favorite_list::FavoriteList;
use favorite_list::Upper;
pub use video::{Dimension, PageInfo, Video};
mod analyzer;
mod client;
mod collection;
mod credential;
mod danmaku;
mod error;
@@ -35,3 +40,55 @@ impl Validate for serde_json::Value {
Ok(self)
}
}
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
/// 注意此处的顺序是有要求的，因为对于 untagged 的 enum 来说，serde 会按照顺序匹配
/// > There is no explicit tag identifying which variant the data contains.
/// > Serde will try to match the data against each variant in order and the first one that deserializes successfully is the one returned.
pub enum VideoInfo {
/// 从视频详情接口获取的视频信息
View {
title: String,
bvid: String,
#[serde(rename = "desc")]
intro: String,
#[serde(rename = "pic")]
cover: String,
#[serde(rename = "owner")]
upper: Upper,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(rename = "pubdate", with = "ts_seconds")]
pubtime: DateTime<Utc>,
pages: Vec<PageInfo>,
state: i32,
},
/// 从收藏夹中获取的视频信息
Detail {
title: String,
#[serde(rename = "type")]
vtype: i32,
bvid: String,
intro: String,
cover: String,
upper: Upper,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(with = "ts_seconds")]
fav_time: DateTime<Utc>,
#[serde(with = "ts_seconds")]
pubtime: DateTime<Utc>,
attr: i32,
},
/// 从视频列表中获取的视频信息
Simple {
bvid: String,
#[serde(rename = "pic")]
cover: String,
#[serde(with = "ts_seconds")]
ctime: DateTime<Utc>,
#[serde(rename = "pubdate", with = "ts_seconds")]
pubtime: DateTime<Utc>,
},
}

22
crates/bili_sync/src/bilibili/video.rs
View File

@@ -7,7 +7,7 @@ use reqwest::Method;
use crate::bilibili::analyzer::PageAnalyzer;
use crate::bilibili::client::BiliClient;
use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply};
use crate::bilibili::Validate;
use crate::bilibili::{Validate, VideoInfo};
static MASK_CODE: u64 = 2251799813685247;
static XOR_CODE: u64 = 23442827791579;
@@ -61,6 +61,22 @@ impl<'a> Video<'a> {
Self { client, aid, bvid }
}
#[allow(dead_code)]
/// 直接调用视频信息接口获取详细的视频信息
pub async fn get_view_info(&self) -> Result<VideoInfo> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view")
.query(&[("aid", &self.aid), ("bvid", &self.bvid)])
.send()
.await?
.error_for_status()?
.json::<serde_json::Value>()
.await?
.validate()?;
Ok(serde_json::from_value(res["data"].take())?)
}
pub async fn get_pages(&self) -> Result<Vec<PageInfo>> {
let mut res = self
.client
@@ -158,8 +174,8 @@ fn bvid_to_aid(bvid: &str) -> u64 {
mod tests {
use super::*;
#[tokio::test]
async fn test_bvid_to_aid() {
#[test]
fn test_bvid_to_aid() {
assert_eq!(bvid_to_aid("BV1Tr421n746"), 1401752220u64);
assert_eq!(bvid_to_aid("BV1sH4y1s7fe"), 1051892992u64);
}

102
crates/bili_sync/src/config.rs
View File

@@ -5,10 +5,30 @@ use std::sync::Arc;
use anyhow::Result;
use arc_swap::ArcSwapOption;
use handlebars::handlebars_helper;
use once_cell::sync::Lazy;
use serde::de::{Deserializer, MapAccess, Visitor};
use serde::ser::SerializeMap;
use serde::{Deserialize, Serialize};
use crate::bilibili::{Credential, DanmakuOption, FilterOption};
use crate::bilibili::{CollectionItem, CollectionType, Credential, DanmakuOption, FilterOption};
pub static TEMPLATE: Lazy<handlebars::Handlebars> = Lazy::new(|| {
let mut handlebars = handlebars::Handlebars::new();
handlebars_helper!(truncate: |s: String, len: usize| {
if s.chars().count() > len {
s.chars().take(len).collect::<String>()
} else {
s.to_string()
}
});
handlebars.register_helper("truncate", Box::new(truncate));
handlebars
.register_template_string("video", &CONFIG.video_name)
.unwrap();
handlebars.register_template_string("page", &CONFIG.page_name).unwrap();
handlebars
});
pub static CONFIG: Lazy<Config> = Lazy::new(|| {
let config = Config::load().unwrap_or_else(|err| {
@@ -42,6 +62,12 @@ pub struct Config {
#[serde(default)]
pub danmaku_option: DanmakuOption,
pub favorite_list: HashMap<String, PathBuf>,
#[serde(
default,
serialize_with = "serialize_collection_list",
deserialize_with = "deserialize_collection_list"
)]
pub collection_list: HashMap<CollectionItem, PathBuf>,
pub video_name: Cow<'static, str>,
pub page_name: Cow<'static, str>,
pub interval: u64,
@@ -65,6 +91,7 @@ impl Default for Config {
filter_option: FilterOption::default(),
danmaku_option: DanmakuOption::default(),
favorite_list: HashMap::new(),
collection_list: HashMap::new(),
video_name: Cow::Borrowed("{{title}}"),
page_name: Cow::Borrowed("{{bvid}}"),
interval: 1200,
@@ -78,9 +105,9 @@ impl Config {
/// 简单的预检查
pub fn check(&self) {
let mut ok = true;
if self.favorite_list.is_empty() {
if self.favorite_list.is_empty() && self.collection_list.is_empty() {
ok = false;
error!("未设置需监听的收藏夹，程序空转没有意义");
error!("未设置需监听的收藏夹或视频合集，程序空转没有意义");
}
for path in self.favorite_list.values() {
if !path.is_absolute() {
@@ -141,6 +168,75 @@ impl Config {
}
}
fn serialize_collection_list<S>(
collection_list: &HashMap<CollectionItem, PathBuf>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut map = serializer.serialize_map(Some(collection_list.len()))?;
for (k, v) in collection_list {
let prefix = match k.collection_type {
CollectionType::Series => "series",
CollectionType::Season => "season",
};
map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?;
}
map.end()
}
fn deserialize_collection_list<'de, D>(deserializer: D) -> Result<HashMap<CollectionItem, PathBuf>, D::Error>
where
D: Deserializer<'de>,
{
struct CollectionListVisitor;
impl<'de> Visitor<'de> for CollectionListVisitor {
type Value = HashMap<CollectionItem, PathBuf>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a map of collection list")
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut collection_list = HashMap::new();
while let Some((key, value)) = map.next_entry::<String, PathBuf>()? {
let collection_item = match key.split(':').collect::<Vec<&str>>().as_slice() {
[prefix, mid, sid] => {
let collection_type = match *prefix {
"series" => CollectionType::Series,
"season" => CollectionType::Season,
_ => {
return Err(serde::de::Error::custom(
"invalid collection type, should be series or season",
))
}
};
CollectionItem {
mid: mid.to_string(),
sid: sid.to_string(),
collection_type,
}
}
_ => {
return Err(serde::de::Error::custom(
"invalid collection key, should be series:mid:sid or season:mid:sid",
))
}
};
collection_list.insert(collection_item, value);
}
Ok(collection_list)
}
}
deserializer.deserialize_map(CollectionListVisitor)
}
use clap::Parser;
#[derive(Parser)]

3
crates/bili_sync/src/core/mod.rs
View File

@@ -1,3 +0,0 @@
pub mod command;
pub mod status;
pub mod utils;

17
crates/bili_sync/src/database.rs
View File

@@ -1,13 +1,15 @@
use anyhow::Result;
use bili_sync_migration::{Migrator, MigratorTrait};
use sea_orm::{ConnectOptions, Database, DatabaseConnection};
use tokio::fs;
use crate::config::CONFIG_DIR;
fn database_url() -> String {
format!("sqlite://{}?mode=rwc", CONFIG_DIR.join("data.sqlite").to_string_lossy())
}
pub async fn database_connection() -> Result<DatabaseConnection> {
let target = CONFIG_DIR.join("data.sqlite");
fs::create_dir_all(&*CONFIG_DIR).await?;
let mut option = ConnectOptions::new(format!("sqlite://{}?mode=rwc", target.to_str().unwrap()));
let mut option = ConnectOptions::new(database_url());
option
.max_connections(100)
.min_connections(5)
@@ -15,6 +17,9 @@ pub async fn database_connection() -> Result<DatabaseConnection> {
Ok(Database::connect(option).await?)
}
pub async fn migrate_database(connection: &DatabaseConnection) -> Result<()> {
Ok(Migrator::up(connection, None).await?)
pub async fn migrate_database() -> Result<()> {
// 注意此处使用内部构造的 DatabaseConnection，而不是通过 database_connection() 获取
// 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题，而使用默认的连接选项不会
let connection = Database::connect(database_url()).await?;
Ok(Migrator::up(&connection, None).await?)
}

34
crates/bili_sync/src/main.rs
View File

@@ -1,34 +1,34 @@
#[macro_use]
extern crate tracing;
mod adapter;
mod bilibili;
mod config;
mod core;
mod database;
mod downloader;
mod error;
mod utils;
mod workflow;
use std::time::Duration;
use config::ARGS;
use once_cell::sync::Lazy;
use tokio::time;
use crate::adapter::Args;
use crate::bilibili::BiliClient;
use crate::config::CONFIG;
use crate::core::command::process_favorite_list;
use crate::core::utils::init_logger;
use crate::config::{ARGS, CONFIG};
use crate::database::{database_connection, migrate_database};
use crate::utils::init_logger;
use crate::workflow::process_video_list;
#[tokio::main]
async fn main() {
Lazy::force(&ARGS);
init_logger(&ARGS.log_level);
Lazy::force(&CONFIG);
migrate_database().await.expect("数据库迁移失败");
let connection = database_connection().await.expect("获取数据库连接失败");
let mut anchor = chrono::Local::now().date_naive();
let bili_client = BiliClient::new();
let connection = database_connection().await.unwrap();
migrate_database(&connection).await.unwrap();
loop {
if let Err(e) = bili_client.is_login().await {
error!("检查登录状态时遇到错误：{e}，等待下一轮执行");
@@ -44,12 +44,20 @@ async fn main() {
anchor = chrono::Local::now().date_naive();
}
for (fid, path) in &CONFIG.favorite_list {
if let Err(e) = process_favorite_list(&bili_client, fid, path, &connection).await {
// 可预期的错误都被内部处理了，这里漏出来应该是大问题
if let Err(e) = process_video_list(Args::Favorite { fid }, &bili_client, path, &connection).await {
error!("处理收藏夹 {fid} 时遇到非预期的错误：{e}");
}
}
info!("所有收藏夹处理完毕，等待下一轮执行");
time::sleep(Duration::from_secs(CONFIG.interval)).await;
info!("所有收藏夹处理完毕");
for (collection_item, path) in &CONFIG.collection_list {
if let Err(e) =
process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await
{
error!("处理合集 {collection_item:?} 时遇到非预期的错误：{e}");
}
}
info!("所有合集处理完毕");
info!("本轮任务执行完毕，等待下一轮执行");
tokio::time::sleep(std::time::Duration::from_secs(CONFIG.interval)).await;
}
}

131
crates/bili_sync/src/utils/convert.rs Normal file
View File

@@ -0,0 +1,131 @@
use sea_orm::ActiveValue::NotSet;
use sea_orm::{IntoActiveModel, Set};
use serde_json::json;
use crate::bilibili::VideoInfo;
use crate::utils::id_time_key;
impl VideoInfo {
/// 将 VideoInfo 转换为 ActiveModel
pub fn to_model(&self, base_model: Option<bili_sync_entity::video::Model>) -> bili_sync_entity::video::ActiveModel {
let base_model = match base_model {
Some(base_model) => base_model.into_active_model(),
None => {
let mut tmp_model = bili_sync_entity::video::Model::default().into_active_model();
// 注意此处要把 id 设置成 NotSet，否则 id 会是 Unchanged(0)
tmp_model.id = NotSet;
tmp_model
}
};
match self {
VideoInfo::Simple {
bvid,
cover,
ctime,
pubtime,
} => bili_sync_entity::video::ActiveModel {
bvid: Set(bvid.clone()),
cover: Set(cover.clone()),
ctime: Set(ctime.naive_utc()),
pubtime: Set(pubtime.naive_utc()),
category: Set(2), // 视频合集里的内容类型肯定是视频
valid: Set(true),
..base_model
},
VideoInfo::Detail {
title,
vtype,
bvid,
intro,
cover,
upper,
ctime,
fav_time,
pubtime,
attr,
} => bili_sync_entity::video::ActiveModel {
bvid: Set(bvid.clone()),
name: Set(title.clone()),
category: Set(*vtype),
intro: Set(intro.clone()),
cover: Set(cover.clone()),
ctime: Set(ctime.naive_utc()),
pubtime: Set(pubtime.naive_utc()),
favtime: Set(fav_time.naive_utc()),
download_status: Set(0),
valid: Set(*attr == 0),
tags: Set(None),
single_page: Set(None),
upper_id: Set(upper.mid),
upper_name: Set(upper.name.clone()),
upper_face: Set(upper.face.clone()),
..base_model
},
VideoInfo::View {
title,
bvid,
intro,
cover,
upper,
ctime,
pubtime,
state,
..
} => bili_sync_entity::video::ActiveModel {
bvid: Set(bvid.clone()),
name: Set(title.clone()),
category: Set(2), // 视频合集里的内容类型肯定是视频
intro: Set(intro.clone()),
cover: Set(cover.clone()),
ctime: Set(ctime.naive_utc()),
pubtime: Set(pubtime.naive_utc()),
favtime: Set(pubtime.naive_utc()), // 合集不包括 fav_time，使用发布时间代替
download_status: Set(0),
valid: Set(*state == 0),
tags: Set(None),
single_page: Set(None),
upper_id: Set(upper.mid),
upper_name: Set(upper.name.clone()),
upper_face: Set(upper.face.clone()),
..base_model
},
}
}
pub fn to_fmt_args(&self) -> Option<serde_json::Value> {
match self {
VideoInfo::Simple { .. } => None, // 不能从简单的视频信息中构造格式化参数
VideoInfo::Detail { title, bvid, upper, .. } => Some(json!({
"bvid": &bvid,
"title": &title,
"upper_name": &upper.name,
"upper_mid": &upper.mid,
})),
VideoInfo::View { title, bvid, upper, .. } => Some(json!({
"bvid": &bvid,
"title": &title,
"upper_name": &upper.name,
"upper_mid": &upper.mid,
})),
}
}
pub fn video_key(&self) -> String {
match self {
// 对于合集没有 fav_time，只能用 pubtime 代替
VideoInfo::Simple { bvid, pubtime, .. } => id_time_key(bvid, pubtime),
VideoInfo::Detail { bvid, fav_time, .. } => id_time_key(bvid, fav_time),
// 详情接口返回的数据仅用于填充详情，不会被作为 video_key
_ => unreachable!(),
}
}
pub fn bvid(&self) -> &str {
match self {
VideoInfo::Simple { bvid, .. } => bvid,
VideoInfo::Detail { bvid, .. } => bvid,
// 同上
_ => unreachable!(),
}
}
}

23
crates/bili_sync/src/utils/mod.rs Normal file
View File

@@ -0,0 +1,23 @@
pub mod convert;
pub mod model;
pub mod nfo;
pub mod status;
use chrono::{DateTime, Utc};
use tracing_subscriber::util::SubscriberInitExt;
pub fn init_logger(log_level: &str) {
tracing_subscriber::fmt::Subscriber::builder()
.with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level))
.with_timer(tracing_subscriber::fmt::time::ChronoLocal::new(
"%Y-%m-%d %H:%M:%S%.3f".to_owned(),
))
.finish()
.try_init()
.expect("初始化日志失败");
}
/// 生成视频的唯一标记，均由 bvid 和时间戳构成
pub fn id_time_key(bvid: &String, time: &DateTime<Utc>) -> String {
format!("{}-{}", bvid, time.timestamp())
}

95
crates/bili_sync/src/utils/model.rs Normal file
View File

@@ -0,0 +1,95 @@
use anyhow::Result;
use bili_sync_entity::*;
use bili_sync_migration::OnConflict;
use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use crate::adapter::{unique_video_columns, VideoListModel};
use crate::bilibili::{PageInfo, VideoInfo};
/// 尝试创建 Video Model，如果发生冲突则忽略
pub async fn create_videos(
videos_info: &[VideoInfo],
video_list_model: &dyn VideoListModel,
connection: &DatabaseConnection,
) -> Result<()> {
let video_models = videos_info
.iter()
.map(|v| video_list_model.video_model_by_info(v, None))
.collect::<Vec<_>>();
video::Entity::insert_many(video_models)
.on_conflict(OnConflict::columns(unique_video_columns()).do_nothing().to_owned())
.do_nothing()
.exec(connection)
.await?;
Ok(())
}
/// 创建视频的所有分 P
pub async fn create_video_pages(
pages_info: &[PageInfo],
video_model: &video::Model,
connection: &impl ConnectionTrait,
) -> Result<()> {
let page_models = pages_info
.iter()
.map(move |p| {
let (width, height) = match &p.dimension {
Some(d) => {
if d.rotate == 0 {
(Some(d.width), Some(d.height))
} else {
(Some(d.height), Some(d.width))
}
}
None => (None, None),
};
page::ActiveModel {
video_id: Set(video_model.id),
cid: Set(p.cid),
pid: Set(p.page),
name: Set(p.name.clone()),
width: Set(width),
height: Set(height),
duration: Set(p.duration),
image: Set(p.first_frame.clone()),
download_status: Set(0),
..Default::default()
}
})
.collect::<Vec<page::ActiveModel>>();
page::Entity::insert_many(page_models)
.on_conflict(
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
.do_nothing()
.to_owned(),
)
.do_nothing()
.exec(connection)
.await?;
Ok(())
}
/// 更新视频 model 的下载状态
pub async fn update_videos_model(videos: Vec<video::ActiveModel>, connection: &DatabaseConnection) -> Result<()> {
video::Entity::insert_many(videos)
.on_conflict(
OnConflict::column(video::Column::Id)
.update_column(video::Column::DownloadStatus)
.to_owned(),
)
.exec(connection)
.await?;
Ok(())
}
/// 更新视频页 model 的下载状态
pub async fn update_pages_model(pages: Vec<page::ActiveModel>, connection: &DatabaseConnection) -> Result<()> {
let query = page::Entity::insert_many(pages).on_conflict(
OnConflict::column(page::Column::Id)
.update_columns([page::Column::DownloadStatus, page::Column::Path])
.to_owned(),
);
query.exec(connection).await?;
Ok(())
}

263
crates/bili_sync/src/core/utils.rs → crates/bili_sync/src/utils/nfo.rs
View File

@@ -1,42 +1,11 @@
use std::collections::HashSet;
use std::path::Path;
use anyhow::Result;
use bili_sync_entity::*;
use bili_sync_migration::OnConflict;
use filenamify::filenamify;
use handlebars::handlebars_helper;
use once_cell::sync::Lazy;
use quick_xml::events::{BytesCData, BytesText};
use quick_xml::writer::Writer;
use quick_xml::Error;
use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::QuerySelect;
use serde_json::json;
use tokio::io::AsyncWriteExt;
use tracing_subscriber::util::SubscriberInitExt;
use crate::bilibili::{FavoriteListInfo, PageInfo, VideoInfo};
use crate::config::{NFOTimeType, CONFIG};
use crate::core::status::Status;
pub static TEMPLATE: Lazy<handlebars::Handlebars> = Lazy::new(|| {
let mut handlebars = handlebars::Handlebars::new();
handlebars_helper!(truncate: |s: String, len: usize| {
if s.chars().count() > len {
s.chars().take(len).collect::<String>()
} else {
s.to_string()
}
});
handlebars.register_helper("truncate", Box::new(truncate));
handlebars
.register_template_string("video", &CONFIG.video_name)
.unwrap();
handlebars.register_template_string("page", &CONFIG.page_name).unwrap();
handlebars
});
use crate::config::NFOTimeType;
#[allow(clippy::upper_case_acronyms)]
pub enum NFOMode {
@@ -53,225 +22,6 @@ pub enum ModelWrapper<'a> {
pub struct NFOSerializer<'a>(pub ModelWrapper<'a>, pub NFOMode);
/// 根据获得的收藏夹信息，插入或更新数据库中的收藏夹，并返回收藏夹对象
pub async fn handle_favorite_info(
info: &FavoriteListInfo,
path: &Path,
connection: &DatabaseConnection,
) -> Result<favorite::Model> {
favorite::Entity::insert(favorite::ActiveModel {
f_id: Set(info.id),
name: Set(info.title.clone()),
path: Set(path.to_string_lossy().to_string()),
..Default::default()
})
.on_conflict(
OnConflict::column(favorite::Column::FId)
.update_columns([favorite::Column::Name, favorite::Column::Path])
.to_owned(),
)
.exec(connection)
.await?;
Ok(favorite::Entity::find()
.filter(favorite::Column::FId.eq(info.id))
.one(connection)
.await?
.unwrap())
}
/// 获取数据库中存在的与该视频 favorite_id 和 bvid 重合的视频中的 bvid 和 favtime
/// 如果 bvid 和 favtime 均相同，说明到达了上次处理到的位置
pub async fn exist_labels(
videos_info: &[VideoInfo],
favorite_model: &favorite::Model,
connection: &DatabaseConnection,
) -> Result<HashSet<(String, DateTime)>> {
let bvids = videos_info.iter().map(|v| v.bvid.clone()).collect::<Vec<String>>();
let exist_labels = video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(favorite_model.id)
.and(video::Column::Bvid.is_in(bvids)),
)
.select_only()
.columns([video::Column::Bvid, video::Column::Favtime])
.into_tuple()
.all(connection)
.await?
.into_iter()
.collect::<HashSet<(String, DateTime)>>();
Ok(exist_labels)
}
/// 尝试创建 Video Model，如果发生冲突则忽略
pub async fn create_videos(
videos_info: &[VideoInfo],
favorite: &favorite::Model,
connection: &DatabaseConnection,
) -> Result<()> {
let video_models = videos_info
.iter()
.map(move |v| video::ActiveModel {
favorite_id: Set(favorite.id),
bvid: Set(v.bvid.clone()),
name: Set(v.title.clone()),
path: Set(Path::new(&favorite.path)
.join(filenamify(
TEMPLATE
.render(
"video",
&json!({
"bvid": &v.bvid,
"title": &v.title,
"upper_name": &v.upper.name,
"upper_mid": &v.upper.mid,
}),
)
.unwrap_or_else(|_| v.bvid.clone()),
))
.to_str()
.unwrap()
.to_owned()),
category: Set(v.vtype),
intro: Set(v.intro.clone()),
cover: Set(v.cover.clone()),
ctime: Set(v.ctime.naive_utc()),
pubtime: Set(v.pubtime.naive_utc()),
favtime: Set(v.fav_time.naive_utc()),
download_status: Set(0),
valid: Set(v.attr == 0),
tags: Set(None),
single_page: Set(None),
upper_id: Set(v.upper.mid),
upper_name: Set(v.upper.name.clone()),
upper_face: Set(v.upper.face.clone()),
..Default::default()
})
.collect::<Vec<video::ActiveModel>>();
video::Entity::insert_many(video_models)
.on_conflict(
OnConflict::columns([video::Column::FavoriteId, video::Column::Bvid])
.do_nothing()
.to_owned(),
)
.do_nothing()
.exec(connection)
.await?;
Ok(())
}
pub async fn total_video_count(favorite_model: &favorite::Model, connection: &DatabaseConnection) -> Result<u64> {
Ok(video::Entity::find()
.filter(video::Column::FavoriteId.eq(favorite_model.id))
.count(connection)
.await?)
}
/// 筛选所有未
pub async fn filter_unfilled_videos(
favorite_model: &favorite::Model,
connection: &DatabaseConnection,
) -> Result<Vec<video::Model>> {
Ok(video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(favorite_model.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null()),
)
.all(connection)
.await?)
}
/// 创建视频的所有分 P
pub async fn create_video_pages(
pages_info: &[PageInfo],
video_model: &video::Model,
connection: &impl ConnectionTrait,
) -> Result<()> {
let page_models = pages_info
.iter()
.map(move |p| {
let (width, height) = match &p.dimension {
Some(d) => {
if d.rotate == 0 {
(Some(d.width), Some(d.height))
} else {
(Some(d.height), Some(d.width))
}
}
None => (None, None),
};
page::ActiveModel {
video_id: Set(video_model.id),
cid: Set(p.cid),
pid: Set(p.page),
name: Set(p.name.clone()),
width: Set(width),
height: Set(height),
duration: Set(p.duration),
image: Set(p.first_frame.clone()),
download_status: Set(0),
..Default::default()
}
})
.collect::<Vec<page::ActiveModel>>();
page::Entity::insert_many(page_models)
.on_conflict(
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
.do_nothing()
.to_owned(),
)
.do_nothing()
.exec(connection)
.await?;
Ok(())
}
/// 获取所有未处理的视频和页
pub async fn unhandled_videos_pages(
favorite_model: &favorite::Model,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
Ok(video::Entity::find()
.filter(
video::Column::FavoriteId
.eq(favorite_model.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(Status::handled()))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null()),
)
.find_with_related(page::Entity)
.all(connection)
.await?)
}
/// 更新视频 model 的下载状态
pub async fn update_videos_model(videos: Vec<video::ActiveModel>, connection: &DatabaseConnection) -> Result<()> {
video::Entity::insert_many(videos)
.on_conflict(
OnConflict::column(video::Column::Id)
.update_column(video::Column::DownloadStatus)
.to_owned(),
)
.exec(connection)
.await?;
Ok(())
}
/// 更新视频页 model 的下载状态
pub async fn update_pages_model(pages: Vec<page::ActiveModel>, connection: &DatabaseConnection) -> Result<()> {
let query = page::Entity::insert_many(pages).on_conflict(
OnConflict::column(page::Column::Id)
.update_columns([page::Column::DownloadStatus, page::Column::Path])
.to_owned(),
);
query.exec(connection).await?;
Ok(())
}
/// serde xml 似乎不太好用，先这么裸着写
/// （真是又臭又长啊
impl<'a> NFOSerializer<'a> {
@@ -483,17 +233,6 @@ impl<'a> NFOSerializer<'a> {
}
}
pub fn init_logger(log_level: &str) {
tracing_subscriber::fmt::Subscriber::builder()
.with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level))
.with_timer(tracing_subscriber::fmt::time::ChronoLocal::new(
"%Y-%m-%d %H:%M:%S%.3f".to_owned(),
))
.finish()
.try_init()
.expect("初始化日志失败");
}
#[cfg(test)]
mod tests {
use super::*;

0
crates/bili_sync/src/core/status.rs → crates/bili_sync/src/utils/status.rs
View File

157
crates/bili_sync/src/core/command.rs → crates/bili_sync/src/workflow.rs
View File

@@ -1,158 +1,96 @@
#![allow(dead_code, unused_variables)]
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use anyhow::{bail, Result};
use bili_sync_entity::{favorite, page, video};
use bili_sync_entity::{page, video};
use filenamify::filenamify;
use futures::stream::{FuturesOrdered, FuturesUnordered};
use futures::{pin_mut, Future, StreamExt};
use futures::{Future, Stream, StreamExt};
use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::TransactionTrait;
use serde_json::json;
use tokio::fs;
use tokio::sync::{Mutex, Semaphore};
use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, FavoriteList, PageInfo, Video};
use crate::config::{ARGS, CONFIG};
use crate::core::status::{PageStatus, VideoStatus};
use crate::core::utils::{
create_video_pages, create_videos, exist_labels, filter_unfilled_videos, handle_favorite_info, total_video_count,
unhandled_videos_pages, update_pages_model, update_videos_model, ModelWrapper, NFOMode, NFOSerializer, TEMPLATE,
};
use crate::adapter::{video_list_from, Args, VideoListModel};
use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Video, VideoInfo};
use crate::config::{ARGS, CONFIG, TEMPLATE};
use crate::downloader::Downloader;
use crate::error::{DownloadAbortError, ProcessPageError};
use crate::utils::model::{create_videos, update_pages_model, update_videos_model};
use crate::utils::nfo::{ModelWrapper, NFOMode, NFOSerializer};
use crate::utils::status::{PageStatus, VideoStatus};
/// 处理某个收藏夹，首先刷新收藏夹信息，然后下载收藏夹中未下载成功的视频
pub async fn process_favorite_list(
pub async fn process_video_list(
args: Args<'_>,
bili_client: &BiliClient,
fid: &str,
path: &Path,
connection: &DatabaseConnection,
) -> Result<()> {
let favorite_model = refresh_favorite_list(bili_client, fid, path, connection).await?;
let favorite_model = fetch_video_details(bili_client, favorite_model, connection).await?;
let (video_list_model, video_streams) = video_list_from(args, path, bili_client, connection).await?;
let video_list_model = refresh_video_list(bili_client, video_list_model, video_streams, connection).await?;
let video_list_model = fetch_video_details(bili_client, video_list_model, connection).await?;
if ARGS.scan_only {
warn!("已开启仅扫描模式，跳过视频下载...");
return Ok(());
}
download_unprocessed_videos(bili_client, favorite_model, connection).await
download_unprocessed_videos(bili_client, video_list_model, connection).await
}
/// 获取收藏夹 Model，从收藏夹列表中获取所有新添加的视频，将其写入数据库
pub async fn refresh_favorite_list(
bili_client: &BiliClient,
fid: &str,
path: &Path,
/// 请求接口，获取视频列表中所有新添加的视频信息，将其写入数据库
pub async fn refresh_video_list<'a>(
bili_client: &'a BiliClient,
video_list_model: Box<dyn VideoListModel>,
video_streams: Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>,
connection: &DatabaseConnection,
) -> Result<favorite::Model> {
let bili_favorite_list = FavoriteList::new(bili_client, fid.to_owned());
let favorite_list_info = bili_favorite_list.get_info().await?;
let favorite_model = handle_favorite_info(&favorite_list_info, path, connection).await?;
info!("开始扫描收藏夹: {} - {}...", favorite_model.f_id, favorite_model.name);
// 每十个视频一组，避免太多的数据库操作
let video_stream = bili_favorite_list.into_video_stream().chunks(10);
pin_mut!(video_stream);
) -> Result<Box<dyn VideoListModel>> {
video_list_model.log_refresh_video_start();
let mut video_streams = video_streams.chunks(10);
let mut got_count = 0;
let total_count = total_video_count(&favorite_model, connection).await?;
while let Some(videos_info) = video_stream.next().await {
let mut new_count = video_list_model.video_count(connection).await?;
while let Some(videos_info) = video_streams.next().await {
got_count += videos_info.len();
let exist_labels = exist_labels(&videos_info, &favorite_model, connection).await?;
let exist_labels = video_list_model.exist_labels(&videos_info, connection).await?;
// 如果发现有视频的收藏时间和 bvid 和数据库中重合，说明到达了上次处理到的地方，可以直接退出
let should_break = videos_info
.iter()
.any(|v| exist_labels.contains(&(v.bvid.clone(), v.fav_time.naive_utc())));
let should_break = videos_info.iter().any(|v| exist_labels.contains(&v.video_key()));
// 将视频信息写入数据库
create_videos(&videos_info, &favorite_model, connection).await?;
create_videos(&videos_info, video_list_model.as_ref(), connection).await?;
if should_break {
info!("到达上一次处理的位置，提前中止");
break;
}
}
let total_count = total_video_count(&favorite_model, connection).await? - total_count;
info!(
"扫描收藏夹: {} - {} 完成, 获取了 {} 条视频, 其中有 {} 条新视频",
favorite_model.f_id, favorite_model.name, got_count, total_count
);
Ok(favorite_model)
new_count = video_list_model.video_count(connection).await? - new_count;
video_list_model.log_refresh_video_end(got_count, new_count);
Ok(video_list_model)
}
/// 筛选出所有没有获取到分页信息和 tag 的视频，请求分页信息和 tag 并写入数据库
/// 筛选出所有未获取到全部信息的视频，尝试补充其详细信息
pub async fn fetch_video_details(
bili_client: &BiliClient,
favorite_model: favorite::Model,
video_list_model: Box<dyn VideoListModel>,
connection: &DatabaseConnection,
) -> Result<favorite::Model> {
info!(
"开始获取收藏夹 {} - {} 的视频与分页信息...",
favorite_model.f_id, favorite_model.name
);
let videos_model = filter_unfilled_videos(&favorite_model, connection).await?;
for video_model in videos_model {
let bili_video = Video::new(bili_client, video_model.bvid.clone());
let tags = match bili_video.get_tags().await {
Ok(tags) => tags,
Err(e) => {
error!(
"获取视频 {} - {} 的标签失败，错误为：{}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(code, _)) = e.downcast_ref::<BiliError>() {
if *code == -404 {
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
}
continue;
}
};
let pages_info = match bili_video.get_pages().await {
Ok(pages) => pages,
Err(e) => {
error!(
"获取视频 {} - {} 的分页信息失败，错误为：{}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(code, _)) = e.downcast_ref::<BiliError>() {
if *code == -404 {
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
}
continue;
}
};
let txn = connection.begin().await?;
// 将分页信息写入数据库
create_video_pages(&pages_info, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.single_page = Set(Some(pages_info.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
info!(
"获取收藏夹 {} - {} 的视频与分页信息完成",
favorite_model.f_id, favorite_model.name
);
Ok(favorite_model)
) -> Result<Box<dyn VideoListModel>> {
video_list_model.log_fetch_video_start();
let videos_model = video_list_model.unfilled_videos(connection).await?;
video_list_model
.fetch_videos_detail(bili_client, videos_model, connection)
.await?;
video_list_model.log_fetch_video_end();
Ok(video_list_model)
}
/// 下载所有未处理成功的视频
pub async fn download_unprocessed_videos(
bili_client: &BiliClient,
favorite_model: favorite::Model,
video_list_model: Box<dyn VideoListModel>,
connection: &DatabaseConnection,
) -> Result<()> {
info!(
"开始下载收藏夹: {} - {} 中所有未处理过的视频...",
favorite_model.f_id, favorite_model.name
);
let unhandled_videos_pages = unhandled_videos_pages(&favorite_model, connection).await?;
video_list_model.log_download_video_start();
let unhandled_videos_pages = video_list_model.unhandled_video_pages(connection).await?;
// 对于视频，允许三个同时下载（视频内还有分页、不同分页还有多种下载任务）
let semaphore = Semaphore::new(3);
let downloader = Downloader::new(bili_client.client.clone());
@@ -197,10 +135,7 @@ pub async fn download_unprocessed_videos(
if !models.is_empty() {
update_videos_model(models, connection).await?;
}
info!(
"下载收藏夹: {} - {} 中未处理过的视频完成",
favorite_model.f_id, favorite_model.name
);
video_list_model.log_download_video_end();
Ok(())
}