diff --git a/src/bilibili/favorite_list.rs b/src/bilibili/favorite_list.rs index 11d13d2..84f49fb 100644 --- a/src/bilibili/favorite_list.rs +++ b/src/bilibili/favorite_list.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use async_stream::stream; +use chrono::serde::ts_seconds; +use chrono::{DateTime, Utc}; use futures_core::stream::Stream; use serde_json::Value; @@ -11,10 +13,9 @@ pub struct FavoriteList { fid: String, } -#[allow(dead_code)] #[derive(Debug, serde::Deserialize)] pub struct FavoriteListInfo { - pub id: u64, + pub id: i32, pub title: String, } @@ -22,19 +23,23 @@ pub struct FavoriteListInfo { pub struct VideoInfo { pub title: String, #[serde(rename = "type")] - pub vtype: u64, + pub vtype: i32, pub bvid: String, pub intro: String, pub cover: String, pub upper: Upper, - pub ctime: u64, - pub fav_time: u64, - pub pubtime: u64, + #[serde(with = "ts_seconds")] + pub ctime: DateTime, + #[serde(with = "ts_seconds")] + pub fav_time: DateTime, + #[serde(with = "ts_seconds")] + pub pubtime: DateTime, + pub attr: i32, } #[derive(Debug, serde::Deserialize)] pub struct Upper { - pub mid: u64, + pub mid: i32, pub name: String, } impl FavoriteList { diff --git a/src/bilibili/mod.rs b/src/bilibili/mod.rs index d8717ea..798947b 100644 --- a/src/bilibili/mod.rs +++ b/src/bilibili/mod.rs @@ -10,4 +10,4 @@ pub use analyzer::{ pub use client::{BiliClient, Client}; pub use credential::Credential; pub use favorite_list::{FavoriteList, FavoriteListInfo, VideoInfo}; -pub use video::Video; +pub use video::{PageInfo, Video}; diff --git a/src/bilibili/video.rs b/src/bilibili/video.rs index bbb5676..6b1a565 100644 --- a/src/bilibili/video.rs +++ b/src/bilibili/video.rs @@ -27,15 +27,14 @@ pub struct Tag { pub tag_name: String, } -#[allow(dead_code)] #[derive(Debug, serde::Deserialize)] -pub struct Page { - cid: u64, - page: u32, +pub struct PageInfo { + pub cid: i32, + pub page: i32, #[serde(rename = "part")] - name: String, + pub name: String, #[serde(default = "String::new")] - first_frame: String, // 可能不存在，默认填充为空 + pub first_frame: String, // 可能不存在，默认填充为空 } impl Video { @@ -44,7 +43,7 @@ impl Video { Self { client, aid, bvid } } - pub async fn get_pages(&self) -> Result> { + pub async fn get_pages(&self) -> Result> { let mut res = self .client .request(Method::GET, "https://api.bilibili.com/x/player/pagelist") @@ -71,7 +70,7 @@ impl Video { Ok(serde_json::from_value(res["data"].take())?) } - pub async fn get_page_analyzer(&self, page: &Page) -> Result { + pub async fn get_page_analyzer(&self, page: &PageInfo) -> Result { let mut res = self .client .request(Method::GET, "https://api.bilibili.com/x/player/wbi/playurl") diff --git a/src/core/command.rs b/src/core/command.rs index e13d2c9..1b0b3a1 100644 --- a/src/core/command.rs +++ b/src/core/command.rs @@ -1,15 +1,12 @@ -use std::collections::HashSet; -use std::path::Path; use std::sync::Arc; -use entity::*; use futures_util::{pin_mut, StreamExt}; -use migration::OnConflict; use sea_orm::entity::prelude::*; -use sea_orm::ActiveValue::Set; -use sea_orm::QuerySelect; -use crate::bilibili::{BiliClient, FavoriteList, FavoriteListInfo, VideoInfo}; +use crate::bilibili::{BiliClient, FavoriteList, Video}; +use crate::core::utils::{ + create_video_pages, create_videos, exists_bvids_favtime, filter_videos, handle_favorite_info, +}; use crate::Result; pub async fn process_favorite( @@ -29,100 +26,23 @@ pub async fn process_favorite( while let Some(videos_info) = video_stream.next().await { let exist_bvids_pubtimes = exists_bvids_favtime(&videos_info, fid, connection.as_ref()).await?; - let video_info_to_create = videos_info + let should_break = videos_info .iter() - .filter(|v| !exist_bvids_pubtimes.contains(&(v.bvid.clone(), v.fav_time.to_string()))) - .collect::>(); - let len = video_info_to_create.len(); - if !video_info_to_create.is_empty() { - create_videos(video_info_to_create, &favorite_obj, connection.as_ref()).await?; + // 出现 bvid 和 fav_time 都相同的记录，说明已经到达了上次处理到的位置 + .any(|v| exist_bvids_pubtimes.contains(&(v.bvid.clone(), v.fav_time.naive_utc()))); + create_videos(&videos_info, &favorite_obj, connection.as_ref()).await?; + let all_unprocessed_videos = + filter_videos(&videos_info, &favorite_obj, true, true, connection.as_ref()).await?; + if !all_unprocessed_videos.is_empty() { + for video in all_unprocessed_videos { + let bili_video = Video::new(bili_client.clone(), video.bvid.clone()); + let pages = bili_video.get_pages().await?; + create_video_pages(&pages, &video, connection.as_ref()).await?; + } } - if videos_info.len() != len { + if should_break { break; } } Ok(()) } - -// 根据获得的收藏夹信息，插入或更新数据库中的收藏夹，并返回收藏夹对象 -async fn handle_favorite_info( - info: &FavoriteListInfo, - connection: &DatabaseConnection, -) -> Result { - Ok(favorite::Entity::insert(favorite::ActiveModel { - f_id: Set(info.id as i32), - name: Set(info.title.to_string()), - path: Set("/home/amtoaer/Documents/code/rust/bili-sync/video".to_string()), - enabled: Set(true), - ..Default::default() - }) - .on_conflict( - OnConflict::column(favorite::Column::FId) - .update_column(favorite::Column::Name) - .update_column(favorite::Column::Path) - .update_column(favorite::Column::Enabled) - .to_owned(), - ) - .exec_with_returning(connection) - .await?) -} - -// 获取数据库中存在的与该视频 favorite_id 和 bvid 重合的视频 -async fn exists_bvids_favtime( - videos_info: &[VideoInfo], - fid: i32, - connection: &DatabaseConnection, -) -> Result> { - let bvids = videos_info - .iter() - .map(|v| v.bvid.clone()) - .collect::>(); - let exist_bvid_favtime = video::Entity::find() - .filter( - video::Column::FavoriteId - .eq(fid) - .and(video::Column::Bvid.is_in(bvids)), - ) - .select_only() - .columns([video::Column::Bvid, video::Column::Favtime]) - .all(connection) - .await? - .into_iter() - .map(|v| (v.bvid, v.favtime)) - .collect::>(); - Ok(exist_bvid_favtime) -} - -async fn create_videos( - videos_info: Vec<&VideoInfo>, - favorite_obj: &favorite::Model, - connection: &DatabaseConnection, -) -> Result { - let video_models = videos_info - .iter() - .map(move |v| video::ActiveModel { - favorite_id: Set(favorite_obj.id), - bvid: Set(v.bvid.clone()), - path: Set(Path::new(favorite_obj.path.as_str()) - .join(&v.title) - .to_str() - .unwrap() - .to_string()), - name: Set(v.title.clone()), - category: Set(v.vtype.to_string()), - intro: Set(v.intro.clone()), - cover: Set(v.cover.clone()), - ctime: Set(v.ctime.to_string()), - pubtime: Set(v.pubtime.to_string()), - favtime: Set(v.fav_time.to_string()), - downloaded: Set(false), - valid: Set(true), - tags: Set("[]".to_string()), - single_page: Set(false), - ..Default::default() - }) - .collect::>(); - Ok(video::Entity::insert_many(video_models) - .exec_without_returning(connection) - .await?) -} diff --git a/src/core/mod.rs b/src/core/mod.rs index 9fe7961..3b143e0 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1 +1,2 @@ pub mod command; +pub mod utils; diff --git a/src/core/utils.rs b/src/core/utils.rs new file mode 100644 index 0000000..88cbdc5 --- /dev/null +++ b/src/core/utils.rs @@ -0,0 +1,172 @@ +use std::collections::HashSet; +use std::path::Path; + +use entity::*; +use migration::OnConflict; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::QuerySelect; + +use crate::bilibili::{FavoriteListInfo, PageInfo, VideoInfo}; +use crate::Result; + +// 根据获得的收藏夹信息，插入或更新数据库中的收藏夹，并返回收藏夹对象 +pub async fn handle_favorite_info( + info: &FavoriteListInfo, + connection: &DatabaseConnection, +) -> Result { + favorite::Entity::insert(favorite::ActiveModel { + f_id: Set(info.id), + name: Set(info.title.to_string()), + path: Set("/home/amtoaer/Documents/code/rust/bili-sync/video".to_string()), + enabled: Set(true), + ..Default::default() + }) + .on_conflict( + OnConflict::column(favorite::Column::FId) + .update_columns([ + favorite::Column::Name, + favorite::Column::Path, + favorite::Column::Enabled, + ]) + .to_owned(), + ) + .exec(connection) + .await?; + Ok(favorite::Entity::find() + .filter(favorite::Column::FId.eq(info.id)) + .one(connection) + .await? + .unwrap()) +} + +// 获取数据库中存在的与该视频 favorite_id 和 bvid 重合的视频 +pub async fn exists_bvids_favtime( + videos_info: &[VideoInfo], + fid: i32, + connection: &DatabaseConnection, +) -> Result> { + let bvids = videos_info + .iter() + .map(|v| v.bvid.clone()) + .collect::>(); + let exist_bvid_favtime = video::Entity::find() + .filter( + video::Column::FavoriteId + .eq(fid) + .and(video::Column::Bvid.is_in(bvids)), + ) + .select_only() + .columns([video::Column::Bvid, video::Column::Favtime]) + .all(connection) + .await? + .into_iter() + .map(|v| (v.bvid, v.favtime)) + .collect::>(); + Ok(exist_bvid_favtime) +} + +// 尝试创建 Video Model，如果发生冲突则忽略 +pub async fn create_videos( + videos_info: &[VideoInfo], + favorite_obj: &favorite::Model, + connection: &DatabaseConnection, +) -> Result<()> { + let video_models = videos_info + .iter() + .map(move |v| video::ActiveModel { + favorite_id: Set(favorite_obj.id), + bvid: Set(v.bvid.clone()), + path: Set(Path::new(favorite_obj.path.as_str()) + .join(&v.title) + .to_str() + .unwrap() + .to_string()), + name: Set(v.title.clone()), + category: Set(v.vtype), + intro: Set(v.intro.clone()), + cover: Set(v.cover.clone()), + ctime: Set(v.ctime.naive_utc()), + pubtime: Set(v.pubtime.naive_utc()), + favtime: Set(v.fav_time.naive_utc()), + handled: Set(false), + valid: Set(v.attr == 0), + tags: Set(None), + single_page: Set(None), + upper_id: Set(v.upper.mid), + upper_name: Set(v.upper.name.clone()), + ..Default::default() + }) + .collect::>(); + video::Entity::insert_many(video_models) + .on_conflict( + OnConflict::columns([video::Column::FavoriteId, video::Column::Bvid]) + .do_nothing() + .to_owned(), + ) + .exec(connection) + .await?; + Ok(()) +} + +// 筛选所有符合条件的视频 +pub async fn filter_videos( + videos_info: &[VideoInfo], + favorite_obj: &favorite::Model, + only_unhandled: bool, + only_no_page: bool, + connection: &DatabaseConnection, +) -> Result> { + let bvids = videos_info + .iter() + .map(|v| v.bvid.clone()) + .collect::>(); + let mut condition = video::Column::FavoriteId + .eq(favorite_obj.id) + .and(video::Column::Bvid.is_in(bvids)) + .and(video::Column::Valid.eq(true)); + if only_unhandled { + condition = condition.and(video::Column::Handled.eq(false)); + } + if only_no_page { + condition = condition.and(video::Column::SinglePage.is_null()); + } + Ok(video::Entity::find() + .filter(condition) + .all(connection) + .await?) +} + +pub async fn create_video_pages( + pages: &[PageInfo], + video_obj: &video::Model, + connection: &DatabaseConnection, +) -> Result<()> { + let page_models = pages + .iter() + .map(move |p| page::ActiveModel { + video_id: Set(video_obj.id), + cid: Set(p.cid), + pid: Set(p.page), + name: Set(p.name.clone()), + path: Set(Path::new(video_obj.path.as_str()) + .join(&p.name) + .to_str() + .unwrap() + .to_string()), + image: Set(p.first_frame.clone()), + valid: Set(video_obj.valid), + download_status: Set(0), + ..Default::default() + }) + .collect::>(); + page::Entity::insert_many(page_models) + .on_conflict( + OnConflict::columns([page::Column::VideoId, page::Column::Pid]) + .do_nothing() + .to_owned(), + ) + .exec(connection) + .await?; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 7dc295b..c5c36b5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,8 +10,11 @@ async fn main() -> ! { let bili_client = Arc::new(BiliClient::new(None)); loop { for fid in [52642258] { - let _ = process_favorite(bili_client.clone(), fid, connection.clone()).await; + let res = process_favorite(bili_client.clone(), fid, connection.clone()).await; + if let Err(e) = res { + eprintln!("Error: {:?}", e); + } } - tokio::time::sleep(std::time::Duration::from_secs(60)).await; + tokio::time::sleep(std::time::Duration::from_secs(600)).await; } }