From 8f37fdf841ed46e95b00adbeb488c5dd62623731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=B4=80=E1=B4=8D=E1=B4=9B=E1=B4=8F=E1=B4=80=E1=B4=87?= =?UTF-8?q?=CA=80?= Date: Wed, 24 Jul 2024 00:36:19 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E6=8A=8A=E5=BE=AA=E7=8E=AF?= =?UTF-8?q?=E6=8B=86=E5=88=86=E5=88=B0=E5=A4=96=E5=B1=82=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E5=8F=96=E5=85=AC=E5=85=B1=E4=BB=A3=E7=A0=81=20(#151)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/bili_sync/src/adapter/collection.rs | 64 +++++++++------------ crates/bili_sync/src/adapter/favorite.rs | 58 ++++++++----------- crates/bili_sync/src/adapter/mod.rs | 25 +++++++- crates/bili_sync/src/adapter/watch_later.rs | 58 ++++++++----------- crates/bili_sync/src/workflow.rs | 10 +++- 5 files changed, 101 insertions(+), 114 deletions(-) diff --git a/crates/bili_sync/src/adapter/collection.rs b/crates/bili_sync/src/adapter/collection.rs index 6c91e1d..c8431f0 100644 --- a/crates/bili_sync/src/adapter/collection.rs +++ b/crates/bili_sync/src/adapter/collection.rs @@ -11,12 +11,12 @@ use sea_orm::entity::prelude::*; use sea_orm::ActiveValue::Set; use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; -use crate::adapter::VideoListModel; -use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo}; -use crate::config::{CONFIG, TEMPLATE}; +use crate::adapter::{error_fetch_video_detail, VideoListModel}; +use crate::bilibili::{self, BiliClient, Collection, CollectionItem, CollectionType, VideoInfo}; +use crate::config::TEMPLATE; +use crate::utils::id_time_key; use crate::utils::model::create_video_pages; use crate::utils::status::Status; -use crate::utils::{delay, id_time_key}; pub async fn collection_from<'a>( collection_item: &'a CollectionItem, @@ -144,42 +144,30 @@ impl VideoListModel for collection::Model { async fn fetch_videos_detail( &self, - bili_clent: &BiliClient, - videos_model: Vec, + video: bilibili::Video<'_>, + video_model: video::Model, connection: &DatabaseConnection, ) -> Result<()> { - for video_model in videos_model { - let video = Video::new(bili_clent, video_model.bvid.clone()); - let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await; - match info { - Ok((tags, view_info)) => { - let VideoInfo::View { pages, .. } = &view_info else { - unreachable!("view_info must be VideoInfo::View") - }; - let txn = connection.begin().await?; - // 将分页信息写入数据库 - create_video_pages(pages, &video_model, &txn).await?; - // 将页标记和 tag 写入数据库 - let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model)); - video_active_model.single_page = Set(Some(pages.len() == 1)); - video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); - video_active_model.save(&txn).await?; - txn.commit().await?; - } - Err(e) => { - error!( - "获取视频 {} - {} 的详细信息失败，错误为：{}", - &video_model.bvid, &video_model.name, e - ); - if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.valid = Set(false); - video_active_model.save(connection).await?; - } - } - }; - delay(CONFIG.delay.fetch_video_detail.as_ref()).await; - } + let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await; + match info { + Ok((tags, view_info)) => { + let VideoInfo::View { pages, .. } = &view_info else { + unreachable!("view_info must be VideoInfo::View") + }; + let txn = connection.begin().await?; + // 将分页信息写入数据库 + create_video_pages(pages, &video_model, &txn).await?; + // 将页标记和 tag 写入数据库 + let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model)); + video_active_model.single_page = Set(Some(pages.len() == 1)); + video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); + video_active_model.save(&txn).await?; + txn.commit().await?; + } + Err(e) => { + error_fetch_video_detail(e, video_model, connection).await?; + } + }; Ok(()) } diff --git a/crates/bili_sync/src/adapter/favorite.rs b/crates/bili_sync/src/adapter/favorite.rs index f4dffe3..fe73224 100644 --- a/crates/bili_sync/src/adapter/favorite.rs +++ b/crates/bili_sync/src/adapter/favorite.rs @@ -11,12 +11,12 @@ use sea_orm::entity::prelude::*; use sea_orm::ActiveValue::Set; use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; -use crate::adapter::VideoListModel; -use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo}; -use crate::config::{CONFIG, TEMPLATE}; +use crate::adapter::{error_fetch_video_detail, VideoListModel}; +use crate::bilibili::{self, BiliClient, FavoriteList, VideoInfo}; +use crate::config::TEMPLATE; +use crate::utils::id_time_key; use crate::utils::model::create_video_pages; use crate::utils::status::Status; -use crate::utils::{delay, id_time_key}; pub async fn favorite_from<'a>( fid: &str, @@ -134,39 +134,27 @@ impl VideoListModel for favorite::Model { async fn fetch_videos_detail( &self, - bili_clent: &BiliClient, - videos_model: Vec, + video: bilibili::Video<'_>, + video_model: video::Model, connection: &DatabaseConnection, ) -> Result<()> { - for video_model in videos_model { - let video = Video::new(bili_clent, video_model.bvid.clone()); - let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await; - match info { - Ok((tags, pages_info)) => { - let txn = connection.begin().await?; - // 将分页信息写入数据库 - create_video_pages(&pages_info, &video_model, &txn).await?; - // 将页标记和 tag 写入数据库 - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.single_page = Set(Some(pages_info.len() == 1)); - video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); - video_active_model.save(&txn).await?; - txn.commit().await?; - } - Err(e) => { - error!( - "获取视频 {} - {} 的详细信息失败，错误为：{}", - &video_model.bvid, &video_model.name, e - ); - if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.valid = Set(false); - video_active_model.save(connection).await?; - } - } - }; - delay(CONFIG.delay.fetch_video_detail.as_ref()).await; - } + let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await; + match info { + Ok((tags, pages_info)) => { + let txn = connection.begin().await?; + // 将分页信息写入数据库 + create_video_pages(&pages_info, &video_model, &txn).await?; + // 将页标记和 tag 写入数据库 + let mut video_active_model: video::ActiveModel = video_model.into(); + video_active_model.single_page = Set(Some(pages_info.len() == 1)); + video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); + video_active_model.save(&txn).await?; + txn.commit().await?; + } + Err(e) => { + error_fetch_video_detail(e, video_model, connection).await?; + } + }; Ok(()) } diff --git a/crates/bili_sync/src/adapter/mod.rs b/crates/bili_sync/src/adapter/mod.rs index 92e796f..dce2959 100644 --- a/crates/bili_sync/src/adapter/mod.rs +++ b/crates/bili_sync/src/adapter/mod.rs @@ -11,10 +11,12 @@ use async_trait::async_trait; pub use collection::collection_from; pub use favorite::favorite_from; use futures::Stream; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue::Set; use sea_orm::DatabaseConnection; use watch_later::watch_later_from; -use crate::bilibili::{BiliClient, CollectionItem, VideoInfo}; +use crate::bilibili::{self, BiliClient, BiliError, CollectionItem, VideoInfo}; pub enum Args<'a> { Favorite { fid: &'a str }, @@ -35,6 +37,23 @@ pub async fn video_list_from<'a>( } } +async fn error_fetch_video_detail( + e: anyhow::Error, + video_model: bili_sync_entity::video::Model, + connection: &DatabaseConnection, +) -> Result<()> { + error!( + "获取视频 {} - {} 的详细信息失败，错误为：{}", + &video_model.bvid, &video_model.name, e + ); + if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { + let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into(); + video_active_model.valid = Set(false); + video_active_model.save(connection).await?; + } + Ok(()) +} + #[async_trait] pub trait VideoListModel { /* 逻辑相关 */ @@ -65,8 +84,8 @@ pub trait VideoListModel { /// 获取视频 model 中缺失的信息 async fn fetch_videos_detail( &self, - bili_client: &BiliClient, - videos_model: Vec, + video: bilibili::Video<'_>, + video_model: bili_sync_entity::video::Model, connection: &DatabaseConnection, ) -> Result<()>; diff --git a/crates/bili_sync/src/adapter/watch_later.rs b/crates/bili_sync/src/adapter/watch_later.rs index 15c90d2..53a1354 100644 --- a/crates/bili_sync/src/adapter/watch_later.rs +++ b/crates/bili_sync/src/adapter/watch_later.rs @@ -11,12 +11,12 @@ use sea_orm::entity::prelude::*; use sea_orm::ActiveValue::Set; use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; -use crate::adapter::VideoListModel; -use crate::bilibili::{BiliClient, BiliError, Video, VideoInfo, WatchLater}; -use crate::config::{CONFIG, TEMPLATE}; +use crate::adapter::{error_fetch_video_detail, VideoListModel}; +use crate::bilibili::{self, BiliClient, VideoInfo, WatchLater}; +use crate::config::TEMPLATE; +use crate::utils::id_time_key; use crate::utils::model::create_video_pages; use crate::utils::status::Status; -use crate::utils::{delay, id_time_key}; pub async fn watch_later_from<'a>( path: &Path, @@ -130,39 +130,27 @@ impl VideoListModel for watch_later::Model { async fn fetch_videos_detail( &self, - bili_clent: &BiliClient, - videos_model: Vec, + video: bilibili::Video<'_>, + video_model: video::Model, connection: &DatabaseConnection, ) -> Result<()> { - for video_model in videos_model { - let video = Video::new(bili_clent, video_model.bvid.clone()); - let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await; - match info { - Ok((tags, pages_info)) => { - let txn = connection.begin().await?; - // 将分页信息写入数据库 - create_video_pages(&pages_info, &video_model, &txn).await?; - // 将页标记和 tag 写入数据库 - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.single_page = Set(Some(pages_info.len() == 1)); - video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); - video_active_model.save(&txn).await?; - txn.commit().await?; - } - Err(e) => { - error!( - "获取视频 {} - {} 的详细信息失败，错误为：{}", - &video_model.bvid, &video_model.name, e - ); - if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.valid = Set(false); - video_active_model.save(connection).await?; - } - } - }; - delay(CONFIG.delay.fetch_video_detail.as_ref()).await; - } + let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await; + match info { + Ok((tags, pages_info)) => { + let txn = connection.begin().await?; + // 将分页信息写入数据库 + create_video_pages(&pages_info, &video_model, &txn).await?; + // 将页标记和 tag 写入数据库 + let mut video_active_model: video::ActiveModel = video_model.into(); + video_active_model.single_page = Set(Some(pages_info.len() == 1)); + video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); + video_active_model.save(&txn).await?; + txn.commit().await?; + } + Err(e) => { + error_fetch_video_detail(e, video_model, connection).await?; + } + }; Ok(()) } diff --git a/crates/bili_sync/src/workflow.rs b/crates/bili_sync/src/workflow.rs index 81755d0..cf9e970 100644 --- a/crates/bili_sync/src/workflow.rs +++ b/crates/bili_sync/src/workflow.rs @@ -75,9 +75,13 @@ pub async fn fetch_video_details( ) -> Result> { video_list_model.log_fetch_video_start(); let videos_model = video_list_model.unfilled_videos(connection).await?; - video_list_model - .fetch_videos_detail(bili_client, videos_model, connection) - .await?; + for video_model in videos_model { + let video = Video::new(bili_client, video_model.bvid.clone()); + video_list_model + .fetch_videos_detail(video, video_model, connection) + .await?; + delay(CONFIG.delay.fetch_video_detail.as_ref()).await; + } video_list_model.log_fetch_video_end(); Ok(video_list_model) }