From a6d0d6b7775b0b15a145b0117d4ae968c47107c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=B4=80=E1=B4=8D=E1=B4=9B=E1=B4=8F=E1=B4=80=E1=B4=87?= =?UTF-8?q?=CA=80?= Date: Mon, 24 Feb 2025 19:48:07 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=B8=8B=E8=BD=BD=E6=97=B6=E8=80=83?= =?UTF-8?q?=E8=99=91=20backup=5Furl=EF=BC=8C=E6=94=AF=E6=8C=81=E6=8C=89?= =?UTF-8?q?=E7=85=A7=20cdn=20=E4=BC=98=E5=85=88=E7=BA=A7=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=20(#288)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/bili_sync/src/bilibili/analyzer.rs | 176 ++++++++++++++-------- crates/bili_sync/src/config/global.rs | 1 + crates/bili_sync/src/config/mod.rs | 3 + crates/bili_sync/src/downloader.rs | 18 ++- crates/bili_sync/src/workflow.rs | 12 +- 5 files changed, 146 insertions(+), 64 deletions(-) diff --git a/crates/bili_sync/src/bilibili/analyzer.rs b/crates/bili_sync/src/bilibili/analyzer.rs index d7a11de..78719ad 100644 --- a/crates/bili_sync/src/bilibili/analyzer.rs +++ b/crates/bili_sync/src/bilibili/analyzer.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use crate::bilibili::error::BiliError; +use crate::config::CONFIG; pub struct PageAnalyzer { info: serde_json::Value, @@ -101,24 +102,43 @@ pub enum Stream { EpisodeTryMp4(String), DashVideo { url: String, + backup_url: Vec, quality: VideoQuality, codecs: VideoCodecs, }, DashAudio { url: String, + backup_url: Vec, quality: AudioQuality, }, } // 通用的获取流链接的方法，交由 Downloader 使用 impl Stream { - pub fn url(&self) -> &str { + pub fn urls(&self) -> Vec<&str> { match self { - Self::Flv(url) => url, - Self::Html5Mp4(url) => url, - Self::EpisodeTryMp4(url) => url, - Self::DashVideo { url, .. } => url, - Self::DashAudio { url, .. } => url, + Self::Flv(url) | Self::Html5Mp4(url) | Self::EpisodeTryMp4(url) => vec![url], + Self::DashVideo { url, backup_url, .. } | Self::DashAudio { url, backup_url, .. } => { + let mut urls = std::iter::once(url.as_str()) + .chain(backup_url.iter().map(|s| s.as_str())) + .collect(); + if !CONFIG.cdn_sorting { + urls + } else { + urls.sort_by_key(|u| { + if u.contains("upos-") { + 0 // 服务商 cdn + } else if u.contains("cn-") { + 1 // 自建 cdn + } else if u.contains("mcdn") { + 2 // mcdn + } else { + 3 // pcdn 或者其它 + } + }); + urls + } + } } } } @@ -180,10 +200,12 @@ impl PageAnalyzer { )]); } let mut streams: Vec = Vec::new(); - for video in self.info["dash"]["video"] - .as_array() + for video in self + .info + .pointer_mut("/dash/video") + .and_then(|v| v.as_array_mut()) .ok_or(BiliError::RiskControlOccurred)? - .iter() + .iter_mut() { let (Some(url), Some(quality), Some(codecs)) = ( video["baseUrl"].as_str(), @@ -211,12 +233,13 @@ impl PageAnalyzer { } streams.push(Stream::DashVideo { url: url.to_string(), + backup_url: serde_json::from_value(video["backupUrl"].take()).unwrap_or_default(), quality, codecs, }); } - if let Some(audios) = self.info["dash"]["audio"].as_array() { - for audio in audios.iter() { + if let Some(audios) = self.info.pointer_mut("/dash/audio").and_then(|a| a.as_array_mut()) { + for audio in audios.iter_mut() { let (Some(url), Some(quality)) = (audio["baseUrl"].as_str(), audio["id"].as_u64()) else { continue; }; @@ -226,34 +249,44 @@ impl PageAnalyzer { } streams.push(Stream::DashAudio { url: url.to_string(), + backup_url: serde_json::from_value(audio["backupUrl"].take()).unwrap_or_default(), quality, }); } } - let flac = &self.info["dash"]["flac"]["audio"]; - if !(filter_option.no_hires || flac.is_null()) { - let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else { - bail!("invalid flac stream"); - }; - let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?; - if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality { - streams.push(Stream::DashAudio { - url: url.to_string(), - quality, - }); + if !filter_option.no_hires { + if let Some(flac) = self.info.pointer_mut("/dash/flac/audio") { + let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else { + bail!("invalid flac stream"); + }; + let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?; + if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality { + streams.push(Stream::DashAudio { + url: url.to_string(), + backup_url: serde_json::from_value(flac["backupUrl"].take()).unwrap_or_default(), + quality, + }); + } } } - let dolby_audio = &self.info["dash"]["dolby"]["audio"][0]; - if !(filter_option.no_dolby_audio || dolby_audio.is_null()) { - let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else { - bail!("invalid dolby audio stream"); - }; - let quality = AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?; - if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality { - streams.push(Stream::DashAudio { - url: url.to_string(), - quality, - }); + if !filter_option.no_dolby_audio { + if let Some(dolby_audio) = self + .info + .pointer_mut("/dash/dolby/audio/0") + .and_then(|a| a.as_object_mut()) + { + let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else { + bail!("invalid dolby audio stream"); + }; + let quality = + AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?; + if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality { + streams.push(Stream::DashAudio { + url: url.to_string(), + backup_url: serde_json::from_value(dolby_audio["backupUrl"].take()).unwrap_or_default(), + quality, + }); + } } } Ok(streams) @@ -270,32 +303,34 @@ impl PageAnalyzer { let (videos, audios): (Vec, Vec) = streams.into_iter().partition(|s| matches!(s, Stream::DashVideo { .. })); Ok(BestStream::VideoAudio { - video: Iterator::max_by(videos.into_iter(), |a, b| match (a, b) { - ( - Stream::DashVideo { - quality: a_quality, - codecs: a_codecs, - .. - }, - Stream::DashVideo { - quality: b_quality, - codecs: b_codecs, - .. - }, - ) => { - if a_quality != b_quality { - return a_quality.cmp(b_quality); - }; - filter_option - .codecs - .iter() - .position(|c| c == b_codecs) - .cmp(&filter_option.codecs.iter().position(|c| c == a_codecs)) - } - _ => unreachable!(), - }) - .context("no video stream found")?, - audio: Iterator::max_by(audios.into_iter(), |a, b| match (a, b) { + video: videos + .into_iter() + .max_by(|a, b| match (a, b) { + ( + Stream::DashVideo { + quality: a_quality, + codecs: a_codecs, + .. + }, + Stream::DashVideo { + quality: b_quality, + codecs: b_codecs, + .. + }, + ) => { + if a_quality != b_quality { + return a_quality.cmp(b_quality); + }; + filter_option + .codecs + .iter() + .position(|c| c == b_codecs) + .cmp(&filter_option.codecs.iter().position(|c| c == a_codecs)) + } + _ => unreachable!(), + }) + .context("no video stream found")?, + audio: audios.into_iter().max_by(|a, b| match (a, b) { (Stream::DashAudio { quality: a_quality, .. }, Stream::DashAudio { quality: b_quality, .. }) => { a_quality.cmp(b_quality) } @@ -389,4 +424,27 @@ mod tests { } } } + + #[test] + fn test_url_sort() { + let stream = Stream::DashVideo { + url: "https://xy116x207x155x163xy240ey95dy1010y700yy8dxy.mcdn.bilivideo.cn:4483".to_owned(), + backup_url: vec![ + "https://upos-sz-mirrorcos.bilivideo.com".to_owned(), + "https://cn-tj-cu-01-11.bilivideo.com".to_owned(), + "https://xxx.v1d.szbdys.com".to_owned(), + ], + quality: VideoQuality::Quality1080p, + codecs: VideoCodecs::AVC, + }; + assert_eq!( + stream.urls(), + vec![ + "https://upos-sz-mirrorcos.bilivideo.com", + "https://cn-tj-cu-01-11.bilivideo.com", + "https://xy116x207x155x163xy240ey95dy1010y700yy8dxy.mcdn.bilivideo.cn:4483", + "https://xxx.v1d.szbdys.com" + ] + ); + } } diff --git a/crates/bili_sync/src/config/global.rs b/crates/bili_sync/src/config/global.rs index b95ded1..a1b9ca5 100644 --- a/crates/bili_sync/src/config/global.rs +++ b/crates/bili_sync/src/config/global.rs @@ -81,6 +81,7 @@ fn load_config() -> Config { }; Config { credential: arc_swap::ArcSwapOption::from(credential), + cdn_sorting: true, ..Default::default() } } diff --git a/crates/bili_sync/src/config/mod.rs b/crates/bili_sync/src/config/mod.rs index 66837c1..60d7956 100644 --- a/crates/bili_sync/src/config/mod.rs +++ b/crates/bili_sync/src/config/mod.rs @@ -69,6 +69,8 @@ pub struct Config { pub concurrent_limit: ConcurrentLimit, #[serde(default = "default_time_format")] pub time_format: String, + #[serde(default)] + pub cdn_sorting: bool, } impl Default for Config { @@ -90,6 +92,7 @@ impl Default for Config { nfo_time_type: NFOTimeType::FavTime, concurrent_limit: ConcurrentLimit::default(), time_format: default_time_format(), + cdn_sorting: false, } } } diff --git a/crates/bili_sync/src/downloader.rs b/crates/bili_sync/src/downloader.rs index 48e3ff5..797221c 100644 --- a/crates/bili_sync/src/downloader.rs +++ b/crates/bili_sync/src/downloader.rs @@ -1,7 +1,7 @@ use core::str; use std::path::Path; -use anyhow::{Result, bail, ensure}; +use anyhow::{Context, Result, bail, ensure}; use futures::TryStreamExt; use reqwest::Method; use tokio::fs::{self, File}; @@ -45,6 +45,22 @@ impl Downloader { Ok(()) } + pub async fn fetch_with_fallback(&self, urls: &[&str], path: &Path) -> Result<()> { + if urls.is_empty() { + bail!("no urls provided"); + } + let mut res = Ok(()); + for url in urls { + match self.fetch(url, path).await { + Ok(_) => return Ok(()), + Err(err) => { + res = Err(err); + } + } + } + res.with_context(|| format!("failed to download from {:?}", urls)) + } + pub async fn merge(&self, video_path: &Path, audio_path: &Path, output_path: &Path) -> Result<()> { let output = tokio::process::Command::new("ffmpeg") .args([ diff --git a/crates/bili_sync/src/workflow.rs b/crates/bili_sync/src/workflow.rs index 1b9c4c2..561c885 100644 --- a/crates/bili_sync/src/workflow.rs +++ b/crates/bili_sync/src/workflow.rs @@ -535,11 +535,11 @@ pub async fn fetch_page_video( .await? .best_stream(&CONFIG.filter_option)?; match streams { - BestStream::Mixed(mix_stream) => downloader.fetch(mix_stream.url(), page_path).await?, + BestStream::Mixed(mix_stream) => downloader.fetch_with_fallback(&mix_stream.urls(), page_path).await?, BestStream::VideoAudio { video: video_stream, audio: None, - } => downloader.fetch(video_stream.url(), page_path).await?, + } => downloader.fetch_with_fallback(&video_stream.urls(), page_path).await?, BestStream::VideoAudio { video: video_stream, audio: Some(audio_stream), @@ -549,8 +549,12 @@ pub async fn fetch_page_video( page_path.with_extension("tmp_audio"), ); let res = async { - downloader.fetch(video_stream.url(), &tmp_video_path).await?; - downloader.fetch(audio_stream.url(), &tmp_audio_path).await?; + downloader + .fetch_with_fallback(&video_stream.urls(), &tmp_video_path) + .await?; + downloader + .fetch_with_fallback(&audio_stream.urls(), &tmp_audio_path) + .await?; downloader.merge(&tmp_video_path, &tmp_audio_path, page_path).await } .await;