Skip to content

Fix retry logic implementation #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nofyso merged 1 commit into from
Jul 8, 2026
Merged

Fix retry logic implementation #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 63 additions & 8 deletions cuscuta-common/src/api/xxxxxx.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,11 +388,13 @@ pub fn calc_friend_delta(
pub mod auto {
use std::time::Duration;

use tokio::time::sleep;
use reqwest::StatusCode;
use tokio::time::{Sleep, sleep};

use crate::api;

/// 一个包装，用于为一定量的[`StatusCode::TOO_MANY_REQUESTS`]错误提供弹性
/// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性
/// 使用简化的错误判断，任何错误均重试
///
/// # Errors
/// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\
Expand All @@ -409,19 +411,72 @@ pub mod auto {
Fut: Future<Output = Result<R, api::Error>> + 'a + Send,
R: Send + 'a,
F: Fn() -> Fut,
{
xxxxxx_safe_call_ex(
max_retries,
exponential_backoff_base_millis,
exponential_backoff_multiplier,
exponential_backoff_max_delay_millis,
|it| !it.is_success(),
f,
)
.await
}

/// 一个包装，用于为一定量的[`StatusCode`]错误和网络错误提供弹性
///
/// # Errors
/// 当本函数因重试次数过多而失败时，返回[`api::Error::TooManyRetries`]\
/// 否则，返回的错误由指定函数所可能引发的错误决定
#[allow(clippy::cast_possible_truncation)]
pub async fn xxxxxx_safe_call_ex<'a, F, R, T, Fut>(
max_retries: u64,
exponential_backoff_base_millis: u64,
exponential_backoff_multiplier: u64,
exponential_backoff_max_delay_millis: u64,
fail_cond: T,
f: F,
) -> Result<R, api::Error>
where
Fut: Future<Output = Result<R, api::Error>> + 'a + Send,
R: Send + 'a,
F: Fn() -> Fut,
T: Fn(StatusCode) -> bool,
{
//TODO add re-login
fn wait(
exponential_backoff_base_millis: u64,
exponential_backoff_multiplier: u64,
exponential_backoff_max_delay_millis: u64,
retries: u64,
) -> Sleep {
sleep(Duration::from_millis(
(exponential_backoff_base_millis
* exponential_backoff_multiplier.pow(retries as u32))
.min(exponential_backoff_max_delay_millis),
))
}
let mut retries = 0;
while retries <= max_retries {
let result = f().await;
match result {
Ok(result) => return Ok(result),
Err(api::Error::BadStatus(code)) if !code.is_success() => {
sleep(Duration::from_millis(
(exponential_backoff_base_millis
* exponential_backoff_multiplier.pow(retries as u32))
.min(exponential_backoff_max_delay_millis),
))
Err(api::Error::Network(_)) => {
wait(
exponential_backoff_base_millis,
exponential_backoff_multiplier,
exponential_backoff_max_delay_millis,
retries,
)
.await;
}
Err(api::Error::BadStatus(code)) if !fail_cond(code) => {
wait(
exponential_backoff_base_millis,
exponential_backoff_multiplier,
exponential_backoff_max_delay_millis,
retries,
)
.await;
}
Err(e) => return Err(e),
Expand Down
9 changes: 9 additions & 0 deletions cuscuta-mock/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,15 @@ async fn add_friend(headers: HeaderMap, form: Form<FriendAddForm>) -> impl IntoR
})),
);
};
if form.friend_code == "123456789" {
return (
StatusCode::NOT_FOUND,
Json(FriendModifyResult::Failed(FriendModifyResultFailed {
success: false,
error_code: 404,
})),
);
}
let i_header = i_header.to_str().unwrap().to_string();
//TODO add random not found fail
let result = internal_write_friends(
Expand Down
14 changes: 10 additions & 4 deletions cuscuta-worker/src/worker/pending_friend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use cuscuta_common::{
api::{
self,
xxxxxx::{FriendDelta, FriendInfo, auto::xxxxxx_safe_call, calc_friend_delta},
xxxxxx::{FriendDelta, FriendInfo, auto::xxxxxx_safe_call_ex, calc_friend_delta},
},
data::BundleData,
db::{
Expand All @@ -11,6 +11,7 @@ use cuscuta_common::{
},
};
use redis::{Client, Connection, TypedCommands};
use reqwest::StatusCode;

use crate::{data::Config, worker::Error};

Expand Down Expand Up @@ -56,11 +57,12 @@ pub async fn try_add_friends(
job.essential.cursor_start = cursor.cast_signed() as i32;
continue;
}
let result = xxxxxx_safe_call(
let result = xxxxxx_safe_call_ex(
config.worker_max_retry_count,
config.worker_exponential_backoff_base_millis,
config.worker_exponential_backoff_multiplier,
config.worker_exponential_backoff_max_delay_millis,
|it| it != StatusCode::TOO_MANY_REQUESTS,
|| {
api::xxxxxx::api_add_friend(
bundle_data,
Expand All @@ -76,9 +78,11 @@ pub async fn try_add_friends(
Err(e) => {
if let api::Error::BadStatus(code) = &e {
if *code == 400 {
log::warn!("friend is already exist but cache is out-dated!");
log::warn!(
"pending_friends: friend is already exist but cache is out-dated!"
);
} else {
log::warn!("failed to add friend: {e}: code: {code}");
log::warn!("pending_friends: failed to add friend: {e}: code: {code}");
job.state = JobState::Failed {
start_timestamp,
failure_info: JobFailure::new(
Expand All @@ -88,7 +92,9 @@ pub async fn try_add_friends(
friend_info: None,
};
}
continue;
}
log::warn!("pending_friends: unexpected error: {e}");
continue;
}
Ok(it) => it.friends,
Expand Down
Loading