mirror of
https://github.com/zhom/donutbrowser.git
synced 2026-10-20 22:41:22 +02:00
refactor: better logs
This commit is contained in:
1 parent
2604994507
commit
bf625fac2c
91 files changed
+4353
-3239
No files matched your search
+200
-329
File diff suppressed because it is too large.
Load diff
@@ -274,10 +274,7 @@ pub fn generate_manifest(
|
||||
let mut max_mtime: i64 = 0;
|
||||
|
||||
if !profile_dir.exists() {
|
||||
log::debug!(
|
||||
"Profile directory doesn't exist: {}, creating empty manifest",
|
||||
profile_dir.display()
|
||||
);
|
||||
log::debug!("Profile dir missing, empty manifest profile={profile_id}");
|
||||
return Ok(manifest);
|
||||
}
|
||||
|
||||
@@ -470,7 +467,7 @@ pub fn compute_diff_with_bias(
|
||||
// would appear "newer" and cause all remote files to be deleted.
|
||||
if local.files.is_empty() && !remote.files.is_empty() {
|
||||
log::info!(
|
||||
"Local manifest is empty but remote has {} files — downloading from remote to recover",
|
||||
"Local manifest empty, recovering from remote files={}",
|
||||
remote.files.len()
|
||||
);
|
||||
diff.files_to_download = remote.files.clone();
|
||||
|
||||
@@ -83,7 +83,7 @@ pub async fn resume_subscription(app_handle: tauri::AppHandle) {
|
||||
|
||||
if !subscription_manager.is_running() {
|
||||
if let Err(e) = subscription_manager.start(app_handle).await {
|
||||
log::warn!("Failed to resume sync subscription: {e}");
|
||||
log::warn!("Sync subscription resume failed err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ pub async fn start_pipeline(app_handle: tauri::AppHandle) {
|
||||
|
||||
let mut subscription_manager = SubscriptionManager::new();
|
||||
let Some(work_rx) = subscription_manager.take_work_receiver() else {
|
||||
log::error!("Sync pipeline has no work receiver; not starting");
|
||||
log::error!("Sync pipeline not started: no work receiver");
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -125,7 +125,7 @@ pub async fn start_pipeline(app_handle: tauri::AppHandle) {
|
||||
// restart path used to give up here, which turned a token hiccup into sync
|
||||
// being dead until the next launch.
|
||||
if let Err(e) = subscription_manager.start(app_handle.clone()).await {
|
||||
log::warn!("Failed to start sync subscription, continuing without live updates: {e}");
|
||||
log::warn!("Sync subscription not started, no live updates err=\"{e}\"");
|
||||
}
|
||||
if let Ok(mut guard) = GLOBAL_SUBSCRIPTION.lock() {
|
||||
*guard = Some(subscription_manager);
|
||||
@@ -143,14 +143,14 @@ pub async fn start_pipeline(app_handle: tauri::AppHandle) {
|
||||
match SyncEngine::create_from_settings(&app_handle).await {
|
||||
Ok(engine) => {
|
||||
if let Err(e) = engine.check_for_missing_synced_profiles(&app_handle).await {
|
||||
log::warn!("Failed to check for missing profiles: {e}");
|
||||
log::warn!("Missing profile check failed err=\"{e}\"");
|
||||
}
|
||||
if let Err(e) = engine.check_for_missing_synced_entities(&app_handle).await {
|
||||
log::warn!("Failed to check for missing entities: {e}");
|
||||
log::warn!("Missing entity check failed err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("Sync not configured, skipping missing profile check: {e}");
|
||||
log::debug!("Missing profile check skipped err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ fn read() -> Vec<PendingDelete> {
|
||||
return Vec::new();
|
||||
};
|
||||
serde_json::from_str(&content).unwrap_or_else(|e| {
|
||||
log::warn!("Ignoring unreadable {FILE_NAME}: {e}");
|
||||
log::warn!("Pending cloud deletes unreadable, ignored file={FILE_NAME} err=\"{e}\"");
|
||||
Vec::new()
|
||||
})
|
||||
}
|
||||
@@ -54,7 +54,7 @@ fn write(entries: &[PendingDelete]) {
|
||||
if entries.is_empty() {
|
||||
if let Err(e) = std::fs::remove_file(&path) {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
log::warn!("Could not remove {}: {e}", path.display());
|
||||
log::warn!("Pending cloud deletes not removed file={FILE_NAME} err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
return;
|
||||
@@ -70,7 +70,7 @@ fn write(entries: &[PendingDelete]) {
|
||||
std::fs::rename(&tmp, &path)
|
||||
});
|
||||
if let Err(e) = result {
|
||||
log::warn!("Could not write {}: {e}", path.display());
|
||||
log::warn!("Pending cloud deletes not written file={FILE_NAME} err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +114,7 @@ pub async fn wait_for(profile_id: &str) {
|
||||
let handle = lock(&RUNNING).remove(profile_id);
|
||||
if let Some(handle) = handle {
|
||||
if let Err(e) = handle.await {
|
||||
log::warn!("The cloud delete of profile {profile_id} ended abnormally: {e}");
|
||||
log::warn!("Cloud profile delete task ended abnormally profile={profile_id} err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ pub async fn check_sync_server(server_url: &str) -> SyncServerCheck {
|
||||
let readyz = match client.get(format!("{base}/readyz")).send().await {
|
||||
Ok(response) => response,
|
||||
Err(e) => {
|
||||
log::warn!("Sync pre-flight: {base}/readyz did not answer: {e}");
|
||||
log::warn!("Sync preflight: readyz unreachable err=\"{e}\"");
|
||||
return check;
|
||||
}
|
||||
};
|
||||
@@ -119,7 +119,7 @@ pub async fn check_sync_server(server_url: &str) -> SyncServerCheck {
|
||||
match probe_storage_endpoint(&client, &endpoint).await {
|
||||
Ok(()) => check.storage_reachable = Some(true),
|
||||
Err(e) => {
|
||||
log::warn!("Sync pre-flight: storage endpoint {endpoint} is unreachable from here: {e}");
|
||||
log::warn!("Sync preflight: storage endpoint unreachable endpoint={endpoint} err=\"{e}\"");
|
||||
check.storage_reachable = Some(false);
|
||||
check.storage_error = Some(e);
|
||||
}
|
||||
|
||||
+143
-106
@@ -2,16 +2,60 @@ use super::engine::SyncEngine;
|
||||
use super::subscription::SyncWorkItem;
|
||||
use crate::events;
|
||||
use crate::log_redaction::Plain;
|
||||
use crate::log_streak::Streak;
|
||||
use crate::profile::ProfileManager;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
static GLOBAL_SCHEDULER: std::sync::Mutex<Option<Arc<SyncScheduler>>> = std::sync::Mutex::new(None);
|
||||
|
||||
static SYNC_ENGINE: Streak = Streak::new(module_path!(), "Sync engine setup");
|
||||
|
||||
/// The last error per `kind=id`, so an entity that keeps failing the same way
|
||||
/// logs once, not on every pass.
|
||||
static FAILING: LazyLock<std::sync::Mutex<HashMap<String, String>>> =
|
||||
LazyLock::new(Default::default);
|
||||
|
||||
/// Tombstones already reported this run. The server replays every tombstone
|
||||
/// to each new SSE connection.
|
||||
static NOTED_TOMBSTONES: LazyLock<std::sync::Mutex<HashSet<String>>> =
|
||||
LazyLock::new(Default::default);
|
||||
|
||||
fn log_sync_failure(kind: &str, id: &str, err: impl Display) {
|
||||
let err = err.to_string();
|
||||
let repeated = FAILING
|
||||
.lock()
|
||||
.map(|mut failing| failing.insert(format!("{kind}={id}"), err.clone()).as_ref() == Some(&err))
|
||||
.unwrap_or(false);
|
||||
if repeated {
|
||||
log::debug!("Sync failed again {kind}={id} err=\"{err}\"");
|
||||
} else {
|
||||
log::error!("Sync failed {kind}={id} err=\"{err}\"");
|
||||
}
|
||||
}
|
||||
|
||||
fn log_sync_success(kind: &str, id: &str) {
|
||||
let recovered = FAILING
|
||||
.lock()
|
||||
.map(|mut failing| failing.remove(&format!("{kind}={id}")).is_some())
|
||||
.unwrap_or(false);
|
||||
if recovered {
|
||||
log::info!("Sync recovered {kind}={id}");
|
||||
}
|
||||
}
|
||||
|
||||
fn first_tombstone_notice(entity_id: &str) -> bool {
|
||||
NOTED_TOMBSTONES
|
||||
.lock()
|
||||
.map(|mut noted| noted.insert(entity_id.to_string()))
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
pub fn get_global_scheduler() -> Option<Arc<SyncScheduler>> {
|
||||
GLOBAL_SCHEDULER.lock().ok().and_then(|g| g.clone())
|
||||
}
|
||||
@@ -171,13 +215,13 @@ impl SyncScheduler {
|
||||
pub async fn mark_profile_running(&self, profile_id: &str) {
|
||||
let mut running = self.running_profiles.lock().await;
|
||||
running.insert(profile_id.to_string());
|
||||
log::debug!("Marked profile {} as running", profile_id);
|
||||
log::debug!("Profile marked running profile={profile_id}");
|
||||
}
|
||||
|
||||
pub async fn mark_profile_stopped(&self, profile_id: &str) {
|
||||
let mut running = self.running_profiles.lock().await;
|
||||
running.remove(profile_id);
|
||||
log::debug!("Marked profile {} as stopped", profile_id);
|
||||
log::debug!("Profile marked stopped profile={profile_id}");
|
||||
|
||||
let mut pending = self.pending_profiles.lock().await;
|
||||
if pending.contains_key(profile_id) {
|
||||
@@ -189,10 +233,7 @@ impl SyncScheduler {
|
||||
queued: true,
|
||||
},
|
||||
);
|
||||
log::debug!(
|
||||
"Profile {} has pending sync, will execute immediately",
|
||||
profile_id
|
||||
);
|
||||
log::debug!("Profile pending sync released profile={profile_id}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -209,10 +250,7 @@ impl SyncScheduler {
|
||||
.is_locked_by_another(profile_id)
|
||||
.await
|
||||
{
|
||||
log::debug!(
|
||||
"Profile {} is locked on another device, treating as running",
|
||||
profile_id
|
||||
);
|
||||
log::debug!("Profile locked on another device, treated as running profile={profile_id}");
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -241,7 +279,7 @@ impl SyncScheduler {
|
||||
},
|
||||
);
|
||||
log::debug!(
|
||||
"Profile {} is running, queued sync for after stop",
|
||||
"Profile sync queued until stop profile={}",
|
||||
Plain(&profile_id)
|
||||
);
|
||||
} else {
|
||||
@@ -253,7 +291,7 @@ impl SyncScheduler {
|
||||
queued: true,
|
||||
},
|
||||
);
|
||||
log::debug!("Profile {} queued for immediate sync", Plain(&profile_id));
|
||||
log::debug!("Profile sync queued profile={}", Plain(&profile_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,14 +331,12 @@ impl SyncScheduler {
|
||||
}
|
||||
|
||||
pub async fn sync_all_enabled_profiles(&self, _app_handle: &tauri::AppHandle) {
|
||||
log::info!("Starting initial sync for all enabled profiles...");
|
||||
|
||||
let profiles = {
|
||||
let profile_manager = ProfileManager::instance();
|
||||
match profile_manager.list_profiles() {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
log::error!("Failed to list profiles for initial sync: {e}");
|
||||
log::error!("Initial sync not queued: profile list failed err=\"{e}\"");
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -312,12 +348,12 @@ impl SyncScheduler {
|
||||
.collect();
|
||||
|
||||
if sync_enabled_profiles.is_empty() {
|
||||
log::debug!("No sync-enabled profiles found");
|
||||
log::debug!("Initial sync skipped: no sync-enabled profiles");
|
||||
return;
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"Found {} sync-enabled profiles, queueing for sync",
|
||||
"Initial sync queued profiles={}",
|
||||
sync_enabled_profiles.len()
|
||||
);
|
||||
|
||||
@@ -335,14 +371,9 @@ impl SyncScheduler {
|
||||
}
|
||||
|
||||
if should_wait {
|
||||
log::info!(
|
||||
"Profile '{}' is {} — will sync after it becomes available",
|
||||
profile.name,
|
||||
if is_running {
|
||||
"running locally"
|
||||
} else {
|
||||
"locked by a team member"
|
||||
}
|
||||
log::debug!(
|
||||
"Initial sync deferred profile={profile_id} reason={}",
|
||||
if is_running { "running" } else { "team_locked" }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -388,11 +419,11 @@ impl SyncScheduler {
|
||||
// Retired while the pipeline was still assembling it. Starting now
|
||||
// would leave a task nothing can stop, because the handle in the global
|
||||
// has already been replaced.
|
||||
log::info!("Sync scheduler was retired before it started; not starting it");
|
||||
log::info!("Sync scheduler not started: retired before start");
|
||||
return false;
|
||||
}
|
||||
StartDecision::AlreadyRunning => {
|
||||
log::warn!("Sync scheduler is already running; ignoring the second start");
|
||||
log::warn!("Sync scheduler already running, second start ignored");
|
||||
return false;
|
||||
}
|
||||
StartDecision::Start => {}
|
||||
@@ -445,9 +476,7 @@ impl SyncScheduler {
|
||||
// The subscription is gone, so no more live updates from other
|
||||
// devices. Local changes and the timer still work, so keep
|
||||
// ticking rather than ending the scheduler.
|
||||
log::warn!(
|
||||
"Sync work channel closed; continuing on the timer without live updates"
|
||||
);
|
||||
log::warn!("Sync work channel closed, timer only from now");
|
||||
work_channel_open = false;
|
||||
}
|
||||
}
|
||||
@@ -503,7 +532,7 @@ impl SyncScheduler {
|
||||
for profile_id in profiles_to_sync {
|
||||
let mut in_flight = self.in_flight_profiles.lock().await;
|
||||
if in_flight.contains(&profile_id) {
|
||||
log::debug!("Profile {} already in-flight, skipping", profile_id);
|
||||
log::debug!("Profile sync already in flight profile={profile_id}");
|
||||
continue;
|
||||
}
|
||||
in_flight.insert(profile_id.clone());
|
||||
@@ -516,7 +545,8 @@ impl SyncScheduler {
|
||||
let app = app_handle.clone();
|
||||
let in_flight = self.in_flight_profiles.clone();
|
||||
sync_set.spawn(async move {
|
||||
log::info!("Executing queued sync for profile {}", profile_id);
|
||||
let started = Instant::now();
|
||||
log::debug!("Profile sync started profile={profile_id}");
|
||||
let _ = events::emit(
|
||||
"profile-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -542,12 +572,13 @@ impl SyncScheduler {
|
||||
|
||||
let result = match SyncEngine::create_from_settings(&app).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
engine
|
||||
.sync_profile_with_bias(&app, &profile, super::DiffBias::Auto)
|
||||
.await
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
SYNC_ENGINE.failed(e);
|
||||
Err(super::types::SyncError::NotConfigured)
|
||||
}
|
||||
};
|
||||
@@ -559,7 +590,11 @@ impl SyncScheduler {
|
||||
|
||||
match result {
|
||||
Ok(super::ProfileSyncOutcome::Completed) => {
|
||||
log::info!("Profile {} synced successfully", profile_id);
|
||||
log_sync_success("profile", &profile_id);
|
||||
log::debug!(
|
||||
"Profile sync completed profile={profile_id} elapsed_ms={}",
|
||||
started.elapsed().as_millis()
|
||||
);
|
||||
let _ = events::emit(
|
||||
"profile-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -569,10 +604,18 @@ impl SyncScheduler {
|
||||
);
|
||||
}
|
||||
Ok(super::ProfileSyncOutcome::Skipped(reason)) => {
|
||||
log::debug!("Profile {profile_id} sync is waiting: {reason}");
|
||||
log::debug!("Profile sync skipped profile={profile_id} reason=\"{reason}\"");
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to sync profile {}: {}", profile_id, e);
|
||||
// Both are logged where they happen.
|
||||
if matches!(
|
||||
e,
|
||||
super::types::SyncError::Cancelled | super::types::SyncError::NotConfigured
|
||||
) {
|
||||
log::debug!("Profile sync ended profile={profile_id} err=\"{e}\"");
|
||||
} else {
|
||||
log_sync_failure("profile", &profile_id, &e);
|
||||
}
|
||||
let _ = events::emit(
|
||||
"profile-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -590,7 +633,7 @@ impl SyncScheduler {
|
||||
if !sync_set.is_empty() {
|
||||
while let Some(result) = sync_set.join_next().await {
|
||||
if let Err(e) = result {
|
||||
log::error!("Profile sync task panicked: {e}");
|
||||
log::error!("Profile sync task panicked err=\"{e}\"");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -609,8 +652,9 @@ impl SyncScheduler {
|
||||
|
||||
match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
for proxy_id in proxies_to_sync {
|
||||
log::info!("Syncing proxy {}", proxy_id);
|
||||
log::debug!("Sync started proxy={proxy_id}");
|
||||
let _ = events::emit(
|
||||
"proxy-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -623,6 +667,7 @@ impl SyncScheduler {
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
log_sync_success("proxy", &proxy_id);
|
||||
let _ = events::emit(
|
||||
"proxy-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -632,7 +677,7 @@ impl SyncScheduler {
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to sync proxy {}: {}", proxy_id, e);
|
||||
log_sync_failure("proxy", &proxy_id, &e);
|
||||
let _ = events::emit(
|
||||
"proxy-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -647,9 +692,7 @@ impl SyncScheduler {
|
||||
|
||||
// Check if all sync work is complete after proxies finish
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
}
|
||||
Err(e) => SYNC_ENGINE.failed(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -666,8 +709,9 @@ impl SyncScheduler {
|
||||
|
||||
match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
for group_id in groups_to_sync {
|
||||
log::info!("Syncing group {}", group_id);
|
||||
log::debug!("Sync started group={group_id}");
|
||||
let _ = events::emit(
|
||||
"group-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -680,6 +724,7 @@ impl SyncScheduler {
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
log_sync_success("group", &group_id);
|
||||
let _ = events::emit(
|
||||
"group-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -689,7 +734,7 @@ impl SyncScheduler {
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to sync group {}: {}", group_id, e);
|
||||
log_sync_failure("group", &group_id, &e);
|
||||
let _ = events::emit(
|
||||
"group-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -704,9 +749,7 @@ impl SyncScheduler {
|
||||
|
||||
// Check if all sync work is complete after groups finish
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
}
|
||||
Err(e) => SYNC_ENGINE.failed(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -723,8 +766,9 @@ impl SyncScheduler {
|
||||
|
||||
match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
for vpn_id in vpns_to_sync {
|
||||
log::info!("Syncing VPN {}", vpn_id);
|
||||
log::debug!("Sync started vpn={vpn_id}");
|
||||
let _ = events::emit(
|
||||
"vpn-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -734,6 +778,7 @@ impl SyncScheduler {
|
||||
);
|
||||
match engine.sync_vpn_by_id_with_handle(&vpn_id, app_handle).await {
|
||||
Ok(()) => {
|
||||
log_sync_success("vpn", &vpn_id);
|
||||
let _ = events::emit(
|
||||
"vpn-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -743,7 +788,7 @@ impl SyncScheduler {
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to sync VPN {}: {}", vpn_id, e);
|
||||
log_sync_failure("vpn", &vpn_id, &e);
|
||||
let _ = events::emit(
|
||||
"vpn-sync-status",
|
||||
serde_json::json!({
|
||||
@@ -756,9 +801,7 @@ impl SyncScheduler {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
}
|
||||
Err(e) => SYNC_ENGINE.failed(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -775,8 +818,9 @@ impl SyncScheduler {
|
||||
|
||||
match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
for ext_id in extensions_to_sync {
|
||||
log::info!("Syncing extension {}", ext_id);
|
||||
log::debug!("Sync started extension={ext_id}");
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": ext_id, "status": "syncing" }),
|
||||
@@ -785,12 +829,13 @@ impl SyncScheduler {
|
||||
.sync_extension_by_id_with_handle(&ext_id, app_handle)
|
||||
.await
|
||||
{
|
||||
log::error!("Failed to sync extension {}: {}", ext_id, e);
|
||||
log_sync_failure("extension", &ext_id, &e);
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": ext_id, "status": "error" }),
|
||||
);
|
||||
} else {
|
||||
log_sync_success("extension", &ext_id);
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": ext_id, "status": "synced" }),
|
||||
@@ -798,9 +843,7 @@ impl SyncScheduler {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
}
|
||||
Err(e) => SYNC_ENGINE.failed(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -817,8 +860,9 @@ impl SyncScheduler {
|
||||
|
||||
match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
for group_id in groups_to_sync {
|
||||
log::info!("Syncing extension group {}", group_id);
|
||||
log::debug!("Sync started extension_group={group_id}");
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": group_id, "status": "syncing" }),
|
||||
@@ -827,12 +871,13 @@ impl SyncScheduler {
|
||||
.sync_extension_group_by_id_with_handle(&group_id, app_handle)
|
||||
.await
|
||||
{
|
||||
log::error!("Failed to sync extension group {}: {}", group_id, e);
|
||||
log_sync_failure("extension_group", &group_id, &e);
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": group_id, "status": "error" }),
|
||||
);
|
||||
} else {
|
||||
log_sync_success("extension_group", &group_id);
|
||||
let _ = events::emit(
|
||||
"extension-sync-status",
|
||||
serde_json::json!({ "id": group_id, "status": "synced" }),
|
||||
@@ -840,9 +885,7 @@ impl SyncScheduler {
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("Failed to create sync engine: {}", e);
|
||||
}
|
||||
Err(e) => SYNC_ENGINE.failed(e),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -884,27 +927,30 @@ impl SyncScheduler {
|
||||
}
|
||||
|
||||
for (entity_type, entity_id) in tombstones {
|
||||
log::info!("Processing tombstone for {} {}", entity_type, entity_id);
|
||||
log::debug!("Tombstone received {entity_type}={entity_id}");
|
||||
match entity_type.as_str() {
|
||||
"profile" => {
|
||||
let profile_manager = ProfileManager::instance();
|
||||
let local = uuid::Uuid::parse_str(&entity_id)
|
||||
.ok()
|
||||
.and_then(|uuid| {
|
||||
profile_manager
|
||||
.list_profiles()
|
||||
.ok()?
|
||||
.into_iter()
|
||||
.find(|p| p.id == uuid)
|
||||
})
|
||||
.filter(|p| p.is_sync_enabled());
|
||||
let found = uuid::Uuid::parse_str(&entity_id).ok().and_then(|uuid| {
|
||||
profile_manager
|
||||
.list_profiles()
|
||||
.ok()?
|
||||
.into_iter()
|
||||
.find(|p| p.id == uuid)
|
||||
});
|
||||
|
||||
let Some(local) = local else {
|
||||
log::info!(
|
||||
"Profile {} has a tombstone but sync is no longer enabled locally — keeping local copy",
|
||||
entity_id
|
||||
);
|
||||
continue;
|
||||
let local = match found {
|
||||
Some(profile) if profile.is_sync_enabled() => profile,
|
||||
Some(_) => {
|
||||
if first_tombstone_notice(&entity_id) {
|
||||
log::info!("Tombstone ignored, sync off locally profile={entity_id}");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
None => {
|
||||
log::debug!("Tombstone ignored, profile not on this device profile={entity_id}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
// The event can be stale. A restore from the trash, or sync switched
|
||||
@@ -912,30 +958,25 @@ impl SyncScheduler {
|
||||
// the profile it named is live again. Only a tombstone that is still
|
||||
// in the cloud erases the local copy.
|
||||
let still_tombstoned = match SyncEngine::create_from_settings(app_handle).await {
|
||||
Ok(engine) => engine.profile_tombstone_exists(&local).await,
|
||||
Ok(engine) => {
|
||||
SYNC_ENGINE.succeeded();
|
||||
engine.profile_tombstone_exists(&local).await
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Could not confirm the tombstone of profile {}: {}",
|
||||
entity_id,
|
||||
e
|
||||
);
|
||||
false
|
||||
SYNC_ENGINE.failed(e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !still_tombstoned {
|
||||
log::info!(
|
||||
"Profile {} tombstone is no longer in the cloud — keeping local copy",
|
||||
entity_id
|
||||
);
|
||||
log::info!("Tombstone stale, kept local profile profile={entity_id}");
|
||||
continue;
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"Profile {} was deleted remotely, deleting locally",
|
||||
entity_id
|
||||
);
|
||||
if let Err(e) = profile_manager.delete_profile_local_only(&entity_id) {
|
||||
log::warn!("Failed to delete tombstoned profile {}: {}", entity_id, e);
|
||||
match profile_manager.delete_profile_local_only(&entity_id) {
|
||||
Ok(_) => log::info!("Profile deleted remotely, removed locally profile={entity_id}"),
|
||||
Err(e) => log::warn!(
|
||||
"Remotely deleted profile not removed locally profile={entity_id} err=\"{e}\""
|
||||
),
|
||||
}
|
||||
}
|
||||
"proxy" => {
|
||||
@@ -943,7 +984,7 @@ impl SyncScheduler {
|
||||
let proxies = proxy_manager.get_stored_proxies();
|
||||
if let Some(proxy) = proxies.iter().find(|p| p.id == entity_id) {
|
||||
if proxy.sync_enabled {
|
||||
log::info!("Proxy {} was deleted remotely, deleting locally", entity_id);
|
||||
log::info!("Proxy deleted remotely, removed locally proxy={entity_id}");
|
||||
let proxy_file = proxy_manager.get_proxy_file_path(&entity_id);
|
||||
if proxy_file.exists() {
|
||||
let _ = std::fs::remove_file(&proxy_file);
|
||||
@@ -958,7 +999,7 @@ impl SyncScheduler {
|
||||
let groups = group_manager.get_all_groups().unwrap_or_default();
|
||||
if let Some(group) = groups.iter().find(|g| g.id == entity_id) {
|
||||
if group.sync_enabled {
|
||||
log::info!("Group {} was deleted remotely, deleting locally", entity_id);
|
||||
log::info!("Group deleted remotely, removed locally group={entity_id}");
|
||||
let _ = group_manager.delete_group_internal(&entity_id);
|
||||
let _ = events::emit("groups-changed", ());
|
||||
}
|
||||
@@ -968,7 +1009,7 @@ impl SyncScheduler {
|
||||
let storage = crate::vpn::VPN_STORAGE.lock().unwrap();
|
||||
if let Ok(vpn) = storage.load_config(&entity_id) {
|
||||
if vpn.sync_enabled {
|
||||
log::info!("VPN {} was deleted remotely, deleting locally", entity_id);
|
||||
log::info!("VPN deleted remotely, removed locally vpn={entity_id}");
|
||||
let _ = storage.delete_config(&entity_id);
|
||||
let _ = events::emit("vpn-configs-changed", ());
|
||||
}
|
||||
@@ -978,10 +1019,7 @@ impl SyncScheduler {
|
||||
let manager = crate::extension_manager::EXTENSION_MANAGER.lock().unwrap();
|
||||
if let Ok(ext) = manager.get_extension(&entity_id) {
|
||||
if ext.sync_enabled {
|
||||
log::info!(
|
||||
"Extension {} was deleted remotely, deleting locally",
|
||||
entity_id
|
||||
);
|
||||
log::info!("Extension deleted remotely, removed locally extension={entity_id}");
|
||||
let _ = manager.delete_extension_internal(&entity_id);
|
||||
let _ = events::emit("extensions-changed", ());
|
||||
}
|
||||
@@ -992,8 +1030,7 @@ impl SyncScheduler {
|
||||
if let Ok(group) = manager.get_group(&entity_id) {
|
||||
if group.sync_enabled {
|
||||
log::info!(
|
||||
"Extension group {} was deleted remotely, deleting locally",
|
||||
entity_id
|
||||
"Extension group deleted remotely, removed locally extension_group={entity_id}"
|
||||
);
|
||||
let _ = manager.delete_group_internal(&entity_id);
|
||||
let _ = events::emit("extensions-changed", ());
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::events;
|
||||
use crate::log_streak::Streak;
|
||||
use crate::settings_manager::SettingsManager;
|
||||
use reqwest::Client;
|
||||
use serde::Deserialize;
|
||||
@@ -41,6 +42,10 @@ const RECONNECT_BASE_DELAY: Duration = Duration::from_secs(6);
|
||||
const RECONNECT_MAX_DELAY: Duration = Duration::from_secs(600);
|
||||
const STOP_POLL_INTERVAL: Duration = Duration::from_secs(1);
|
||||
|
||||
static SSE_CONNECT: Streak = Streak::new(module_path!(), "Sync SSE connect");
|
||||
// Cloud token failures are already logged by `cloud_auth`.
|
||||
static SELF_HOSTED_TOKEN: Streak = Streak::new(module_path!(), "Self-hosted sync token read");
|
||||
|
||||
#[derive(Debug)]
|
||||
enum StreamFailure {
|
||||
Refused(u16),
|
||||
@@ -85,7 +90,7 @@ fn stream_failure(status: u16, headers: &reqwest::header::HeaderMap) -> StreamFa
|
||||
match status {
|
||||
401 | 403 => StreamFailure::Refused(status),
|
||||
429 => StreamFailure::RateLimited(crate::cloud_auth::retry_after(headers)),
|
||||
_ => StreamFailure::Failed(format!("SSE connection failed with status: {status}")),
|
||||
_ => StreamFailure::Failed(format!("status={status}")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,8 +138,7 @@ impl SyncSubscription {
|
||||
let url = crate::cloud_auth::CLOUD_SYNC_URL.to_string();
|
||||
let token = crate::cloud_auth::CLOUD_AUTH
|
||||
.get_or_refresh_sync_token()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to get cloud sync token: {e}"))?;
|
||||
.await?;
|
||||
let Some(token) = token else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -191,6 +195,7 @@ impl SyncSubscription {
|
||||
tokio::spawn(async move {
|
||||
let mut refused: Option<String> = None;
|
||||
let mut failures: u32 = 0;
|
||||
let mut stop_reason = "requested";
|
||||
|
||||
while running.load(Ordering::SeqCst) {
|
||||
let mut wait_at_least = Duration::ZERO;
|
||||
@@ -211,10 +216,10 @@ impl SyncSubscription {
|
||||
}
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
log::info!("SSE connection closed gracefully");
|
||||
log::debug!("Sync SSE stream closed by server");
|
||||
}
|
||||
Err(StreamFailure::Refused(status)) => {
|
||||
log::warn!("Sync subscription refused with status {status}");
|
||||
log::warn!("Sync subscription refused status={status}");
|
||||
if matches!(source, TokenSource::Cloud) {
|
||||
crate::cloud_auth::CloudAuthManager::discard_cloud_sync_token(¤t);
|
||||
}
|
||||
@@ -222,11 +227,11 @@ impl SyncSubscription {
|
||||
token = None;
|
||||
}
|
||||
Err(StreamFailure::RateLimited(retry)) => {
|
||||
log::warn!("Sync subscription rate limited for {}s", retry.as_secs());
|
||||
SSE_CONNECT.failed(format!("rate limited retry_after_s={}", retry.as_secs()));
|
||||
wait_at_least = retry;
|
||||
}
|
||||
Err(StreamFailure::Failed(e)) => {
|
||||
log::warn!("SSE connection error: {e}");
|
||||
SSE_CONNECT.failed(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -237,7 +242,7 @@ impl SyncSubscription {
|
||||
|
||||
failures = failures.saturating_add(1);
|
||||
let delay = reconnect_delay(failures).max(wait_at_least);
|
||||
log::debug!("Reconnecting the sync subscription in {}s", delay.as_secs());
|
||||
log::debug!("Sync SSE reconnect scheduled delay_s={}", delay.as_secs());
|
||||
sleep_while_running(&running, delay).await;
|
||||
|
||||
if running.load(Ordering::SeqCst) {
|
||||
@@ -246,8 +251,11 @@ impl SyncSubscription {
|
||||
// the construction-time token otherwise produces an endless 401
|
||||
// reconnect loop until the app is restarted.
|
||||
let fetched = Self::fetch_sync_token(source, &app_handle).await;
|
||||
if let Err(e) = &fetched {
|
||||
log::warn!("Failed to refresh sync token: {e}");
|
||||
if matches!(source, TokenSource::SelfHosted) {
|
||||
match &fetched {
|
||||
Ok(_) => SELF_HOSTED_TOKEN.succeeded(),
|
||||
Err(e) => SELF_HOSTED_TOKEN.failed(e),
|
||||
}
|
||||
}
|
||||
match next_token(refused.as_deref(), fetched) {
|
||||
TokenStep::Use(fresh) => {
|
||||
@@ -256,7 +264,7 @@ impl SyncSubscription {
|
||||
}
|
||||
TokenStep::Keep | TokenStep::Wait => {}
|
||||
TokenStep::Stop => {
|
||||
log::info!("Sync token no longer available; stopping subscription");
|
||||
stop_reason = "no_token";
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -264,7 +272,7 @@ impl SyncSubscription {
|
||||
}
|
||||
|
||||
running.store(false, Ordering::SeqCst);
|
||||
log::info!("Sync subscription stopped");
|
||||
log::info!("Sync subscription stopped reason={stop_reason}");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -275,14 +283,15 @@ impl SyncSubscription {
|
||||
app_handle: &tauri::AppHandle,
|
||||
) -> Result<Option<String>, String> {
|
||||
match source {
|
||||
TokenSource::Cloud => crate::cloud_auth::CLOUD_AUTH
|
||||
.get_or_refresh_sync_token()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to refresh cloud sync token: {e}")),
|
||||
TokenSource::Cloud => {
|
||||
crate::cloud_auth::CLOUD_AUTH
|
||||
.get_or_refresh_sync_token()
|
||||
.await
|
||||
}
|
||||
TokenSource::SelfHosted => SettingsManager::instance()
|
||||
.get_sync_token(app_handle)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to refresh self-hosted sync token: {e}")),
|
||||
.map_err(|e| e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,7 +311,7 @@ impl SyncSubscription {
|
||||
.header("Accept", "text/event-stream")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| StreamFailure::Failed(format!("Failed to connect to SSE: {e}")))?;
|
||||
.map_err(|e| StreamFailure::Failed(e.to_string()))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(stream_failure(
|
||||
@@ -312,17 +321,24 @@ impl SyncSubscription {
|
||||
}
|
||||
|
||||
*connected = true;
|
||||
log::info!("Connected to sync subscription");
|
||||
log::debug!("Sync SSE connected");
|
||||
let _ = events::emit("sync-subscription-status", "connected");
|
||||
|
||||
let mut buffer = String::new();
|
||||
let mut bytes_stream = response.bytes_stream();
|
||||
// A proxy can accept the connection and cut it before any data; only data
|
||||
// proves the link works, so that is what ends a failure streak.
|
||||
let mut proven = false;
|
||||
|
||||
use futures_util::StreamExt;
|
||||
|
||||
while running.load(Ordering::SeqCst) {
|
||||
match tokio::time::timeout(Duration::from_secs(60), bytes_stream.next()).await {
|
||||
Ok(Some(Ok(bytes))) => {
|
||||
if !proven {
|
||||
proven = true;
|
||||
SSE_CONNECT.succeeded();
|
||||
}
|
||||
let chunk = String::from_utf8_lossy(&bytes);
|
||||
buffer.push_str(&chunk);
|
||||
|
||||
@@ -336,13 +352,13 @@ impl SyncSubscription {
|
||||
}
|
||||
}
|
||||
Ok(Some(Err(e))) => {
|
||||
return Err(StreamFailure::Failed(format!("SSE stream error: {e}")));
|
||||
return Err(StreamFailure::Failed(format!("stream: {e}")));
|
||||
}
|
||||
Ok(None) => {
|
||||
return Ok(());
|
||||
}
|
||||
Err(_) => {
|
||||
log::debug!("SSE timeout, continuing...");
|
||||
log::debug!("Sync SSE idle for 60s");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -463,7 +479,7 @@ impl SyncSubscription {
|
||||
};
|
||||
|
||||
if let Some(item) = work_item {
|
||||
log::debug!("Queueing sync work: {:?}", item);
|
||||
log::debug!("Sync work queued item={item:?}");
|
||||
let _ = work_tx.send(item);
|
||||
}
|
||||
}
|
||||
@@ -511,9 +527,9 @@ impl SubscriptionManager {
|
||||
if let Some(sub) = subscription {
|
||||
sub.start(app_handle).await;
|
||||
self.subscription = Some(sub);
|
||||
log::info!("Sync subscription manager started");
|
||||
log::info!("Sync subscription started");
|
||||
} else {
|
||||
log::debug!("Sync not configured, subscription not started");
|
||||
log::debug!("Sync subscription not started: sync not configured");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -524,7 +540,6 @@ impl SubscriptionManager {
|
||||
sub.stop();
|
||||
}
|
||||
self.subscription = None;
|
||||
log::info!("Sync subscription manager stopped");
|
||||
}
|
||||
|
||||
pub fn is_running(&self) -> bool {
|
||||
|
||||
Reference in new issue
Block a user