refactor: cleanup

This commit is contained in:
zhom
2026-08-27 09:16:25 +04:00
parent 63f673e7d4
commit 15b51f8d2d
31 changed files with 1471 additions and 339 deletions
+39 -13
View File
@@ -1,5 +1,27 @@
use super::types::*;
use reqwest::Client;
use std::time::Duration;
/// How long to wait for a storage host to accept a connection.
///
/// This client had no timeouts at all. A host that neither accepts nor refuses,
/// which is what a dropping firewall or a black-holed address looks like, held
/// every attempt for the operating system's own connect backoff: measured at
/// 21 s on Windows and 134 s on Linux. With `MAX_FILE_RETRIES` and its backoff
/// that is minutes for one file, and a profile of two hundred files reports
/// nothing for most of an hour.
///
/// Matches the pre-flight probe, so a host that fails the check fails a
/// transfer the same way and in the same time.
const CONNECT_TIMEOUT: Duration = Duration::from_secs(8);
/// How long a transfer may make no progress at all.
///
/// Deliberately an inactivity timeout and not a deadline on the whole request.
/// Profile files run to tens of megabytes and a slow link is not a broken one,
/// so a total timeout would start failing syncs that were working. This fires
/// only when nothing arrives for a full minute.
const READ_TIMEOUT: Duration = Duration::from_secs(60);
#[derive(Clone)]
pub struct SyncClient {
@@ -11,7 +33,15 @@ pub struct SyncClient {
impl SyncClient {
pub fn new(base_url: String, token: String) -> Self {
Self {
client: Client::new(),
client: Client::builder()
.connect_timeout(CONNECT_TIMEOUT)
.read_timeout(READ_TIMEOUT)
.build()
// A builder failure here means the TLS backend did not start. The
// default client cannot transfer either, so fall back and let the first
// real request report it, rather than making this constructor fallible
// for a condition no caller can act on.
.unwrap_or_default(),
base_url: base_url.trim_end_matches('/').to_string(),
token,
}
@@ -235,14 +265,13 @@ impl SyncClient {
}
// The storage host here comes from the presigned URL, so on a self-hosted
// server it is whatever the server signed against frequently an address
// server it is whatever the server signed against, frequently an address
// only the server can resolve. `reqwest`'s own Display collapses that to
// "error sending request", which is why this failure used to be
// undiagnosable; report the innermost cause instead.
let response = req
.send()
.await
.map_err(|e| SyncError::NetworkError(super::preflight::transport_reason(&e)))?;
// undiagnosable; report the innermost cause and the host it names.
let response = req.send().await.map_err(|e| {
SyncError::NetworkError(super::preflight::transport_reason_for(presigned_url, &e))
})?;
if !response.status().is_success() {
let status = response.status();
@@ -256,12 +285,9 @@ impl SyncClient {
}
pub async fn download_bytes(&self, presigned_url: &str) -> SyncResult<Vec<u8>> {
let response = self
.client
.get(presigned_url)
.send()
.await
.map_err(|e| SyncError::NetworkError(super::preflight::transport_reason(&e)))?;
let response = self.client.get(presigned_url).send().await.map_err(|e| {
SyncError::NetworkError(super::preflight::transport_reason_for(presigned_url, &e))
})?;
if !response.status().is_success() {
return Err(SyncError::NetworkError(format!(
+108 -3
View File
@@ -146,9 +146,14 @@ fn critical_failure_message(action: &str, failures: &[(String, String)]) -> Stri
/// Transfers go straight to the storage host named in the presigned URL, not
/// through the sync server, so a self-hosted server that signs URLs against an
/// address only it can resolve fails every file here while its own `/health`
/// and `/readyz` stay green. The cause string already carries the host; without
/// this line it still reads as an unexplained network fault, and the setting
/// that fixes it lives on the server, where the user is not looking.
/// and `/readyz` stay green. The cause string names the host, which says which
/// address is wrong; this line says where to change it, because the setting
/// lives on the server, where the user is not looking.
///
/// The host only started appearing in that string when the transfer path moved
/// to `transport_reason_for`. Before that this comment claimed a host that was
/// never there, and every report of this bug arrived with a list of file names
/// and nothing to act on.
fn storage_endpoint_hint(cause: &str) -> String {
let lowered = cause.to_ascii_lowercase();
let is_transport_failure = [
@@ -4321,6 +4326,106 @@ pub async fn rollover_encryption_for_all_entities(
mod tests {
use super::*;
/// The whole of issue 534, at the only place the user ever sees it.
///
/// A self-hosted server signs every presigned URL against the address it uses
/// for storage itself. In the documented compose file that is a Docker
/// service name, so the server is healthy, `/health` and `/readyz` are green,
/// and the client cannot open a single one of the URLs it is handed. The
/// reporters got a list of file names, no host and no setting, and there was
/// nothing in it to act on.
///
/// The message has to carry three things: which files, which host refused
/// them, and which setting fixes it.
#[test]
fn a_transfer_failure_names_the_host_and_the_setting_that_fixes_it() {
// Exactly the text the transfer path now produces. The trailing host comes
// from `preflight::transport_reason_for`, which the two transfer call sites
// in `client.rs` use.
let cause = "connection failed: No such host is known. (os error 11001) \
(storage host minio:9000)";
let failures = vec![
("profile/Default/Cookies".to_string(), cause.to_string()),
("profile/Default/Login Data".to_string(), cause.to_string()),
("profile/Local State".to_string(), cause.to_string()),
];
let message = critical_failure_message("upload", &failures);
assert!(
message.contains("minio:9000"),
"the reader has to learn which host refused the transfer: {message}"
);
assert!(
message.contains("S3_PUBLIC_ENDPOINT"),
"the setting that fixes it lives on the server, so the message has to \
name it: {message}"
);
assert!(
message.contains("profile/Default/Cookies"),
"the affected files still belong in the message: {message}"
);
}
/// The same guarantee, but driven through the real transfer path instead of a
/// hand-written cause string.
///
/// The test above pins the message builder. This one pins the join: that an
/// upload which cannot reach its host actually produces a cause carrying that
/// host. Dropping back to a reason that omits the host, which is how this
/// shipped for months, breaks this test and not the one above.
///
/// No server is involved. `.invalid` never resolves (RFC 2606), so the
/// failure is the real one, offline and deterministic.
#[tokio::test]
async fn an_unreachable_storage_host_survives_the_whole_way_to_the_message() {
let client = SyncClient::new("http://127.0.0.1:1".to_string(), "unused".to_string());
let presigned = "http://donut-storage.invalid:9000/bucket/profiles/p1/Cookies\
?X-Amz-Signature=deadbeef";
let error = client
.upload_bytes(presigned, b"payload", None)
.await
.expect_err("a host that cannot resolve must not report a successful upload");
let message = critical_failure_message(
"upload",
&[("profile/Default/Cookies".to_string(), error.to_string())],
);
assert!(
message.contains("donut-storage.invalid:9000"),
"the host has to survive from the transfer to the message: {message}"
);
assert!(
message.contains("S3_PUBLIC_ENDPOINT"),
"an unreachable storage host has one fix, and it is on the server: {message}"
);
assert!(
!message.contains("X-Amz-Signature"),
"the signature must never reach the message: {message}"
);
}
/// The hint is for a transfer that never connected. A server that answered
/// and refused is a different problem with a different fix, and pointing that
/// user at their storage endpoint would send them the wrong way.
#[test]
fn a_rejected_transfer_is_not_blamed_on_the_storage_endpoint() {
let failures = vec![(
"profile/Default/Cookies".to_string(),
"Upload failed with status 403 Forbidden: SignatureDoesNotMatch".to_string(),
)];
let message = critical_failure_message("upload", &failures);
assert!(message.contains("SignatureDoesNotMatch"), "{message}");
assert!(
!message.contains("S3_PUBLIC_ENDPOINT"),
"a 403 is not an unreachable host: {message}"
);
}
#[test]
fn test_critical_failure_message_carries_the_cause() {
// A self-hosted server that hands out unreachable presigned URLs fails
+93
View File
@@ -31,6 +31,99 @@ pub use scheduler::{get_global_scheduler, set_global_scheduler, SyncScheduler};
pub use subscription::{SubscriptionManager, SyncWorkItem};
pub use types::{SyncError, SyncResult};
/// The live subscription, held so it can be stopped.
///
/// It used to be a local inside whichever task built the pipeline. Dropping a
/// `SubscriptionManager` does not end its work: `SyncSubscription::start`
/// spawns a task holding clones of the running flag and the work sender, so the
/// task outlived the handle and nothing could reach it. Every restart added one
/// more live SSE connection, each with its own poll loop on the server, and
/// disconnecting left an authenticated stream open to a server the user had
/// just removed.
static GLOBAL_SUBSCRIPTION: std::sync::Mutex<Option<SubscriptionManager>> =
std::sync::Mutex::new(None);
/// Held for the whole of `start_pipeline`, so only one pipeline is ever being
/// assembled at a time.
static PIPELINE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
/// Retire the running pipeline, both halves of it.
pub fn stop_pipeline() {
if let Some(scheduler) = get_global_scheduler() {
scheduler.stop();
}
if let Ok(mut guard) = GLOBAL_SUBSCRIPTION.lock() {
if let Some(subscription) = guard.as_mut() {
subscription.stop();
}
*guard = None;
}
}
/// Build and start the sync pipeline. Safe to call again to restart it.
///
/// Startup and `restart_sync_service` each held their own copy of this, and the
/// copies had drifted. The restart copy stopped the old scheduler first and then
/// returned early if the subscription failed to start, so it left the
/// application holding a scheduler whose task had already exited. Everything
/// queued afterwards went into `pending_profiles` and was never drained, and
/// sync was silently dead until the app was restarted. One function cannot
/// drift from itself.
pub async fn start_pipeline(app_handle: tauri::AppHandle) {
// Two restarts arriving together would otherwise interleave: the second
// retires what the first has not published yet, then both start, and one
// scheduler is left ticking with nothing able to reach it. Building the
// pipeline is rare and already awaits the network, so serialising it costs
// nothing worth measuring.
let _building = PIPELINE_LOCK.lock().await;
stop_pipeline();
let mut subscription_manager = SubscriptionManager::new();
let Some(work_rx) = subscription_manager.take_work_receiver() else {
log::error!("Sync pipeline has no work receiver; not starting");
return;
};
// A subscription failure costs live updates from other devices. It does not
// stop this device syncing its own changes on the timer, so carry on. The
// restart path used to give up here, which turned a token hiccup into sync
// being dead until the next launch.
if let Err(e) = subscription_manager.start(app_handle.clone()).await {
log::warn!("Failed to start sync subscription, continuing without live updates: {e}");
}
if let Ok(mut guard) = GLOBAL_SUBSCRIPTION.lock() {
*guard = Some(subscription_manager);
}
let scheduler = std::sync::Arc::new(SyncScheduler::new());
// Published before the loop starts, because the checks below await the
// network and anything queued in the meantime has to land in this scheduler.
// `stop()` marks it cancelled, so a restart arriving during that window still
// retires it and `start` below becomes a no-op.
set_global_scheduler(scheduler.clone());
scheduler.sync_all_enabled_profiles(&app_handle).await;
match SyncEngine::create_from_settings(&app_handle).await {
Ok(engine) => {
if let Err(e) = engine.check_for_missing_synced_profiles(&app_handle).await {
log::warn!("Failed to check for missing profiles: {e}");
}
if let Err(e) = engine.check_for_missing_synced_entities(&app_handle).await {
log::warn!("Failed to check for missing entities: {e}");
}
}
Err(e) => {
log::warn!("Sync not configured, skipping missing profile check: {e}");
}
}
if scheduler.clone().start(app_handle, work_rx).await {
log::info!("Sync scheduler started");
}
}
/// Queue a profile sync if the profile has sync enabled. No-op otherwise.
///
/// Called from profile metadata update paths so a rename / tag edit / proxy
+71
View File
@@ -173,6 +173,38 @@ pub(crate) fn transport_reason(error: &reqwest::Error) -> String {
}
}
/// The same reason, naming the host that would not answer.
///
/// A transfer goes straight to the host inside the presigned URL, and that host
/// is chosen by the server, not by this device. It is therefore the one fact the
/// user has never seen and the only one that points at the fix. Leaving it out
/// is what produced reports of "connection failed" with nothing to act on.
///
/// `reqwest::Error::url()` is empty for connect-stage failures, which are
/// exactly the ones that matter here, so take the host from the URL the caller
/// already holds.
pub(crate) fn transport_reason_for(url: &str, error: &reqwest::Error) -> String {
let reason = transport_reason(error);
match storage_host(url) {
Some(host) => format!("{reason} (storage host {host})"),
None => reason,
}
}
/// Host and port, and nothing else.
///
/// A presigned URL carries the signature and the object key in its query, and
/// this string reaches log files and toasts. Only the authority is safe to
/// repeat, and it is the whole of what the reader needs.
fn storage_host(url: &str) -> Option<String> {
let parsed = url::Url::parse(url).ok()?;
let host = parsed.host_str()?;
match parsed.port() {
Some(port) => Some(format!("{host}:{port}")),
None => Some(host.to_string()),
}
}
/// Pre-flight a sync server before saving it, and before trusting it to sync.
#[tauri::command]
pub async fn check_sync_server_connection(server_url: String) -> Result<SyncServerCheck, String> {
@@ -260,4 +292,43 @@ mod tests {
// The bare reqwest Display is what this exists to avoid.
assert_ne!(error, "error sending request");
}
#[test]
fn a_presigned_url_yields_only_its_authority() {
// The query carries the signature and the key. Neither may reach a log.
let signed = "http://minio:9000/donut/profiles/p1/profile/Default/Cookies\
?X-Amz-Signature=deadbeef&X-Amz-Credential=minioadmin";
assert_eq!(storage_host(signed).as_deref(), Some("minio:9000"));
assert_eq!(
storage_host("https://storage.example.com/bucket/key").as_deref(),
Some("storage.example.com")
);
assert_eq!(storage_host("not a url").as_deref(), None);
}
#[tokio::test]
async fn a_failed_transfer_names_the_host_that_refused_it() {
// The whole point of the message. Issue 534 reporters saw a list of file
// names and a bare "connection failed", and could not tell that the host
// their server had signed into every URL was one only the server could
// resolve.
let url = "http://minio.invalid:9000/donut/profiles/p1/Cookies?X-Amz-Signature=abc";
let error = probe_client()
.put(url)
.body(b"payload".to_vec())
.send()
.await
.expect_err("an unresolvable host must not succeed");
let message = transport_reason_for(url, &error);
assert!(
message.contains("minio.invalid:9000"),
"the failure has to name the storage host, got: {message}"
);
assert!(
!message.contains("X-Amz-Signature"),
"the signature must never reach the message, got: {message}"
);
}
}
+153 -15
View File
@@ -8,7 +8,6 @@ use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::mpsc;
use tokio::sync::Mutex;
use tokio::time::sleep;
static GLOBAL_SCHEDULER: std::sync::Mutex<Option<Arc<SyncScheduler>>> = std::sync::Mutex::new(None);
@@ -22,6 +21,17 @@ pub fn set_global_scheduler(scheduler: Arc<SyncScheduler>) {
}
}
/// What `start` should do, given the flags.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StartDecision {
/// Nothing is running and nothing retired it. Spawn the loop.
Start,
/// A loop is already ticking on this scheduler.
AlreadyRunning,
/// `stop` was called on it, possibly before it ever ran.
Retired,
}
#[derive(Debug, Clone)]
struct ProfileStopTime {
#[allow(dead_code)]
@@ -31,6 +41,15 @@ struct ProfileStopTime {
pub struct SyncScheduler {
running: Arc<AtomicBool>,
/// Set by `stop()` and never cleared. A scheduler is one-shot.
///
/// The pipeline publishes a scheduler before it starts its loop, because work
/// queued during the network checks in between has to land somewhere. That
/// left a window where `stop()` cleared a `running` flag that was still
/// false, so it did nothing, and the scheduler then started anyway and ticked
/// forever with no way to reach it. `running` cannot express "retired before
/// it ever ran", so this does.
cancelled: Arc<AtomicBool>,
pending_profiles: Arc<Mutex<HashMap<String, ProfileStopTime>>>,
pending_proxies: Arc<Mutex<HashSet<String>>>,
pending_groups: Arc<Mutex<HashSet<String>>>,
@@ -52,6 +71,7 @@ impl SyncScheduler {
pub fn new() -> Self {
Self {
running: Arc::new(AtomicBool::new(false)),
cancelled: Arc::new(AtomicBool::new(false)),
pending_profiles: Arc::new(Mutex::new(HashMap::new())),
pending_proxies: Arc::new(Mutex::new(HashSet::new())),
pending_groups: Arc::new(Mutex::new(HashSet::new())),
@@ -68,7 +88,12 @@ impl SyncScheduler {
self.running.load(Ordering::SeqCst)
}
/// Retire this scheduler for good.
///
/// Order matters: mark it cancelled before clearing `running`, so a `start()`
/// racing this call cannot slip between the two and begin ticking.
pub fn stop(&self) {
self.cancelled.store(true, Ordering::SeqCst);
self.running.store(false, Ordering::SeqCst);
}
@@ -334,35 +359,93 @@ impl SyncScheduler {
}
}
/// The decision `start` makes before it spawns anything.
///
/// Split out so it can be tested. `start` needs a `tauri::AppHandle`, which a
/// unit test cannot build, and the retirement rule is the part worth pinning
/// down. The `running` check stays a `swap` so two concurrent starts cannot
/// both win.
fn claim_start_slot(&self) -> StartDecision {
if self.cancelled.load(Ordering::SeqCst) {
return StartDecision::Retired;
}
if self.running.swap(true, Ordering::SeqCst) {
return StartDecision::AlreadyRunning;
}
StartDecision::Start
}
/// Begin ticking. Returns whether a loop was actually started, so the caller
/// can log the truth instead of assuming.
pub async fn start(
self: Arc<Self>,
app_handle: tauri::AppHandle,
mut work_rx: mpsc::UnboundedReceiver<SyncWorkItem>,
) {
if self.running.swap(true, Ordering::SeqCst) {
return;
) -> bool {
match self.claim_start_slot() {
StartDecision::Retired => {
// Retired while the pipeline was still assembling it. Starting now
// would leave a task nothing can stop, because the handle in the global
// has already been replaced.
log::info!("Sync scheduler was retired before it started; not starting it");
return false;
}
StartDecision::AlreadyRunning => {
log::warn!("Sync scheduler is already running; ignoring the second start");
return false;
}
StartDecision::Start => {}
}
let scheduler = self.clone();
let app_handle_clone = app_handle.clone();
tokio::spawn(async move {
// A fresh `sleep` inside the `select!` restarts from zero on every
// iteration, so a steady stream of work items kept resetting it and
// `process_pending` never ran: queued profiles sat there for as long as
// the stream lasted. An interval keeps its own schedule regardless of how
// often the other arm fires. `Delay` rather than `Burst` so a slow
// `process_pending` does not come back to a pile of missed ticks and run
// itself back to back.
let mut ticker = tokio::time::interval(Duration::from_millis(2000));
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
// The first tick of an interval resolves immediately. The old shape
// always waited 2000 ms before its first pass, so consume it here and
// keep that behaviour.
ticker.tick().await;
// Once the senders are gone `recv()` resolves instantly and forever, so
// the arm has to be disabled or the loop spins hot on a dead channel.
let mut work_channel_open = true;
while scheduler.running.load(Ordering::SeqCst) {
tokio::select! {
Some(work_item) = work_rx.recv() => {
match work_item {
SyncWorkItem::Profile(id) => scheduler.queue_profile_sync(id).await,
SyncWorkItem::Proxy(id) => scheduler.queue_proxy_sync(id).await,
SyncWorkItem::Group(id) => scheduler.queue_group_sync(id).await,
SyncWorkItem::Vpn(id) => scheduler.queue_vpn_sync(id).await,
SyncWorkItem::Extension(id) => scheduler.queue_extension_sync(id).await,
SyncWorkItem::ExtensionGroup(id) => scheduler.queue_extension_group_sync(id).await,
SyncWorkItem::Tombstone(entity_type, entity_id) => {
scheduler.queue_tombstone(entity_type, entity_id).await
received = work_rx.recv(), if work_channel_open => {
match received {
Some(work_item) => match work_item {
SyncWorkItem::Profile(id) => scheduler.queue_profile_sync(id).await,
SyncWorkItem::Proxy(id) => scheduler.queue_proxy_sync(id).await,
SyncWorkItem::Group(id) => scheduler.queue_group_sync(id).await,
SyncWorkItem::Vpn(id) => scheduler.queue_vpn_sync(id).await,
SyncWorkItem::Extension(id) => scheduler.queue_extension_sync(id).await,
SyncWorkItem::ExtensionGroup(id) => scheduler.queue_extension_group_sync(id).await,
SyncWorkItem::Tombstone(entity_type, entity_id) => {
scheduler.queue_tombstone(entity_type, entity_id).await
}
},
None => {
// The subscription is gone, so no more live updates from other
// devices. Local changes and the timer still work, so keep
// ticking rather than ending the scheduler.
log::warn!(
"Sync work channel closed; continuing on the timer without live updates"
);
work_channel_open = false;
}
}
}
_ = sleep(Duration::from_millis(2000)) => {
_ = ticker.tick() => {
scheduler.process_pending(&app_handle_clone).await;
}
}
@@ -370,6 +453,8 @@ impl SyncScheduler {
log::info!("Sync scheduler stopped");
});
true
}
async fn process_pending(&self, app_handle: &tauri::AppHandle) {
@@ -853,3 +938,56 @@ impl SyncScheduler {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_fresh_scheduler_starts_once() {
let scheduler = SyncScheduler::new();
assert_eq!(scheduler.claim_start_slot(), StartDecision::Start);
assert!(scheduler.is_running());
assert_eq!(
scheduler.claim_start_slot(),
StartDecision::AlreadyRunning,
"a second start must not spawn a second loop on the same scheduler"
);
}
#[test]
fn a_scheduler_retired_before_it_ran_never_starts() {
// The pipeline publishes a scheduler, then awaits two network checks, then
// starts the loop. A restart landing in that window calls `stop()` on a
// scheduler that has not started yet. `running` was already false, so the
// old `stop()` did nothing at all, the loop started afterwards, and it
// ticked forever with the global already pointing elsewhere.
let scheduler = SyncScheduler::new();
assert!(!scheduler.is_running());
scheduler.stop();
assert_eq!(
scheduler.claim_start_slot(),
StartDecision::Retired,
"a scheduler stopped before starting must stay stopped"
);
assert!(
!scheduler.is_running(),
"refusing to start must not leave the running flag set"
);
}
#[test]
fn stopping_a_running_scheduler_retires_it_for_good() {
let scheduler = SyncScheduler::new();
assert_eq!(scheduler.claim_start_slot(), StartDecision::Start);
scheduler.stop();
assert!(!scheduler.is_running());
// A scheduler is one-shot. Restarting the pipeline builds a new one, so a
// retired instance coming back to life could only ever be a duplicate.
assert_eq!(scheduler.claim_start_slot(), StartDecision::Retired);
}
}