/* This file is part of the Notesnook Sync Server project (https://notesnook.com/) Copyright (C) 2023 Streetwriters (Private) Limited This program is free software: you can redistribute it and/or modify it under the terms of the Affero GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Affero GNU General Public License for more details. You should have received a copy of the Affero GNU General Public License along with this program. If not, see . */ using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Amazon.S3; using Amazon.S3.Model; using Microsoft.Extensions.Logging; namespace Notesnook.API.Helpers { /// /// Configuration for S3 failover behavior /// public class S3FailoverConfig { /// /// Maximum number of retry attempts per endpoint /// public int MaxRetries { get; set; } = 3; /// /// Delay between retries in milliseconds /// public int RetryDelayMs { get; set; } = 1000; /// /// Whether to use exponential backoff for retries /// public bool UseExponentialBackoff { get; set; } = true; /// /// Whether to allow failover for write operations (PUT, POST, DELETE). /// Default is false to prevent data consistency issues. /// public bool AllowWriteFailover { get; set; } = false; /// /// List of exception types that should trigger failover /// public HashSet FailoverExceptions { get; set; } = new() { typeof(AmazonS3Exception), typeof(System.Net.Http.HttpRequestException), typeof(System.Net.Sockets.SocketException), typeof(System.Threading.Tasks.TaskCanceledException), typeof(TimeoutException) }; /// /// List of S3 error codes that should trigger failover /// public HashSet FailoverErrorCodes { get; set; } = new() { "ServiceUnavailable", "SlowDown", "InternalError", "RequestTimeout" }; } /// /// Result of a failover operation /// public class S3FailoverResult { public T? Result { get; set; } public bool UsedFailover { get; set; } public int ClientIndex { get; set; } = 0; public int AttemptsUsed { get; set; } public Exception? LastException { get; set; } } /// /// Helper class for S3 operations with automatic failover to multiple endpoints /// public class S3FailoverHelper { private readonly List clients; private readonly S3FailoverConfig config; private readonly ILogger? logger; /// /// Initialize with a list of S3 clients (first is primary, rest are failover endpoints) /// public S3FailoverHelper( IEnumerable clients, S3FailoverConfig? config = null, ILogger? logger = null) { if (clients == null) throw new ArgumentNullException(nameof(clients)); this.clients = new List(clients); if (this.clients.Count == 0) throw new ArgumentException("At least one S3 client is required", nameof(clients)); this.config = config ?? new S3FailoverConfig(); this.logger = logger; } /// /// Initialize with params array of S3 clients /// public S3FailoverHelper( S3FailoverConfig? config = null, ILogger? logger = null, params AmazonS3Client[] clients) { if (clients == null || clients.Length == 0) throw new ArgumentException("At least one S3 client is required", nameof(clients)); this.clients = new List(clients); this.config = config ?? new S3FailoverConfig(); this.logger = logger; } /// /// Execute an S3 operation with automatic failover /// /// The S3 operation to execute /// Name of the operation for logging /// Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default. public async Task ExecuteWithFailoverAsync( Func> operation, string operationName = "S3Operation", bool isWriteOperation = false) { var result = await ExecuteWithFailoverInternalAsync(operation, operationName, isWriteOperation); if (result.Result == null) { throw result.LastException ?? new Exception($"Failed to execute {operationName} on all endpoints"); } return result.Result; } /// /// Execute an S3 operation with automatic failover and return detailed result /// /// The S3 operation to execute /// Name of the operation for logging /// Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default. private async Task> ExecuteWithFailoverInternalAsync( Func> operation, string operationName = "S3Operation", bool isWriteOperation = false) { var result = new S3FailoverResult(); Exception? lastException = null; // Determine max clients to try based on write operation flag var maxClientsToTry = (isWriteOperation && !config.AllowWriteFailover) ? 1 : clients.Count; if (isWriteOperation && !config.AllowWriteFailover && clients.Count > 1) { logger?.LogDebug( "Write operation {Operation} will only use primary endpoint. Failover is disabled for write operations.", operationName); } // Try each client in sequence (first is primary, rest are failovers) for (int i = 0; i < maxClientsToTry; i++) { var client = clients[i]; var clientName = i == 0 ? "primary" : $"failover-{i}"; var isPrimary = i == 0; if (!isPrimary && lastException != null) { logger?.LogWarning(lastException, "Previous S3 endpoint failed for {Operation}. Attempting {ClientName} (endpoint {Index}/{Total}).", operationName, clientName, i + 1, maxClientsToTry); } var (success, value, exception, attempts) = await TryExecuteAsync(client, operation, operationName, clientName); result.AttemptsUsed += attempts; if (success && value != null) { result.Result = value; result.UsedFailover = !isPrimary; result.ClientIndex = i; if (!isPrimary) { logger?.LogInformation( "Successfully failed over to {ClientName} S3 endpoint for {Operation}", clientName, operationName); } return result; } lastException = exception; // If this is not the last client and should retry, log and continue if (i < maxClientsToTry - 1 && ShouldFailover(exception)) { logger?.LogWarning(exception, "Endpoint {ClientName} failed for {Operation}. {Remaining} endpoint(s) remaining.", clientName, operationName, maxClientsToTry - i - 1); } } // All clients failed result.LastException = lastException; logger?.LogError(lastException, "All S3 endpoints failed for {Operation}. Total endpoints tried: {EndpointCount}, Total attempts: {Attempts}", operationName, maxClientsToTry, result.AttemptsUsed); return result; } /// /// Try to execute an operation with retries /// private async Task<(bool success, T? value, Exception? exception, int attempts)> TryExecuteAsync( AmazonS3Client client, Func> operation, string operationName, string endpointName) { Exception? lastException = null; int attempts = 0; for (int retry = 0; retry <= config.MaxRetries; retry++) { attempts++; try { var result = await operation(client); return (true, result, null, attempts); } catch (Exception ex) { lastException = ex; if (retry < config.MaxRetries && ShouldRetry(ex)) { var delay = CalculateRetryDelay(retry); logger?.LogWarning(ex, "Attempt {Attempt}/{MaxAttempts} failed for {Operation} on {Endpoint}. Retrying in {Delay}ms", retry + 1, config.MaxRetries + 1, operationName, endpointName, delay); await Task.Delay(delay); } else { logger?.LogError(ex, "Operation {Operation} failed on {Endpoint} after {Attempts} attempts", operationName, endpointName, attempts); break; } } } return (false, default, lastException, attempts); } /// /// Determine if an exception should trigger a retry /// private bool ShouldRetry(Exception exception) { // Check if exception type is in the retry list var exceptionType = exception.GetType(); if (config.FailoverExceptions.Contains(exceptionType)) { // For S3 exceptions, check error codes if (exception is AmazonS3Exception s3Exception) { return config.FailoverErrorCodes.Contains(s3Exception.ErrorCode); } return true; } return false; } /// /// Determine if an exception should trigger failover to secondary endpoint /// private bool ShouldFailover(Exception? exception) { if (exception == null) return false; return ShouldRetry(exception); } /// /// Calculate delay for retry based on retry attempt number /// private int CalculateRetryDelay(int retryAttempt) { if (!config.UseExponentialBackoff) { return config.RetryDelayMs; } // Exponential backoff: delay * 2^retryAttempt return config.RetryDelayMs * (int)Math.Pow(2, retryAttempt); } /// /// Execute a void operation with automatic failover /// /// The S3 operation to execute /// Name of the operation for logging /// Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default. public async Task ExecuteWithFailoverAsync( Func operation, string operationName = "S3Operation", bool isWriteOperation = false) { await ExecuteWithFailoverAsync(async (client) => { await operation(client); return null; }, operationName, isWriteOperation); } } public static class S3ClientFactory { public static List CreateS3Clients( string serviceUrls, string regions, string accessKeyIds, string secretKeys, bool forcePathStyle = true) { if (string.IsNullOrWhiteSpace(serviceUrls)) return new List(); var urls = SplitAndTrim(serviceUrls); var regionList = SplitAndTrim(regions); var keyIds = SplitAndTrim(accessKeyIds); var secrets = SplitAndTrim(secretKeys); if (urls.Length != regionList.Length || urls.Length != keyIds.Length || urls.Length != secrets.Length) { throw new ArgumentException("All S3 configuration parameters must have the same number of values"); } var clients = new List(); for (int i = 0; i < urls.Length; i++) { var url = urls[i]; if (string.IsNullOrWhiteSpace(url)) continue; // Get corresponding values from other arrays var region = regionList[i]; var keyId = keyIds[i]; var secret = secrets[i]; // Validate that all required values are present if (string.IsNullOrWhiteSpace(region) || string.IsNullOrWhiteSpace(keyId) || string.IsNullOrWhiteSpace(secret)) { System.Diagnostics.Debug.WriteLine( $"Skipping S3 client at index {i}: Missing required values (URL={url}, Region={region}, KeyId={keyId?.Length > 0}, Secret={secret?.Length > 0})"); continue; } try { var config = new AmazonS3Config { ServiceURL = url, AuthenticationRegion = region, ForcePathStyle = forcePathStyle, SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA256, SignatureVersion = "4" }; var client = new AmazonS3Client(keyId, secret, config); clients.Add(client); } catch (Exception ex) { // Log configuration error but continue with other clients System.Diagnostics.Debug.WriteLine($"Failed to create S3 client for URL {url}: {ex.Message}"); } } return clients; } private static string[] SplitAndTrim(string? input) { if (string.IsNullOrWhiteSpace(input)) return Array.Empty(); return input.Split(';', StringSplitOptions.None) .Select(s => s.Trim()) .ToArray(); } } }