s3: re-enable uploads with rate limiting

encryption verifier is used for verifying the encryption key during password resets. It can be any encrypted item.

Notesnook.API/Controllers/S3Controller.cs

@@ -20,10 +20,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using System;
 using System.Net.Http;
 using System.Security.Claims;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using Notesnook.API.Helpers;
@@ -32,7 +34,6 @@ using Notesnook.API.Models;
 using Streetwriters.Common;
 using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
 {
@@ -40,9 +41,10 @@ namespace Notesnook.API.Controllers
     [Route("s3")]
     [ResponseCache(NoStore = true, Location = ResponseCacheLocation.None)]
     [Authorize("Sync")]
-    public class S3Controller(IS3Service s3Service, ISyncItemsRepositoryAccessor repositories, WampServiceAccessor serviceAccessor, ILogger<S3Controller> logger) : ControllerBase
+    public class S3Controller(IS3Service s3Service, ISyncItemsRepositoryAccessor repositories, WampServiceAccessor serviceAccessor, IHttpClientFactory httpClientFactory, ILogger<S3Controller> logger) : ControllerBase
     {
         [HttpPut]
+        [EnableRateLimiting("s3-direct")]
         public async Task<IActionResult> Upload([FromQuery] string name)
         {
             try
@@ -62,6 +64,10 @@ namespace Notesnook.API.Controllers
 
                 return Ok();
             }
+            catch (Exception ex) when (S3TransientErrorClassifier.IsTransient(ex))
+            {
+                return StorageUnavailable(ex);
+            }
             catch (Exception ex)
             {
                 logger.LogError(ex, "Error uploading attachment for user.");
@@ -71,7 +77,9 @@ namespace Notesnook.API.Controllers
 
         private async Task UploadFileWithChecksAsync(string userId, string name, long fileSize)
         {
-            var userSettings = await repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            var userSettings = await repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId)
+                ?? throw new Exception("User settings not found.");
+            var observedStorageLimit = userSettings.StorageLimit;
 
             var subscription = await serviceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
 
@@ -84,35 +92,51 @@ namespace Notesnook.API.Controllers
 
             var uploadedFileSize = await UploadFileAsync(userId, name, fileSize);
 
-            userSettings.StorageLimit.Value += uploadedFileSize;
-            await repositories.UsersSettings.Collection.UpdateOneAsync(
-                Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
-                Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
-            );
-
-            // extra check in case user sets wrong ContentLength in the HTTP header
-            if (uploadedFileSize != fileSize && StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
+            try
             {
-                await s3Service.DeleteObjectAsync(userId, name);
-                throw new Exception("Storage limit exceeded.");
+                await s3Service.IncrementStorageUsageAsync(userId, uploadedFileSize, observedStorageLimit);
             }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to account for attachment usage for user {UserId}.", userId);
+            }
+
         }
 
         private async Task<long> UploadFileAsync(string userId, string name, long fileSize)
         {
             var url = await s3Service.GetInternalUploadObjectUrlAsync(userId, name) ?? throw new Exception("Could not create signed url.");
 
-            var httpClient = new HttpClient();
+            var httpClient = httpClientFactory.CreateClient("S3Upload");
             var content = new StreamContent(HttpContext.Request.BodyReader.AsStream());
             content.Headers.ContentLength = fileSize;
-            var response = await httpClient.SendRequestAsync<Response>(url, null, HttpMethod.Put, content);
-            if (!response.Success) throw new Exception(response.Content != null ? await response.Content.ReadAsStringAsync() : "Could not upload file.");
+            using var request = new HttpRequestMessage(HttpMethod.Put, url)
+            {
+                Content = content
+            };
+            using var uploadTimeout = CancellationTokenSource.CreateLinkedTokenSource(HttpContext.RequestAborted);
+            uploadTimeout.CancelAfter(TimeSpan.FromMinutes(15 + (15 * fileSize / (1024d * 1024d * 1024d))));
+            using var response = await httpClient.SendAsync(
+                request,
+                HttpCompletionOption.ResponseHeadersRead,
+                uploadTimeout.Token);
+            if (!response.IsSuccessStatusCode)
+            {
+                var statusCode = (int)response.StatusCode;
+                if (S3TransientErrorClassifier.IsTransient(statusCode))
+                    throw new S3StorageUnavailableException("Attachment storage is temporarily unavailable.");
 
-            return await s3Service.GetObjectSizeAsync(userId, name);
+                throw new Exception(await response.Content.ReadAsStringAsync(uploadTimeout.Token));
+            }
+
+            // The PUT response confirms this request. A follow-up HEAD can lag or
+            // be throttled, so direct accounting uses the accepted request bytes.
+            return fileSize;
         }
 
 
         [HttpGet("multipart")]
+        [EnableRateLimiting("s3-multipart-control")]
         public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string? uploadId)
         {
             var userId = this.User.GetUserId();
@@ -121,6 +145,10 @@ namespace Notesnook.API.Controllers
                 var meta = await s3Service.StartMultipartUploadAsync(userId, name, parts, uploadId);
                 return Ok(meta);
             }
+            catch (Exception ex) when (S3TransientErrorClassifier.IsTransient(ex))
+            {
+                return StorageUnavailable(ex);
+            }
             catch (Exception ex)
             {
                 logger.LogError(ex, "Error starting multipart upload for user.");
@@ -137,6 +165,10 @@ namespace Notesnook.API.Controllers
                 await s3Service.AbortMultipartUploadAsync(userId, name, uploadId);
                 return Ok();
             }
+            catch (Exception ex) when (S3TransientErrorClassifier.IsTransient(ex))
+            {
+                return StorageUnavailable(ex);
+            }
             catch (Exception ex)
             {
                 logger.LogError(ex, "Error aborting multipart upload for user.");
@@ -145,6 +177,7 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPost("multipart")]
+        [EnableRateLimiting("s3-multipart-control")]
         public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequestWrapper uploadRequestWrapper)
         {
             var userId = this.User.GetUserId();
@@ -153,6 +186,10 @@ namespace Notesnook.API.Controllers
                 await s3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
                 return Ok();
             }
+            catch (Exception ex) when (S3TransientErrorClassifier.IsTransient(ex))
+            {
+                return StorageUnavailable(ex);
+            }
             catch (Exception ex)
             {
                 logger.LogError(ex, "Error completing multipart upload for user.");
@@ -160,6 +197,12 @@ namespace Notesnook.API.Controllers
             }
         }
 
+        private IActionResult StorageUnavailable(Exception exception)
+        {
+            logger.LogWarning(exception, "Attachment storage is temporarily unavailable.");
+            return StatusCode(503, new { error = "Attachment storage is temporarily unavailable. Please try again later." });
+        }
+
         [HttpGet]
         public async Task<IActionResult> Download([FromQuery] string name)
         {

Notesnook.API/Controllers/UsersController.cs

@@ -93,39 +93,56 @@ namespace Notesnook.API.Controllers
         [HttpPatch("password/{type}")]
         public async Task<IActionResult> ChangePassword([FromRoute] string type, [FromBody] ChangePasswordForm form)
         {
-            return BadRequest(new { error = "Password change is currently disabled." });
-            // var userId = User.GetUserId();
-            // var clientId = User.FindFirstValue("client_id");
-            // var jti = User.FindFirstValue("jti");
-            // var isPasswordReset = type == "reset";
-            // try
-            // {
-            //     var result = isPasswordReset ? await serviceAccessor.UserAccountService.ResetPasswordAsync(userId, form.NewPassword) : await serviceAccessor.UserAccountService.ChangePasswordAsync(userId, form.OldPassword, form.NewPassword);
-            //     if (!result)
-            //         return BadRequest("Failed to change password.");
+            var userId = User.GetUserId();
+            var clientId = User.FindFirstValue("client_id");
+            var jti = User.FindFirstValue("jti");
+            var isPasswordReset = type == "reset";
+            try
+            {
+                var result = isPasswordReset ? await serviceAccessor.UserAccountService.ResetPasswordAsync(userId, form.NewPassword) : await serviceAccessor.UserAccountService.ChangePasswordAsync(userId, form.OldPassword, form.NewPassword);
+                if (!result)
+                    return BadRequest("Failed to change password.");
 
-            //     await UserService.SetUserKeysAsync(userId, form.UserKeys);
+                await UserService.SetUserKeysAsync(userId, form.UserKeys);
 
-            //     await serviceAccessor.UserAccountService.ClearSessionsAsync(userId, clientId, all: false, jti, null);
+                await serviceAccessor.UserAccountService.ClearSessionsAsync(userId, clientId, all: false, jti, null);
 
-            //     await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
-            //     {
-            //         UserId = userId,
-            //         OriginTokenId = jti,
-            //         Message = new Message
-            //         {
-            //             Type = "logout",
-            //             Data = JsonSerializer.Serialize(new { reason = "Password changed." })
-            //         }
-            //     });
+                await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+                {
+                    UserId = userId,
+                    OriginTokenId = jti,
+                    Message = new Message
+                    {
+                        Type = "logout",
+                        Data = JsonSerializer.Serialize(new { reason = "Password changed." })
+                    }
+                });
 
-            //     return Ok();
-            // }
-            // catch (Exception ex)
-            // {
-            //     logger.LogError(ex, "Failed to change password");
-            //     return BadRequest(new { error = ex.Message });
-            // }
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to change password");
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+
+
+        [HttpGet("verifier")]
+        public async Task<IActionResult> GetEncryptionVerifier()
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                var response = await UserService.GetEncryptionVerifier(userId);
+                if (response == null) return NotFound();
+                return Ok(response);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to get encryption verifier for user id: {UserId}", userId);
+                return BadRequest(new { error = ex.Message });
+            }
         }
 
         [HttpPost("reset")]

Notesnook.API/Helpers/S3FailoverHelper.cs

@@ -20,6 +20,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
+using System.Net.Sockets;
 using System.Threading.Tasks;
 using Amazon.S3;
 using Amazon.S3.Model;
@@ -47,12 +49,6 @@ namespace Notesnook.API.Helpers
         /// </summary>
         public bool UseExponentialBackoff { get; set; } = true;
 
-        /// <summary>
-        /// Whether to allow failover for write operations (PUT, POST, DELETE).
-        /// Default is false to prevent data consistency issues.
-        /// </summary>
-        public bool AllowWriteFailover { get; set; } = false;
-
         /// <summary>
         /// List of exception types that should trigger failover
         /// </summary>
@@ -77,6 +73,39 @@ namespace Notesnook.API.Helpers
         };
     }
 
+    public sealed class S3StorageUnavailableException : Exception
+    {
+        public S3StorageUnavailableException(string message, Exception? innerException = null)
+            : base(message, innerException)
+        {
+        }
+    }
+
+    public static class S3TransientErrorClassifier
+    {
+        public static bool IsTransient(int statusCode)
+        {
+            return statusCode == 408 || statusCode == 429 || statusCode >= 500;
+        }
+
+        public static bool IsTransient(Exception exception)
+        {
+            if (exception is S3StorageUnavailableException)
+                return true;
+
+            if (exception is AmazonS3Exception s3Exception)
+            {
+                return IsTransient((int)s3Exception.StatusCode)
+                    || s3Exception.ErrorCode is "SlowDown" or "ServiceUnavailable" or "InternalError" or "RequestTimeout";
+            }
+
+            if (exception is HttpRequestException or SocketException or TimeoutException or TaskCanceledException)
+                return true;
+
+            return exception.InnerException != null && IsTransient(exception.InnerException);
+        }
+    }
+
     /// <summary>
     /// Result of a failover operation
     /// </summary>
@@ -161,10 +190,11 @@ namespace Notesnook.API.Helpers
             var result = new S3FailoverResult<T>();
             Exception? lastException = null;
 
-            // Determine max clients to try based on write operation flag
-            var maxClientsToTry = (isWriteOperation && !config.AllowWriteFailover) ? 1 : clients.Count;
+            // Writes stay on the provider that owns the object or multipart upload.
+            // They get one application attempt and never fail over.
+            var maxClientsToTry = isWriteOperation ? 1 : clients.Count;
 
-            if (isWriteOperation && !config.AllowWriteFailover && clients.Count > 1)
+            if (isWriteOperation && clients.Count > 1)
             {
                 logger?.LogDebug(
                     "Write operation {Operation} will only use primary endpoint. Failover is disabled for write operations.",
@@ -185,7 +215,12 @@ namespace Notesnook.API.Helpers
                         operationName, clientName, i + 1, maxClientsToTry);
                 }
 
-                var (success, value, exception, attempts) = await TryExecuteAsync(client, operation, operationName, clientName);
+                var (success, value, exception, attempts) = await TryExecuteAsync(
+                    client,
+                    operation,
+                    operationName,
+                    clientName,
+                    isWriteOperation ? 0 : config.MaxRetries);
                 result.AttemptsUsed += attempts;
 
                 if (success && value != null)
@@ -222,19 +257,22 @@ namespace Notesnook.API.Helpers
                 operationName, maxClientsToTry, result.AttemptsUsed);
 
             return result;
-        }        /// <summary>
-                 /// Try to execute an operation with retries
-                 /// </summary>
+        }
+
+        /// <summary>
+        /// Try to execute an operation with retries
+        /// </summary>
         private async Task<(bool success, T? value, Exception? exception, int attempts)> TryExecuteAsync<T>(
             AmazonS3Client client,
             Func<AmazonS3Client, Task<T>> operation,
             string operationName,
-            string endpointName)
+            string endpointName,
+            int maxRetries)
         {
             Exception? lastException = null;
             int attempts = 0;
 
-            for (int retry = 0; retry <= config.MaxRetries; retry++)
+            for (int retry = 0; retry <= maxRetries; retry++)
             {
                 attempts++;
                 try
@@ -246,12 +284,12 @@ namespace Notesnook.API.Helpers
                 {
                     lastException = ex;
 
-                    if (retry < config.MaxRetries && ShouldRetry(ex))
+                    if (retry < maxRetries && ShouldRetry(ex))
                     {
                         var delay = CalculateRetryDelay(retry);
                         logger?.LogWarning(ex,
                             "Attempt {Attempt}/{MaxAttempts} failed for {Operation} on {Endpoint}. Retrying in {Delay}ms",
-                            retry + 1, config.MaxRetries + 1, operationName, endpointName, delay);
+                            retry + 1, maxRetries + 1, operationName, endpointName, delay);
 
                         await Task.Delay(delay);
                     }
@@ -322,10 +360,10 @@ namespace Notesnook.API.Helpers
             string operationName = "S3Operation",
             bool isWriteOperation = false)
         {
-            await ExecuteWithFailoverAsync<object?>(async (client) =>
+            await ExecuteWithFailoverAsync(async (client) =>
             {
                 await operation(client);
-                return null;
+                return true;
             }, operationName, isWriteOperation);
         }
     }
@@ -384,6 +422,7 @@ namespace Notesnook.API.Helpers
                         ServiceURL = url,
                         AuthenticationRegion = region,
                         ForcePathStyle = forcePathStyle,
+                        MaxErrorRetry = 0,
                         SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA256,
                         SignatureVersion = "4"
                     };

Notesnook.API/Interfaces/IS3Service.cs

@@ -35,5 +35,6 @@ namespace Notesnook.API.Interfaces
         Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string? uploadId = null);
         Task AbortMultipartUploadAsync(string userId, string name, string uploadId);
         Task CompleteMultipartUploadAsync(string userId, CompleteMultipartUploadRequest uploadRequest);
+        Task IncrementStorageUsageAsync(string userId, long fileSize, Limit? observedStorageLimit);
     }
 }

Notesnook.API/Interfaces/IUserService.cs

@@ -31,6 +31,7 @@ namespace Notesnook.API.Interfaces
         Task DeleteUserAsync(string userId, string? jti, string password);
         Task<bool> ResetUserAsync(string userId, bool removeAttachments);
         Task<UserResponse> GetUserAsync(string userId);
+        Task<EncryptedData?> GetEncryptionVerifier(string userId);
         Task SetUserKeysAsync(string userId, UserKeys keys);
     }
 }

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -36,6 +36,7 @@ using Streetwriters.Common;
 using Streetwriters.Data.DbContexts;
 using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
+using AspNetCore.Identity.Mongo.Mongo;
 
 namespace Notesnook.API.Repositories
 {

Notesnook.API/Services/S3Service.cs

@@ -60,9 +60,11 @@ namespace Notesnook.API.Services
         // That is why we create 2 separate S3 clients. One for internal traffic and one for external.
         private readonly S3FailoverHelper S3InternalClient;
         private readonly HttpClient httpClient = new();
+        private readonly ILogger<S3Service> logger;
 
         public S3Service(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, WampServiceAccessor wampServiceAccessor, ILogger<S3Service> logger)
         {
+            this.logger = logger;
             Repositories = syncItemsRepositoryAccessor;
             ServiceAccessor = wampServiceAccessor;
             S3Client = new S3FailoverHelper(
@@ -277,12 +279,32 @@ namespace Notesnook.API.Services
 
         private async Task<long> GetMultipartUploadSizeAsync(string userId, string key, string uploadId)
         {
-            var objectName = GetFullObjectName(userId, key);
-            var parts = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.ListPartsAsync(INTERNAL_BUCKET_NAME, objectName, uploadId), operationName: "ListParts");
+            var objectName = GetFullObjectName(userId, key) ?? throw new Exception("Invalid object name.");
             long totalSize = 0;
-            foreach (var part in parts.Parts)
+            var marker = 0;
+            while (true)
             {
-                totalSize += part.Size;
+                var request = new ListPartsRequest
+                {
+                    BucketName = INTERNAL_BUCKET_NAME,
+                    Key = objectName,
+                    UploadId = uploadId,
+                    PartNumberMarker = marker.ToString()
+                };
+                var parts = await S3InternalClient.ExecuteWithFailoverAsync(
+                    (client) => client.ListPartsAsync(request),
+                    operationName: "ListParts",
+                    isWriteOperation: true);
+                foreach (var part in parts.Parts)
+                {
+                    totalSize = checked(totalSize + part.Size);
+                }
+
+                if (!parts.IsTruncated)
+                    break;
+                if (parts.NextPartNumberMarker <= marker)
+                    throw new Exception("S3 returned an invalid parts page.");
+                marker = parts.NextPartNumberMarker;
             }
             return totalSize;
         }
@@ -298,21 +320,22 @@ namespace Notesnook.API.Services
                 await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                 throw new Exception("User settings not found.");
             }
-            userSettings.StorageLimit ??= StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            var observedStorageLimit = userSettings.StorageLimit;
+            userSettings.StorageLimit = StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            long fileSize = 0;
 
             if (!Constants.IS_SELF_HOSTED)
             {
                 var subscription = await ServiceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
 
-                long fileSize = await GetMultipartUploadSizeAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
+                fileSize = await GetMultipartUploadSizeAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                 if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
                 {
                     await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                     throw new Exception("Max file size exceeded.");
                 }
 
-                userSettings.StorageLimit.Value += fileSize;
-                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
+                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value + fileSize))
                 {
                     await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                     throw new Exception("Storage limit reached.");
@@ -326,14 +349,75 @@ namespace Notesnook.API.Services
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-                await Repositories.UsersSettings.Collection.UpdateOneAsync(
-                    Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
-                    Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
-                );
+                try
+                {
+                    await IncrementStorageUsageAsync(userId, fileSize, observedStorageLimit);
+                }
+                catch (Exception ex)
+                {
+                    // The object is already committed. Accounting is best effort so
+                    // a Mongo failure cannot turn a successful upload into a retry.
+                    logger.LogError(ex, "Failed to account for multipart attachment usage for user {UserId}.", userId);
+                }
             }
         }
 
+        public async Task IncrementStorageUsageAsync(string userId, long fileSize, Limit? observedStorageLimit)
+        {
+            var collection = Repositories.UsersSettings.Collection;
+            var userFilter = Builders<UserSettings>.Filter.Eq(u => u.UserId, userId);
+
+            for (var attempt = 0; attempt < 3; attempt++)
+            {
+                var now = DateTimeOffset.UtcNow;
+                var monthStart = new DateTimeOffset(now.Year, now.Month, 1, 0, 0, 0, TimeSpan.Zero);
+                var nextMonthStart = monthStart.AddMonths(1);
+                var nowMilliseconds = now.ToUnixTimeMilliseconds();
+                var monthStartMilliseconds = monthStart.ToUnixTimeMilliseconds();
+                var nextMonthStartMilliseconds = nextMonthStart.ToUnixTimeMilliseconds();
+                var currentStorageLimit = observedStorageLimit;
+
+                if (attempt > 0)
+                {
+                    currentStorageLimit = (await collection.Find(userFilter).FirstOrDefaultAsync())?.StorageLimit;
+                }
+
+                var isCurrentMonth = currentStorageLimit != null
+                    && currentStorageLimit.UpdatedAt >= monthStartMilliseconds
+                    && currentStorageLimit.UpdatedAt < nextMonthStartMilliseconds;
+
+                if (!isCurrentMonth)
+                {
+                    var observedFilter = currentStorageLimit == null
+                        ? Builders<UserSettings>.Filter.Eq(u => u.StorageLimit, null)
+                        : Builders<UserSettings>.Filter.Eq("StorageLimit.UpdatedAt", currentStorageLimit.UpdatedAt);
+
+                    var resetResult = await collection.UpdateOneAsync(
+                        Builders<UserSettings>.Filter.And(userFilter, observedFilter),
+                        Builders<UserSettings>.Update.Set(
+                            "StorageLimit",
+                            new Limit { Value = 0, UpdatedAt = nowMilliseconds }));
+
+                    if (resetResult.MatchedCount == 0)
+                        continue;
+                }
+
+                var result = await collection.UpdateOneAsync(
+                    Builders<UserSettings>.Filter.And(
+                        userFilter,
+                        Builders<UserSettings>.Filter.Gte("StorageLimit.UpdatedAt", monthStartMilliseconds),
+                        Builders<UserSettings>.Filter.Lt("StorageLimit.UpdatedAt", nextMonthStartMilliseconds)),
+                    Builders<UserSettings>.Update.Combine(
+                        Builders<UserSettings>.Update.Inc("StorageLimit.Value", fileSize),
+                        Builders<UserSettings>.Update.Set("StorageLimit.UpdatedAt", nowMilliseconds)));
+
+                if (result.MatchedCount > 0)
+                    return;
+            }
+
+            throw new Exception("Storage usage counter was not available after rollover retries.");
+        }
+
         private async Task<string?> GetPresignedURLAsync(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
             var objectName = GetFullObjectName(userId, name);
@@ -393,4 +477,4 @@ namespace Notesnook.API.Services
             return ((int)statusCode >= 200) && ((int)statusCode <= 299);
         }
     }
-}
+}

Notesnook.API/Services/UserService.cs

@@ -30,6 +30,7 @@ using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
+using Notesnook.API.Repositories;
 using Streetwriters.Common;
 using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Enums;
@@ -192,6 +193,26 @@ namespace Notesnook.API.Services
             await Repositories.UsersSettings.UpdateAsync(userSettings.Id, userSettings);
         }
 
+        public async Task<EncryptedData?> GetEncryptionVerifier(string userId)
+        {
+            SyncItemsRepository[] repositories = [Repositories.Notes, Repositories.Notebooks, Repositories.Shortcuts, Repositories.Contents, Repositories.Settings, Repositories.LegacySettings, Repositories.Attachments, Repositories.Reminders, Repositories.Relations, Repositories.Colors, Repositories.Tags, Repositories.Vaults, Repositories.InboxItemsHistory];
+            foreach (var repo in repositories)
+            {
+                var item = await repo.FindOneAsync((s) => s.UserId == userId && (s.KeyVersion == null || s.KeyVersion == 0));
+                if (item != null)
+                {
+                    return new EncryptedData
+                    {
+                        Cipher = item.Cipher,
+                        IV = item.IV,
+                        Salt = "",
+                        Length = item.Length
+                    };
+                }
+            }
+            return null;
+        }
+
         public async Task DeleteUserAsync(string userId)
         {
             logger.LogInformation("Deleting user {UserId}", userId);

Notesnook.API/Startup.cs

@@ -23,6 +23,7 @@ using System.IO.Compression;
 using System.Security.Claims;
 using System.Text;
 using System.Text.Encodings.Web;
+using System.Threading;
 using System.Threading.Tasks;
 using Amazon.Runtime;
 using StackExchange.Redis;
@@ -35,6 +36,7 @@ using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Connections;
 using Microsoft.AspNetCore.HttpOverrides;
+using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.AspNetCore.ResponseCompression;
 using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Caching.Memory;
@@ -68,6 +70,7 @@ using Streetwriters.Data;
 using Streetwriters.Data.DbContexts;
 using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
+using System.Threading.RateLimiting;
 
 namespace Notesnook.API
 {
@@ -125,6 +128,27 @@ namespace Notesnook.API
                 options.DefaultPolicy = options.GetPolicy("Notesnook") ?? throw new Exception("Notesnook policy not found");
             }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>();
 
+            services.AddRateLimiter(options =>
+            {
+                options.RejectionStatusCode = StatusCodes.Status503ServiceUnavailable;
+                options.AddPolicy("s3-direct", _ => RateLimitPartition.GetConcurrencyLimiter(
+                    "s3-direct",
+                    _ => new ConcurrencyLimiterOptions
+                    {
+                        PermitLimit = ReadRateLimit("S3_DIRECT_UPLOAD_CONCURRENCY", 128),
+                        QueueLimit = 0,
+                        QueueProcessingOrder = QueueProcessingOrder.OldestFirst
+                    }));
+                options.AddPolicy("s3-multipart-control", _ => RateLimitPartition.GetConcurrencyLimiter(
+                    "s3-multipart-control",
+                    _ => new ConcurrencyLimiterOptions
+                    {
+                        PermitLimit = ReadRateLimit("S3_MULTIPART_CONTROL_CONCURRENCY", 32),
+                        QueueLimit = 0,
+                        QueueProcessingOrder = QueueProcessingOrder.OldestFirst
+                    }));
+            });
+
             services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
             .AddOAuth2Introspection("introspection", options =>
             {
@@ -204,6 +228,10 @@ namespace Notesnook.API
             services.AddScoped<SyncDeviceService>();
             services.AddScoped<IUserService, UserService>();
             services.AddScoped<IS3Service, S3Service>();
+            services.AddHttpClient("S3Upload", client =>
+            {
+                client.Timeout = Timeout.InfiniteTimeSpan;
+            });
             services.AddScoped<IURLAnalyzer, URLAnalyzer>();
 
             services.AddWampServiceAccessor(Servers.NotesnookAPI);
@@ -314,6 +342,7 @@ namespace Notesnook.API
 
             app.UseAuthentication();
             app.UseAuthorization();
+            app.UseRateLimiter();
 
             app.UseEndpoints(endpoints =>
             {
@@ -326,6 +355,13 @@ namespace Notesnook.API
                 });
             });
         }
+
+        private int ReadRateLimit(string name, int defaultValue)
+        {
+            return int.TryParse(Configuration[name], out var value) && value > 0
+                ? value
+                : defaultValue;
+        }
     }
 
     public static class ServiceCollectionMongoCollectionExtensions

Streetwriters.Common/Constants.cs

@@ -78,7 +78,7 @@ namespace Streetwriters.Common
         public static string? SUBSCRIPTIONS_SERVER_HOST => ReadSecret("SUBSCRIPTIONS_SERVER_HOST");
         public static string? SUBSCRIPTIONS_CERT_PATH => ReadSecret("SUBSCRIPTIONS_CERT_PATH");
         public static string? SUBSCRIPTIONS_CERT_KEY_PATH => ReadSecret("SUBSCRIPTIONS_CERT_KEY_PATH");
-        public static string[] NOTESNOOK_CORS_ORIGINS => ReadSecret("NOTESNOOK_CORS")?.Split(",") ?? [];
+        public static string[] NOTESNOOK_CORS_ORIGINS => ReadSecret("NOTESNOOK_CORS_ORIGINS")?.Split(",") ?? [];
         public static string? SIGNALR_REDIS_CONNECTION_STRING => ReadSecret("SIGNALR_REDIS_CONNECTION_STRING");
         public static string MONOGRAPH_PUBLIC_URL => ReadSecret("MONOGRAPH_PUBLIC_URL") ?? "https://monogr.ph";
 

Streetwriters.Common/Services/EmailSender.cs

@@ -14,9 +14,8 @@ using Streetwriters.Common.Models;
 
 namespace Streetwriters.Common.Services
 {
-    public class EmailSender : IEmailSender, IAsyncDisposable
+    public class EmailSender : IEmailSender
     {
-        private readonly SmtpClient mailClient = new();
         private readonly ILogger<EmailSender> logger;
 
         public EmailSender(ILogger<EmailSender> logger)
@@ -32,27 +31,25 @@ namespace Streetwriters.Common.Services
             Dictionary<string, byte[]>? attachments = null
         )
         {
-            if (!mailClient.IsConnected)
+            using var mailClient = new SmtpClient();
+
+            if (int.TryParse(Common.Constants.SMTP_PORT, out int port))
             {
-                if (int.TryParse(Common.Constants.SMTP_PORT, out int port))
-                {
-                    await mailClient.ConnectAsync(
-                        Common.Constants.SMTP_HOST,
-                        port,
-                        MailKit.Security.SecureSocketOptions.Auto
-                    );
-                }
-                else
-                {
-                    throw new InvalidDataException("SMTP_PORT is not a valid integer value.");
-                }
+                await mailClient.ConnectAsync(
+                    Common.Constants.SMTP_HOST,
+                    port,
+                    MailKit.Security.SecureSocketOptions.Auto
+                );
+            }
+            else
+            {
+                throw new InvalidDataException("SMTP_PORT is not a valid integer value.");
             }
 
-            if (!mailClient.IsAuthenticated)
-                await mailClient.AuthenticateAsync(
-                    Common.Constants.SMTP_USERNAME,
-                    Common.Constants.SMTP_PASSWORD
-                );
+            await mailClient.AuthenticateAsync(
+                Common.Constants.SMTP_USERNAME,
+                Common.Constants.SMTP_PASSWORD
+            );
 
             var message = new MimeMessage();
             message.From.Add(new MailboxAddress(from.DisplayName, from.Address));
@@ -70,6 +67,11 @@ namespace Streetwriters.Common.Services
             );
 
             await mailClient.SendAsync(message);
+
+            if (mailClient.IsConnected)
+            {
+                await mailClient.DisconnectAsync(true);
+            }
         }
 
         private async Task<MimeEntity> GetEmailBodyAsync(
@@ -129,10 +131,5 @@ namespace Streetwriters.Common.Services
             }
         }
 
-        async ValueTask IAsyncDisposable.DisposeAsync()
-        {
-            await mailClient.DisconnectAsync(true);
-            mailClient.Dispose();
-        }
     }
 }

Streetwriters.Identity/Controllers/AccountController.cs

@@ -116,6 +116,7 @@ namespace Streetwriters.Identity.Controllers
 
                         if (await UserManager.IsInRoleAsync(user, client.Id) && client.OnEmailConfirmed != null)
                         {
+                            logger.LogInformation("Email confirmed for user {UserId} on client {ClientId}. Triggering OnEmailConfirmed callback.", userId, client.Id);
                             await client.OnEmailConfirmed(userId);
                         }
 

cors-proxy/src/index.ts

@@ -279,8 +279,8 @@ function serveYouTubeEmbed(url: string) {
     </style>
 </head>
 <body>
-    <iframe src="${transformYouTubeUrl(
-      url,
+    <iframe src="${escapeHtmlAttr(
+      transformYouTubeUrl(url),
     )}" allow="accelerometer;autoplay;clipboard-write;encrypted-media;gyroscope;picture-in-picture;web-share" allowfullscreen referrerpolicy="strict-origin-when-cross-origin" title="Video player"></iframe>
 </body>
 </html>`;
@@ -299,6 +299,14 @@ function isYouTubeEmbed(urlString: string) {
   );
 }
 
+function escapeHtmlAttr(str: string): string {
+  return str
+    .replace(/&/g, "&amp;")
+    .replace(/"/g, "&quot;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;");
+}
+
 // Transform YouTube URLs to use youtube-nocookie.com for enhanced privacy
 function transformYouTubeUrl(urlString: string): string {
   try {
@@ -315,7 +323,7 @@ function transformYouTubeUrl(urlString: string): string {
       return url.toString();
     }
 
-    return urlString;
+    return url.toString();
   } catch {
     return urlString;
   }

docker-compose.yml

@@ -167,7 +167,6 @@ services:
       S3_SERVICE_URL: "${ATTACHMENTS_SERVER_PUBLIC_URL}"
       S3_REGION: "us-east-1"
       S3_BUCKET_NAME: "attachments"
-      NOTESNOOK_CORS: ${NOTESNOOK_CORS_ORIGINS:-}
 
   sse-server:
     image: streetwriters/sse:latest